From 5cdc03b31b94ddc60e6a806a33e0101bb3bb6ea0 Mon Sep 17 00:00:00 2001 From: kpamnany Date: Mon, 19 Jan 2015 18:06:43 +0530 Subject: [PATCH 0001/1938] Merging threads branch. --- Make.inc | 4 +- Make.user | 5 + base/cartesian.jl | 15 +- base/sysimg.jl | 3 + base/threading.jl | 91 +++++ src/Makefile | 7 +- src/ast.c | 6 +- src/builtins.c | 4 + src/cgutils.cpp | 2 +- src/codegen.cpp | 26 +- src/gc.c | 354 ++++++++++------- src/gf.c | 9 +- src/ia_misc.h | 87 +++++ src/init.c | 42 +- src/julia.h | 116 +++++- src/julia_internal.h | 16 +- src/options.h | 17 + src/task.c | 99 +++-- src/threadgroup.c | 236 ++++++++++++ src/threadgroup.h | 79 ++++ src/threading.c | 468 +++++++++++++++++++++++ src/threading.h | 96 +++++ test/perf/threads/laplace3d/README | 19 + test/perf/threads/laplace3d/laplace3d.c | 375 ++++++++++++++++++ test/perf/threads/laplace3d/laplace3d.jl | 133 +++++++ test/perf/threads/laplace3d/laplace3d.m | 36 ++ test/runtests.jl | 2 +- test/threads.jl | 49 +++ 28 files changed, 2191 insertions(+), 205 deletions(-) create mode 100644 Make.user create mode 100644 base/threading.jl create mode 100644 src/ia_misc.h create mode 100644 src/threadgroup.c create mode 100644 src/threadgroup.h create mode 100644 src/threading.c create mode 100644 src/threading.h create mode 100644 test/perf/threads/laplace3d/README create mode 100644 test/perf/threads/laplace3d/laplace3d.c create mode 100644 test/perf/threads/laplace3d/laplace3d.jl create mode 100644 test/perf/threads/laplace3d/laplace3d.m create mode 100644 test/threads.jl diff --git a/Make.inc b/Make.inc index 3f21eabb3901a..370950c7f8582 100644 --- a/Make.inc +++ b/Make.inc @@ -281,7 +281,7 @@ CC = $(CC_BASE) CXX = $(CXX_BASE) JCFLAGS = -std=gnu99 -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 JCPPFLAGS = -JCXXFLAGS = -pipe $(fPIC) -fno-rtti +JCXXFLAGS = -pipe $(fPIC) -fno-rtti -std=gnu++11 DEBUGFLAGS = -O0 -ggdb3 -DJL_DEBUG_BUILD -fstack-protector-all SHIPFLAGS = -O3 -ggdb3 -falign-functions endif @@ -293,7 +293,7 @@ CC = $(CC_BASE) CXX = $(CXX_BASE) JCFLAGS = -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 JCPPFLAGS = -JCXXFLAGS = -pipe $(fPIC) -fno-rtti +JCXXFLAGS = -pipe $(fPIC) -fno-rtti -std=gnu++11 DEBUGFLAGS = -O0 -g -DJL_DEBUG_BUILD -fstack-protector-all SHIPFLAGS = -O3 -g ifeq ($(OS), Darwin) diff --git a/Make.user b/Make.user new file mode 100644 index 0000000000000..69a5cadd04efa --- /dev/null +++ b/Make.user @@ -0,0 +1,5 @@ +CC = clang +CFLAGS = -ftls-model=global-dynamic +CXX = clang++ +LLVM_VER = svn +LLVM_GIT_URL_LLVM = git@github.com:JuliaLang/llvm.git -b jl/thread-local-storage diff --git a/base/cartesian.jl b/base/cartesian.jl index 39e293929cb77..06c7aa24d4230 100644 --- a/base/cartesian.jl +++ b/base/cartesian.jl @@ -1,6 +1,6 @@ module Cartesian -export @ngenerate, @nsplat, @nloops, @nref, @ncall, @nexprs, @nextract, @nall, @ntuple, @nif, ngenerate +export @ngenerate, @nsplat, @nloops, @nref, @ncall, @nexprs, @nextract, @nall, @nany, @ntuple, @nif, ngenerate const CARTESIAN_DIMS = 4 @@ -365,6 +365,19 @@ function _nall(N::Int, criterion::Expr) Expr(:&&, conds...) end +# Check whether any of variables i1, i2, ... satisfy criterion +macro nany(N, criterion) + _nany(N, criterion) +end + +function _nany(N::Int, criterion::Expr) + if criterion.head != :-> + error("Second argument must be an anonymous function expression yielding the criterion") + end + conds = [Expr(:escape, inlineanonymous(criterion, i)) for i = 1:N] + Expr(:||, conds...) +end + macro ntuple(N, ex) _ntuple(N, ex) end diff --git a/base/sysimg.jl b/base/sysimg.jl index 868885f62d075..b8ee1bacd5e55 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -289,6 +289,9 @@ import .Dates: Date, DateTime, now # Some basic documentation include("basedocs.jl") +# threading +include("threading.jl") + function __init__() # Base library init reinit_stdio() diff --git a/base/threading.jl b/base/threading.jl new file mode 100644 index 0000000000000..cbea2ebbab83f --- /dev/null +++ b/base/threading.jl @@ -0,0 +1,91 @@ +module Threading + +export threadid, maxthreads, nthreads, @threads + +threadid() = int(ccall(:jl_threadid, Int16, ())+1) +maxthreads() = int(unsafe_load(cglobal(:jl_max_threads, Cint))) +nthreads() = int(unsafe_load(cglobal(:jl_n_threads, Cint))) + +function _threadsfor(forexpr) + fun = gensym("_threadsfor") + lidx = forexpr.args[1].args[1] # index + lf = forexpr.args[1].args[2].args[1] # first + ll = forexpr.args[1].args[2].args[2] # last + lbody = forexpr.args[2] # body + quote + function $fun() + tid = threadid() + # divide loop iterations among threads + len, rem = divrem($ll-$lf+1, nthreads()) + # not enough iterations for all the threads? + if len == 0 + if tid > rem + return + end + len, rem = 1, 0 + end + # compute this thread's range + f = $lf + ((tid-1) * len) + l = f + len - 1 + # distribute remaining iterations evenly + if rem > 0 + if tid <= rem + f = f + (tid-1) + l = l + tid + else + f = f + rem + l = l + rem + end + end + # run this thread's iterations + for $(esc(lidx)) = f:l + $(esc(lbody)) + end + end + ccall(:jl_threading_run, Void, (Any, Any), $fun, ()) + end +end + +function _threadsblock(blk) + fun = gensym("_threadsblock") + esc(quote + function $fun() + $blk + end + ccall(:jl_threading_run, Void, (Any, Any), $fun, ()) + end) +end + +function _threadscall(callexpr) + fun = callexpr.args[1] + esc(quote + ccall(:jl_threading_run, Void, (Any, Any), $fun, $(Expr(:tuple, callexpr.args[2:end]...))) + end) +end + +macro threads(args...) + na = length(args) + if na != 2 + throw(ArgumentError("wrong number of arguments in @threads")) + end + tg = args[1] + if !is(tg, :all) + throw(ArgumentError("only 'all' supported as thread group for @threads")) + end + ex = args[2] + if !isa(ex, Expr) + throw(ArgumentError("need an expression argument to @threads")) + end + if is(ex.head, :for) + return _threadsfor(ex) + elseif is(ex.head, :block) + return _threadsblock(ex) + elseif is(ex.head, :call) + return _threadscall(ex) + else + throw(ArgumentError("unrecognized argument to @threads")) + end +end + +end # module + diff --git a/src/Makefile b/src/Makefile index 58caeb708fd54..bd0f07642b15f 100644 --- a/src/Makefile +++ b/src/Makefile @@ -10,7 +10,8 @@ BUILDDIR ?= . SRCS = \ jltypes gf ast builtins module codegen disasm debuginfo interpreter \ - alloc dlload sys init task array dump toplevel jl_uv jlapi profile llvm-simdloop + alloc dlload sys init task array dump toplevel jl_uv jlapi profile \ + llvm-simdloop threadgroup threading HEADERS = julia.h julia_internal.h julia_version.h options.h $(wildcard support/*.h) $(LIBUV_INC)/uv.h @@ -59,9 +60,9 @@ $(BUILDDIR)/%.o: %.c $(HEADERS) | $(BUILDDIR) $(BUILDDIR)/%.dbg.obj: %.c $(HEADERS) | $(BUILDDIR) @$(call PRINT_CC, $(CC) $(CPPFLAGS) $(CFLAGS) $(DEBUGFLAGS) -c $< -o $@) $(BUILDDIR)/%.o: %.cpp $(HEADERS) | $(BUILDDIR) - @$(call PRINT_CC, $(CXX) $(call exec,$(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(SHIPFLAGS) -c $< -o $@) + @$(call PRINT_CC, $(CXX) $(call exec,$(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(SHIPFLAGS) -c $< -o $@) -std=c++0x $(BUILDDIR)/%.dbg.obj: %.cpp $(HEADERS) | $(BUILDDIR) - @$(call PRINT_CC, $(CXX) $(call exec,$(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(DEBUGFLAGS) -c $< -o $@) + @$(call PRINT_CC, $(CXX) $(call exec,$(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(DEBUGFLAGS) -c $< -o $@) -std=c++0x $(BUILDDIR)/julia_flisp.boot.inc: $(BUILDDIR)/julia_flisp.boot $(BUILDDIR)/flisp/libflisp.a @$(call PRINT_FLISP, $(call spawn,$(BUILDDIR)/flisp/flisp) ./bin2hex.scm < $< > $@) diff --git a/src/ast.c b/src/ast.c index 767d9bd71151e..42c8ede3378fc 100644 --- a/src/ast.c +++ b/src/ast.c @@ -902,9 +902,9 @@ jl_value_t *jl_prepare_ast(jl_lambda_info_t *li, jl_tuple_t *sparams) return ast; } -DLLEXPORT int jl_is_operator(char *sym) { - return fl_applyn(1, symbol_value(symbol("operator?")), symbol(sym)) - == FL_T; +DLLEXPORT int jl_is_operator(char *sym) +{ + return fl_applyn(1, symbol_value(symbol("operator?")), symbol(sym)) == FL_T; } DLLEXPORT int jl_operator_precedence(char *sym) { diff --git a/src/builtins.c b/src/builtins.c index 5fe62fd024b51..cb1c472c7f824 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -850,8 +850,11 @@ extern int jl_in_inference; extern int jl_boot_file_loaded; int jl_eval_with_compiler_p(jl_expr_t *expr, int compileloops); +JL_DEFINE_MUTEX_EXT(codegen) + void jl_trampoline_compile_function(jl_function_t *f, int always_infer, jl_tuple_t *sig) { + JL_LOCK(codegen) assert(f->linfo != NULL); // to run inference on all thunks. slows down loading files. // NOTE: if this call to inference is removed, type_annotate in inference.jl @@ -874,6 +877,7 @@ void jl_trampoline_compile_function(jl_function_t *f, int always_infer, jl_tuple if (jl_boot_file_loaded && jl_is_expr(f->linfo->ast)) { f->linfo->ast = jl_compress_ast(f->linfo, f->linfo->ast); } + JL_UNLOCK(codegen) } JL_CALLABLE(jl_trampoline) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index d13e115139178..5a83449563f9d 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -25,7 +25,7 @@ static GlobalVariable *prepare_global(GlobalVariable *G) if (!gv) { gv = new GlobalVariable(*jl_Module, G->getType()->getElementType(), G->isConstant(), GlobalVariable::ExternalLinkage, - NULL, G->getName()); + NULL, G->getName(), NULL, G->getThreadLocalMode()); } return gv; } diff --git a/src/codegen.cpp b/src/codegen.cpp index a9f2e388787ce..3119093789e47 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -413,7 +413,8 @@ void jl_dump_objfile(char *fname, int jit_model) #else jit_model ? Reloc::PIC_ : Reloc::Default, #endif - jit_model ? CodeModel::JITDefault : CodeModel::Default, + // jit_model ? CodeModel::JITDefault : CodeModel::Default, + CodeModel::Default, CodeGenOpt::Aggressive // -O3 )); @@ -538,12 +539,15 @@ static void jl_rethrow_with_add(const char *fmt, ...) jl_rethrow(); } +JL_DEFINE_MUTEX_EXT(codegen) + // --- entry point --- //static int n_emit=0; static Function *emit_function(jl_lambda_info_t *lam, bool cstyle); //static int n_compile=0; static Function *to_function(jl_lambda_info_t *li, bool cstyle) { + JL_LOCK(codegen) JL_SIGATOMIC_BEGIN(); assert(!li->inInference); BasicBlock *old = nested_compile ? builder.GetInsertBlock() : NULL; @@ -565,6 +569,7 @@ static Function *to_function(jl_lambda_info_t *li, bool cstyle) builder.SetCurrentDebugLocation(olddl); } JL_SIGATOMIC_END(); + JL_UNLOCK(codegen) jl_rethrow_with_add("error compiling %s", li->name->name); } assert(f != NULL); @@ -598,6 +603,7 @@ static Function *to_function(jl_lambda_info_t *li, bool cstyle) builder.SetCurrentDebugLocation(olddl); } JL_SIGATOMIC_END(); + JL_UNLOCK(codegen) return f; } @@ -632,6 +638,7 @@ static void jl_setup_module(Module *m, bool add) extern "C" void jl_generate_fptr(jl_function_t *f) { + JL_LOCK(codegen) // objective: assign li->fptr jl_lambda_info_t *li = f->linfo; assert(li->functionObject); @@ -672,6 +679,7 @@ extern "C" void jl_generate_fptr(jl_function_t *f) } } f->fptr = li->fptr; + JL_UNLOCK(codegen) } extern "C" void jl_compile(jl_function_t *f) @@ -795,6 +803,8 @@ const jl_value_t *jl_dump_llvmf(void *f, bool dumpasm) llvm::formatted_raw_ostream fstream(stream); Function *llvmf = (Function*)f; if (dumpasm == false) { + // To print whole module + //llvmf->getParent()->print(stream, NULL); llvmf->print(stream); } else { @@ -4400,10 +4410,18 @@ static void init_julia_llvm_env(Module *m) jlpgcstack_var = new GlobalVariable(*m, jl_ppvalue_llvmt, false, GlobalVariable::ExternalLinkage, - NULL, "jl_pgcstack"); + NULL, "jl_pgcstack", NULL, + GlobalValue::GeneralDynamicTLSModel); add_named_global(jlpgcstack_var, (void*)&jl_pgcstack); #endif + jlexc_var = + new GlobalVariable(*m, jl_pvalue_llvmt, + false, GlobalVariable::ExternalLinkage, + NULL, "jl_exception_in_transit", NULL, + GlobalValue::GeneralDynamicTLSModel); + add_named_global(jlexc_var, (void*)&jl_exception_in_transit); + global_to_llvm("__stack_chk_guard", (void*)&__stack_chk_guard, m); Function *jl__stack_chk_fail = Function::Create(FunctionType::get(T_void, false), @@ -4415,8 +4433,6 @@ static void init_julia_llvm_env(Module *m) jltrue_var = global_to_llvm("jl_true", (void*)&jl_true, m); jlfalse_var = global_to_llvm("jl_false", (void*)&jl_false, m); jlnull_var = global_to_llvm("jl_null", (void*)&jl_null, m); - jlexc_var = global_to_llvm("jl_exception_in_transit", - (void*)&jl_exception_in_transit, m); jldiverr_var = global_to_llvm("jl_diverror_exception", (void*)&jl_diverror_exception, m); jlundeferr_var = global_to_llvm("jl_undefref_exception", @@ -4991,6 +5007,8 @@ extern "C" void jl_init_codegen(void) #endif .setTargetOptions(options) #if defined(USE_MCJIT) && !defined(LLVM36) + .setCodeModel(CodeModel::Default) + .setRelocationModel(Reloc::PIC_) .setUseMCJIT(true) #endif ; diff --git a/src/gc.c b/src/gc.c index 42b6bbad33aff..64dea54b531e7 100644 --- a/src/gc.c +++ b/src/gc.c @@ -20,6 +20,7 @@ #endif #include "julia.h" #include "julia_internal.h" +#include "threading.h" #ifdef _P64 # ifdef USE_MMAP @@ -78,8 +79,8 @@ typedef struct _bigval_t { } bigval_t; // GC knobs and self-measurement variables -static size_t allocd_bytes = 0; -static int64_t total_allocd_bytes = 0; +static volatile size_t allocd_bytes = 0; +static volatile int64_t total_allocd_bytes = 0; static int64_t last_gc_total_bytes = 0; static size_t freed_bytes = 0; static uint64_t total_gc_time=0; @@ -91,7 +92,7 @@ static size_t max_collect_interval = 1250000000UL; static size_t max_collect_interval = 500000000UL; #endif static size_t collect_interval = default_collect_interval; -int jl_in_gc; // referenced from switchto task.c +int jl_in_gc=0; // referenced from switchto task.c #ifdef OBJPROFILE static htable_t obj_counts; @@ -108,6 +109,27 @@ static size_t total_freed_bytes=0; #define gc_setmark_buf(o) gc_setmark(gc_val_buf(o)) #define gc_typeof(v) ((jl_value_t*)(((uptrint_t)jl_typeof(v))&~1UL)) +typedef struct _mallocarray_t { + jl_array_t *a; + struct _mallocarray_t *next; +} mallocarray_t; + +#define N_POOLS 42 + +// per-thread heaps +typedef struct _jl_thread_heap_t { + arraylist_t preserved_values; + arraylist_t weak_refs; + htable_t finalizer_table; + bigval_t *big_objects; + mallocarray_t *mallocarrays; + mallocarray_t *mafreelist; + pool_t norm_pools[N_POOLS]; + pool_t ephe_pools[N_POOLS]; + pool_t *pools; +} jl_thread_heap_t; + + // malloc wrappers, aligned allocation #ifdef _P64 @@ -138,7 +160,7 @@ DLLEXPORT void *jl_gc_counted_malloc(size_t sz) { if (allocd_bytes > collect_interval) jl_gc_collect(); - allocd_bytes += sz; + JL_ATOMIC_FETCH_AND_ADD(allocd_bytes,sz); void *b = malloc(sz); if (b == NULL) jl_throw(jl_memory_exception); @@ -148,7 +170,18 @@ DLLEXPORT void *jl_gc_counted_malloc(size_t sz) DLLEXPORT void jl_gc_counted_free(void *p, size_t sz) { free(p); - freed_bytes += sz; + JL_ATOMIC_FETCH_AND_ADD(freed_bytes,sz); +} + +DLLEXPORT void *jl_gc_counted_realloc(void *p, size_t sz) +{ + if (allocd_bytes > collect_interval) + jl_gc_collect(); + JL_ATOMIC_FETCH_AND_ADD(allocd_bytes,((sz+1)/2)); // NOTE: wild guess at growth amount + void *b = realloc(p, sz); + if (b == NULL) + jl_throw(jl_memory_exception); + return b; } DLLEXPORT void *jl_gc_counted_realloc_with_old_size(void *p, size_t old, size_t sz) @@ -156,7 +189,7 @@ DLLEXPORT void *jl_gc_counted_realloc_with_old_size(void *p, size_t old, size_t if (allocd_bytes > collect_interval) jl_gc_collect(); if (sz > old) - allocd_bytes += (sz-old); + JL_ATOMIC_FETCH_AND_ADD(allocd_bytes,sz-old); void *b = realloc(p, sz); if (b == NULL) jl_throw(jl_memory_exception); @@ -171,7 +204,7 @@ void *jl_gc_managed_malloc(size_t sz) void *b = malloc_a16(sz); if (b == NULL) jl_throw(jl_memory_exception); - allocd_bytes += sz; + JL_ATOMIC_FETCH_AND_ADD(allocd_bytes,sz); return b; } @@ -200,47 +233,43 @@ void *jl_gc_managed_realloc(void *d, size_t sz, size_t oldsz, int isaligned) #endif if (b == NULL) jl_throw(jl_memory_exception); - allocd_bytes += sz; + JL_ATOMIC_FETCH_AND_ADD(allocd_bytes,sz); return b; } // preserved values -static arraylist_t preserved_values; - DLLEXPORT int jl_gc_n_preserved_values(void) { - return preserved_values.len; + return jl_thread_heap->preserved_values.len; } DLLEXPORT void jl_gc_preserve(jl_value_t *v) { - arraylist_push(&preserved_values, (void*)v); + arraylist_push(&jl_thread_heap->preserved_values, (void*)v); } DLLEXPORT void jl_gc_unpreserve(void) { - (void)arraylist_pop(&preserved_values); + (void)arraylist_pop(&jl_thread_heap->preserved_values); } // weak references -static arraylist_t weak_refs; - DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value) { jl_weakref_t *wr = (jl_weakref_t*)alloc_2w(); wr->type = (jl_value_t*)jl_weakref_type; wr->value = value; - arraylist_push(&weak_refs, wr); + arraylist_push(&jl_thread_heap->weak_refs, wr); return wr; } -static void sweep_weak_refs(void) +static void sweep_weak_refs(jl_thread_heap_t *h) { - size_t n=0, ndel=0, l=weak_refs.len; + size_t n=0, ndel=0, l=h->weak_refs.len; jl_weakref_t *wr; - void **lst = weak_refs.items; + void **lst = h->weak_refs.items; void *tmp; #define SWAP_wr(a,b) (tmp=a,a=b,b=tmp,1) if (l == 0) @@ -258,12 +287,11 @@ static void sweep_weak_refs(void) } } while ((n < l-ndel) && SWAP_wr(lst[n],lst[n+ndel])); - weak_refs.len -= ndel; + h->weak_refs.len -= ndel; } // finalization -static htable_t finalizer_table; static arraylist_t to_finalize; static void schedule_finalization(void *o) @@ -300,9 +328,9 @@ static int finalize_object(jl_value_t *o) jl_value_t *ff = NULL; int success = 0; JL_GC_PUSH1(&ff); - ff = (jl_value_t*)ptrhash_get(&finalizer_table, o); + ff = (jl_value_t*)ptrhash_get(&jl_thread_heap->finalizer_table, o); if (ff != HT_NOTFOUND) { - ptrhash_remove(&finalizer_table, o); + ptrhash_remove(&jl_thread_heap->finalizer_table, o); run_finalizer((jl_value_t*)o, ff); success = 1; } @@ -313,21 +341,33 @@ static int finalize_object(jl_value_t *o) static void run_finalizers(void) { void *o = NULL; - JL_GC_PUSH1(&o); + jl_value_t *ff = NULL; + int t; + JL_GC_PUSH2(&o, &ff); while (to_finalize.len > 0) { o = arraylist_pop(&to_finalize); - int ok = finalize_object((jl_value_t*)o); - assert(ok); (void)ok; + t = -1; + do { + t++; + ff = (jl_value_t*)ptrhash_get(&jl_all_heaps[t]->finalizer_table, o); + } while (ff == HT_NOTFOUND); + //assert(ff != HT_NOTFOUND); + assert(t < jl_n_threads); + ptrhash_remove(&jl_all_heaps[t]->finalizer_table, o); + run_finalizer((jl_value_t*)o, ff); } JL_GC_POP(); } void jl_gc_run_all_finalizers(void) { - for(size_t i=0; i < finalizer_table.size; i+=2) { - jl_value_t *f = (jl_value_t*)finalizer_table.table[i+1]; - if (f != HT_NOTFOUND && !jl_is_cpointer(f)) { - schedule_finalization(finalizer_table.table[i]); + for(int t=0; t < jl_n_threads; t++) { + htable_t *ft = &jl_all_heaps[t]->finalizer_table; + for(size_t i=0; i < ft->size; i+=2) { + jl_value_t *f = (jl_value_t*)ft->table[i+1]; + if (f != HT_NOTFOUND && !jl_is_cpointer(f)) { + schedule_finalization(ft->table[i]); + } } } run_finalizers(); @@ -335,13 +375,11 @@ void jl_gc_run_all_finalizers(void) void jl_gc_add_finalizer(jl_value_t *v, jl_function_t *f) { - jl_value_t **bp = (jl_value_t**)ptrhash_bp(&finalizer_table, v); - if (*bp == HT_NOTFOUND) { + jl_value_t **bp = (jl_value_t**)ptrhash_bp(&jl_thread_heap->finalizer_table, v); + if (*bp == HT_NOTFOUND) *bp = (jl_value_t*)f; - } - else { + else *bp = (jl_value_t*)jl_tuple2((jl_value_t*)f, *bp); - } } void jl_finalize(jl_value_t *o) @@ -351,8 +389,6 @@ void jl_finalize(jl_value_t *o) // big value list -static bigval_t *big_objects = NULL; - static void *alloc_big(size_t sz) { if (allocd_bytes > collect_interval) @@ -362,7 +398,7 @@ static void *alloc_big(size_t sz) jl_throw(jl_memory_exception); size_t allocsz = (sz+offs+15) & -16; bigval_t *v = (bigval_t*)malloc_a16(allocsz); - allocd_bytes += allocsz; + JL_ATOMIC_FETCH_AND_ADD(allocd_bytes,allocsz); if (v == NULL) jl_throw(jl_memory_exception); #ifdef MEMDEBUG @@ -370,15 +406,15 @@ static void *alloc_big(size_t sz) #endif v->sz = sz; v->flags = 0; - v->next = big_objects; - big_objects = v; + v->next = jl_thread_heap->big_objects; + jl_thread_heap->big_objects = v; return &v->_data[0]; } -static void sweep_big(void) +static void sweep_big(jl_thread_heap_t *h) { - bigval_t *v = big_objects; - bigval_t **pv = &big_objects; + bigval_t *v = h->big_objects; + bigval_t **pv = &h->big_objects; while (v != NULL) { bigval_t *nxt = v->next; if (v->marked) { @@ -399,27 +435,19 @@ static void sweep_big(void) // tracking Arrays with malloc'd storage -typedef struct _mallocarray_t { - jl_array_t *a; - struct _mallocarray_t *next; -} mallocarray_t; - -static mallocarray_t *mallocarrays = NULL; -static mallocarray_t *mafreelist = NULL; - void jl_gc_track_malloced_array(jl_array_t *a) { mallocarray_t *ma; - if (mafreelist == NULL) { + if (jl_thread_heap->mafreelist == NULL) { ma = (mallocarray_t*)malloc(sizeof(mallocarray_t)); } else { - ma = mafreelist; - mafreelist = mafreelist->next; + ma = jl_thread_heap->mafreelist; + jl_thread_heap->mafreelist = jl_thread_heap->mafreelist->next; } ma->a = a; - ma->next = mallocarrays; - mallocarrays = ma; + ma->next = jl_thread_heap->mallocarrays; + jl_thread_heap->mallocarrays = ma; } static size_t array_nbytes(jl_array_t *a) @@ -442,10 +470,10 @@ void jl_gc_free_array(jl_array_t *a) } } -static void sweep_malloced_arrays() +static void sweep_malloced_arrays(jl_thread_heap_t *h) { - mallocarray_t *ma = mallocarrays; - mallocarray_t **pma = &mallocarrays; + mallocarray_t *ma = h->mallocarrays; + mallocarray_t **pma = &h->mallocarrays; while (ma != NULL) { mallocarray_t *nxt = ma->next; if (gc_marked(ma->a)) { @@ -455,8 +483,8 @@ static void sweep_malloced_arrays() *pma = nxt; assert(ma->a->how == 2); jl_gc_free_array(ma->a); - ma->next = mafreelist; - mafreelist = ma; + ma->next = h->mafreelist; + h->mafreelist = ma; } ma = nxt; } @@ -464,11 +492,6 @@ static void sweep_malloced_arrays() // pool allocation -#define N_POOLS 42 -static pool_t norm_pools[N_POOLS]; -static pool_t ephe_pools[N_POOLS]; -static pool_t *pools = &norm_pools[0]; - static void add_page(pool_t *p) { #ifdef USE_MMAP @@ -500,10 +523,9 @@ static inline void *pool_alloc(pool_t *p) { if (allocd_bytes > collect_interval) jl_gc_collect(); - allocd_bytes += p->osize; - if (p->freelist == NULL) { + JL_ATOMIC_FETCH_AND_ADD(allocd_bytes,p->osize); + if (p->freelist == NULL) add_page(p); - } assert(p->freelist != NULL); gcval_t *v = p->freelist; p->freelist = p->freelist->next; @@ -619,12 +641,15 @@ extern void jl_unmark_symbols(void); static void gc_sweep(void) { - sweep_malloced_arrays(); - sweep_big(); - int i; - for(i=0; i < N_POOLS; i++) { - sweep_pool(&norm_pools[i]); - sweep_pool(&ephe_pools[i]); + int t, i; + for(t=0; t < jl_n_threads; t++) { + jl_thread_heap_t *h = jl_all_heaps[t]; + sweep_malloced_arrays(h); + sweep_big(h); + for(i=0; i < N_POOLS; i++) { + sweep_pool(&h->norm_pools[i]); + sweep_pool(&h->ephe_pools[i]); + } } jl_unmark_symbols(); } @@ -709,7 +734,8 @@ static void gc_mark_task(jl_task_t *ta, int d) ptrint_t offset; if (ta == jl_current_task) { offset = 0; - gc_mark_stack(jl_pgcstack, offset, d); + for(int t=0; t < jl_n_threads; t++) + gc_mark_stack(*jl_all_pgcstacks[t], offset, d); } else { offset = (char *)ta->stkbuf - ((char *)jl_stackbase - ta->ssize); @@ -830,7 +856,6 @@ static void visit_mark_stack() void jl_mark_box_caches(void); -extern jl_value_t * volatile jl_task_arg_in_transit; #if defined(GCTIME) || defined(GC_FINAL_STATS) double clock_now(void); #endif @@ -841,11 +866,16 @@ extern jl_array_t *jl_module_init_order; static void gc_mark(void) { + int t; // mark all roots // active tasks - gc_push_root(jl_root_task, 0); - gc_push_root(jl_current_task, 0); + for(t=0; t < jl_n_threads; t++) { + gc_push_root(*jl_all_task_states[t].proot_task, 0); + gc_push_root(*jl_all_task_states[t].pcurrent_task, 0); + gc_push_root(*jl_all_task_states[t].pexception_in_transit, 0); + gc_push_root(*jl_all_task_states[t].ptask_arg_in_transit, 0); + } // modules gc_push_root(jl_main_module, 0); @@ -855,8 +885,6 @@ static void gc_mark(void) // invisible builtin values if (jl_an_empty_cell) gc_push_root(jl_an_empty_cell, 0); - gc_push_root(jl_exception_in_transit, 0); - gc_push_root(jl_task_arg_in_transit, 0); gc_push_root(jl_unprotect_stack_func, 0); gc_push_root(jl_bottom_func, 0); gc_push_root(jl_typetype_type, 0); @@ -875,8 +903,11 @@ static void gc_mark(void) size_t i; // stuff randomly preserved - for(i=0; i < preserved_values.len; i++) { - gc_push_root((jl_value_t*)preserved_values.items[i], 0); + for(t=0; t < jl_n_threads; t++) { + arraylist_t *pv = &jl_all_heaps[t]->preserved_values; + for(i=0; i < pv->len; i++) { + gc_push_root((jl_value_t*)pv->items[i], 0); + } } // objects currently being finalized @@ -888,22 +919,25 @@ static void gc_mark(void) // find unmarked objects that need to be finalized. // this must happen last. - for(i=0; i < finalizer_table.size; i+=2) { - if (finalizer_table.table[i+1] != HT_NOTFOUND) { - jl_value_t *v = (jl_value_t*)finalizer_table.table[i]; - if (!gc_marked(v)) { - jl_value_t *fin = (jl_value_t*)finalizer_table.table[i+1]; - if (gc_typeof(fin) == (jl_value_t*)jl_voidpointer_type) { - void *p = ((void**)fin)[1]; - if (p) - ((void (*)(void*))p)(jl_data_ptr(v)); - finalizer_table.table[i+1] = HT_NOTFOUND; - continue; + for(t=0; t < jl_n_threads; t++) { + htable_t *ft = &jl_all_heaps[t]->finalizer_table; + for(i=0; i < ft->size; i+=2) { + if (ft->table[i+1] != HT_NOTFOUND) { + jl_value_t *v = (jl_value_t*)ft->table[i]; + if (!gc_marked(v)) { + jl_value_t *fin = (jl_value_t*)ft->table[i+1]; + if (gc_typeof(fin) == (jl_value_t*)jl_voidpointer_type) { + void *p = ((void**)fin)[1]; + if (p) + ((void (*)(void*))p)(jl_data_ptr(v)); + ft->table[i+1] = HT_NOTFOUND; + continue; + } + gc_push_root(v, 0); + schedule_finalization(v); } - gc_push_root(v, 0); - schedule_finalization(v); + gc_push_root(ft->table[i+1], 0); } - gc_push_root(finalizer_table.table[i+1], 0); } } @@ -929,8 +963,8 @@ int64_t diff_gc_total_bytes(void) } void sync_gc_total_bytes(void) {last_gc_total_bytes = jl_gc_total_bytes();} -void jl_gc_ephemeral_on(void) { pools = &ephe_pools[0]; } -void jl_gc_ephemeral_off(void) { pools = &norm_pools[0]; } +void jl_gc_ephemeral_on(void) { jl_thread_heap->pools = &jl_thread_heap->ephe_pools[0]; } +void jl_gc_ephemeral_off(void) { jl_thread_heap->pools = &jl_thread_heap->norm_pools[0]; } #if defined(MEMPROFILE) static void all_pool_stats(void); @@ -952,12 +986,27 @@ static void print_obj_profile(void) void jl_gc_collect(void) { - size_t actual_allocd = allocd_bytes; + if (!is_gc_enabled) { + allocd_bytes = 0; + return; + } + + ti_threadgroup_barrier(tgworld, ti_tid); + + if (ti_tid != 0) { + ti_threadgroup_barrier(tgworld, ti_tid); + return; + } + + jl_in_gc = 1; + + size_t actual_allocd; + + gc_collect_top: + actual_allocd = allocd_bytes; total_allocd_bytes += allocd_bytes; - allocd_bytes = 0; if (is_gc_enabled) { JL_SIGATOMIC_BEGIN(); - jl_in_gc = 1; uint64_t t0 = jl_hrtime(); gc_mark(); #ifdef GCTIME @@ -970,14 +1019,18 @@ void jl_gc_collect(void) #ifdef GCTIME uint64_t t1 = jl_hrtime(); #endif - sweep_weak_refs(); + int t; + for(t=0; t < jl_n_threads; t++) + sweep_weak_refs(jl_all_heaps[t]); gc_sweep(); #ifdef GCTIME JL_PRINTF(JL_STDERR, "sweep time %.3f ms\n", (jl_hrtime()-t1)*1.0e6); #endif int nfinal = to_finalize.len; - run_finalizers(); jl_in_gc = 0; + jl_gc_disable(); + run_finalizers(); + jl_gc_enable(); JL_SIGATOMIC_END(); total_gc_time += (jl_hrtime()-t0); #if defined(GC_FINAL_STATS) @@ -1005,8 +1058,14 @@ void jl_gc_collect(void) // if a lot of objects were finalized, re-run GC to finish freeing // their storage if possible. if (nfinal > 100000) - jl_gc_collect(); + goto gc_collect_top; + } + else { + jl_in_gc = 0; } + + allocd_bytes = 0; + ti_threadgroup_barrier(tgworld, ti_tid); } // allocator entry points @@ -1022,7 +1081,7 @@ void *allocb(size_t sz) b = alloc_big(sz); } else { - b = pool_alloc(&pools[szclass(sz)]); + b = pool_alloc(&jl_thread_heap->pools[szclass(sz)]); } #endif return (void*)((void**)b + 1); @@ -1035,7 +1094,7 @@ DLLEXPORT void *allocobj(size_t sz) #endif if (sz > 2048) return alloc_big(sz); - return pool_alloc(&pools[szclass(sz)]); + return pool_alloc(&jl_thread_heap->pools[szclass(sz)]); } DLLEXPORT void *alloc_2w(void) @@ -1044,9 +1103,9 @@ DLLEXPORT void *alloc_2w(void) return alloc_big(2*sizeof(void*)); #endif #ifdef _P64 - return pool_alloc(&pools[2]); + return pool_alloc(&jl_thread_heap->pools[2]); #else - return pool_alloc(&pools[0]); + return pool_alloc(&jl_thread_heap->pools[0]); #endif } @@ -1056,9 +1115,9 @@ DLLEXPORT void *alloc_3w(void) return alloc_big(3*sizeof(void*)); #endif #ifdef _P64 - return pool_alloc(&pools[4]); + return pool_alloc(&jl_thread_heap->pools[4]); #else - return pool_alloc(&pools[1]); + return pool_alloc(&jl_thread_heap->pools[1]); #endif } @@ -1068,9 +1127,9 @@ DLLEXPORT void *alloc_4w(void) return alloc_big(4*sizeof(void*)); #endif #ifdef _P64 - return pool_alloc(&pools[6]); + return pool_alloc(&jl_thread_heap->pools[6]); #else - return pool_alloc(&pools[2]); + return pool_alloc(&jl_thread_heap->pools[2]); #endif } @@ -1093,33 +1152,46 @@ void jl_print_gc_stats(JL_STREAM *s) // initialization -void jl_gc_init(void) +jl_thread_heap_t *jl_mk_thread_heap(void) { - int szc[N_POOLS] = { 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, - 64, 72, 80, 88, 96, //#=18 + static int szc[N_POOLS] = { 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, + 64, 72, 80, 88, 96, //#=18 + + 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, - 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, + 288, 320, 352, 384, 416, 448, 480, 512, - 288, 320, 352, 384, 416, 448, 480, 512, + 640, 768, 896, 1024, - 640, 768, 896, 1024, + 1536, 2048 }; + jl_thread_heap_t *h = malloc(sizeof(jl_thread_heap_t)); - 1536, 2048 }; int i; for(i=0; i < N_POOLS; i++) { - norm_pools[i].osize = szc[i]; - norm_pools[i].pages = NULL; - norm_pools[i].freelist = NULL; + h->norm_pools[i].osize = szc[i]; + h->norm_pools[i].pages = NULL; + h->norm_pools[i].freelist = NULL; - ephe_pools[i].osize = szc[i]; - ephe_pools[i].pages = NULL; - ephe_pools[i].freelist = NULL; + h->ephe_pools[i].osize = szc[i]; + h->ephe_pools[i].pages = NULL; + h->ephe_pools[i].freelist = NULL; } - htable_new(&finalizer_table, 0); + h->big_objects = NULL; + h->mallocarrays = NULL; + h->mafreelist = NULL; + h->pools = &h->norm_pools[0]; + + htable_new(&h->finalizer_table, 0); + arraylist_new(&h->preserved_values, 0); + arraylist_new(&h->weak_refs, 0); + + return h; +} + +void jl_gc_init(void) +{ arraylist_new(&to_finalize, 0); - arraylist_new(&preserved_values, 0); - arraylist_new(&weak_refs, 0); #ifdef OBJPROFILE htable_new(&obj_counts, 0); @@ -1176,18 +1248,20 @@ static size_t pool_stats(pool_t *p, size_t *pwaste) static void all_pool_stats(void) { - int i; + int i,j; size_t nb=0, w, tw=0, no=0, b; - for(i=0; i < N_POOLS; i++) { - b = pool_stats(&norm_pools[i], &w); - nb += b; - no += (b/norm_pools[i].osize); - tw += w; - - b = pool_stats(&ephe_pools[i], &w); - nb += b; - no += (b/ephe_pools[i].osize); - tw += w; + for(j=0; j < N_GC_THREADS; j++) { + for(i=0; i < N_POOLS; i++) { + b = pool_stats(&norm_pools[j][i], &w); + nb += b; + no += (b/norm_pools[j][i].osize); + tw += w; + + b = pool_stats(&ephe_pools[j][i], &w); + nb += b; + no += (b/ephe_pools[j][i].osize); + tw += w; + } } JL_PRINTF(JL_STDOUT, "%d objects, %d total allocated, %d total fragments\n", diff --git a/src/gf.c b/src/gf.c index bac10b160fb7e..c393ee55e66f1 100644 --- a/src/gf.c +++ b/src/gf.c @@ -382,9 +382,11 @@ extern jl_function_t *jl_typeinf_func; can be equal to "li" if not applicable. */ int jl_in_inference = 0; +JL_DEFINE_MUTEX_EXT(codegen) void jl_type_infer(jl_lambda_info_t *li, jl_tuple_t *argtypes, jl_lambda_info_t *def) { + JL_LOCK(codegen) int last_ii = jl_in_inference; jl_in_inference = 1; if (jl_typeinf_func != NULL) { @@ -411,6 +413,7 @@ void jl_type_infer(jl_lambda_info_t *li, jl_tuple_t *argtypes, li->inInference = 0; } jl_in_inference = last_ii; + JL_UNLOCK(codegen) } static jl_value_t *nth_slot_type(jl_tuple_t *sig, size_t i) @@ -471,6 +474,7 @@ static jl_function_t *cache_method(jl_methtable_t *mt, jl_tuple_t *type, jl_function_t *method, jl_tuple_t *decl, jl_tuple_t *sparams) { + JL_LOCK(codegen) size_t i; int need_guard_entries = 0; jl_value_t *temp=NULL; @@ -814,6 +818,7 @@ static jl_function_t *cache_method(jl_methtable_t *mt, jl_tuple_t *type, newmeth = jl_reinstantiate_method(method, li); (void)jl_method_cache_insert(mt, type, newmeth); JL_GC_POP(); + JL_UNLOCK(codegen) return newmeth; } else { @@ -886,6 +891,7 @@ static jl_function_t *cache_method(jl_methtable_t *mt, jl_tuple_t *type, jl_type_infer(newmeth->linfo, type, method->linfo); } JL_GC_POP(); + JL_UNLOCK(codegen) return newmeth; } @@ -1354,7 +1360,7 @@ void NORETURN jl_no_method_error(jl_function_t *f, jl_value_t **args, size_t na) // not reached } -static jl_tuple_t *arg_type_tuple(jl_value_t **args, size_t nargs) +jl_tuple_t *arg_type_tuple(jl_value_t **args, size_t nargs) { jl_tuple_t *tt = jl_alloc_tuple(nargs); JL_GC_PUSH1(&tt); @@ -1586,6 +1592,7 @@ static void show_call(jl_value_t *F, jl_value_t **args, uint32_t nargs) } #endif + JL_CALLABLE(jl_apply_generic) { jl_methtable_t *mt = jl_gf_mtable(F); diff --git a/src/ia_misc.h b/src/ia_misc.h new file mode 100644 index 0000000000000..18cf8a24d0d9b --- /dev/null +++ b/src/ia_misc.h @@ -0,0 +1,87 @@ +#ifndef IA_MISC_H +#define IA_MISC_H + +#include +#include + +#if defined(__i386__) + +static __inline__ unsigned long long rdtsc(void) +{ + unsigned long long int x; + __asm__ volatile (".byte 0x0f, 0x31" : "=A" (x)); + return x; +} + +#elif defined(__x86_64__) + +static inline uint64_t rdtsc() +{ + unsigned hi, lo; + __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi)); + return ((uint64_t)lo) | (((uint64_t)hi) << 32); +} + +#endif /* __i386__ */ + +#if (__MIC__) + +static inline void cpu_pause() +{ + _mm_delay_64(100); +} + +static inline void cpu_delay(int64_t cycles) +{ + _mm_delay_64(cycles); +} + +static inline void cpu_mfence() +{ + __asm__ __volatile__ ("":::"memory"); +} + +static inline void cpu_sfence() +{ + __asm__ __volatile__ ("":::"memory"); +} + +static inline void cpu_lfence() +{ + __asm__ __volatile__ ("":::"memory"); +} + +#else /* !__MIC__ */ + +static inline void cpu_pause() +{ + _mm_pause(); +} + +static inline void cpu_delay(int64_t cycles) +{ + uint64_t s = rdtsc(); + while ((rdtsc() - s) < cycles) + _mm_pause(); +} + +static inline void cpu_mfence() +{ + _mm_mfence(); +} + +static inline void cpu_sfence() +{ + _mm_sfence(); +} + +static inline void cpu_lfence() +{ + _mm_lfence(); +} + +#endif /* __MIC__ */ + + +#endif /* IA_MISC_H */ + diff --git a/src/init.c b/src/init.c index ac6071a8e0a6d..49c8f062ac6cf 100644 --- a/src/init.c +++ b/src/init.c @@ -37,6 +37,7 @@ #include "julia.h" #include "julia_internal.h" +#include "threading.h" #include #ifdef __cplusplus @@ -970,6 +971,20 @@ void _julia_init(JL_IMAGE_SEARCH rel) } #endif +#if defined(__linux__) + int ncores = jl_cpu_cores(); + if (ncores > 1) { + cpu_set_t cpumask; + CPU_ZERO(&cpumask); + for(int i=0; i < ncores; i++) { + CPU_SET(i, &cpumask); + } + sched_setaffinity(0, sizeof(cpu_set_t), &cpumask); + } +#endif + + jl_init_threading(); + #ifdef JL_GC_MARKSWEEP jl_gc_init(); jl_gc_disable(); @@ -977,7 +992,11 @@ void _julia_init(JL_IMAGE_SEARCH rel) jl_init_frontend(); jl_init_types(); jl_init_tasks(jl_stack_lo, jl_stack_hi-jl_stack_lo); + jl_init_root_task(); jl_init_codegen(); + + jl_start_threads(); + jl_an_empty_cell = (jl_value_t*)jl_alloc_cell_1d(0); jl_init_serializer(); @@ -991,7 +1010,10 @@ void _julia_init(JL_IMAGE_SEARCH rel) jl_internal_main_module = jl_main_module; jl_current_module = jl_core_module; - jl_root_task->current_module = jl_current_module; + int t; + for(t=0; t < jl_n_threads; t++) { + (*jl_all_task_states[t].proot_task)->current_module = jl_current_module; + } jl_load("boot.jl"); jl_get_builtin_hooks(); @@ -1033,7 +1055,10 @@ void _julia_init(JL_IMAGE_SEARCH rel) // eval() uses Main by default, so Main.eval === Core.eval jl_module_import(jl_main_module, jl_core_module, jl_symbol("eval")); jl_current_module = jl_main_module; - jl_root_task->current_module = jl_current_module; + int t; + for(t=0; t < jl_n_threads; t++) { + (*jl_all_task_states[t].proot_task)->current_module = jl_current_module; + } #ifndef _OS_WINDOWS_ signal_stack = malloc(sig_stack_size); @@ -1243,11 +1268,14 @@ static jl_value_t *basemod(char *name) // fetch references to things defined in boot.jl void jl_get_builtin_hooks(void) { - jl_root_task->tls = jl_nothing; - jl_root_task->consumers = jl_nothing; - jl_root_task->donenotify = jl_nothing; - jl_root_task->exception = jl_nothing; - jl_root_task->result = jl_nothing; + int t; + for(t=0; t < jl_n_threads; t++) { + (*jl_all_task_states[t].proot_task)->tls = jl_nothing; + (*jl_all_task_states[t].proot_task)->consumers = jl_nothing; + (*jl_all_task_states[t].proot_task)->donenotify = jl_nothing; + (*jl_all_task_states[t].proot_task)->exception = jl_nothing; + (*jl_all_task_states[t].proot_task)->result = jl_nothing; + } jl_char_type = (jl_datatype_t*)core("Char"); jl_int8_type = (jl_datatype_t*)core("Int8"); diff --git a/src/julia.h b/src/julia.h index 578cdb3c68b48..8254f59d69132 100644 --- a/src/julia.h +++ b/src/julia.h @@ -21,10 +21,12 @@ extern "C" { # else # define MAX_ALIGN sizeof(void*) # endif +# define __JL_THREAD __thread #else # define jl_jmp_buf jmp_buf # include //for _resetstkoflw # define MAX_ALIGN 8 +# define __JL_THREAD __declspec(thread) #endif #ifdef _P64 @@ -1021,7 +1023,7 @@ typedef struct _jl_gcframe_t { // jl_value_t *x=NULL, *y=NULL; JL_GC_PUSH(&x, &y); // x = f(); y = g(); foo(x, y) -extern DLLEXPORT jl_gcframe_t *jl_pgcstack; +extern DLLEXPORT __JL_THREAD jl_gcframe_t *jl_pgcstack; #define JL_GC_PUSH(...) \ void *__gc_stkf[] = {(void*)((VA_NARG(__VA_ARGS__)<<1)|1), jl_pgcstack, \ @@ -1104,6 +1106,91 @@ STATIC_INLINE void *alloc_4w() { return allocobj(4*sizeof(void*)); } #define allocobj(nb) malloc(nb) #endif +// threading ------------------------------------------------------------------ + +DLLEXPORT int16_t jl_threadid(void); +DLLEXPORT void *jl_threadgroup(void); +DLLEXPORT void jl_cpu_pause(void); +DLLEXPORT jl_value_t *jl_threading_run(jl_function_t *fun, jl_tuple_t *args); +DLLEXPORT void jl_threading_profile(); + +#if __GNUC__ +# define JL_ATOMIC_FETCH_AND_ADD(a,b) \ + __sync_fetch_and_add(&(a), (b)) +# define JL_ATOMIC_COMPARE_AND_SWAP(a,b,c) \ + __sync_bool_compare_and_swap(&(a), (b), (c)) +# define JL_ATOMIC_TEST_AND_SET(a) \ + __sync_lock_test_and_set(&(a), 1) +# define JL_ATOMIC_RELEASE(a) \ + __sync_lock_release(&(a)) +#elif _WIN32 +# define JL_ATOMIC_FETCH_AND_ADD(a,b) \ + _InterlockedExchangeAdd((volatile LONG *)&(a), (b)) +# define JL_ATOMIC_COMPARE_AND_SWAP(a,b,c) \ + _InterlockedCompareExchange64(&(a), (c), (b)) +# define JL_ATOMIC_TEST_AND_SET(a) \ + _InterlockedExchange64(&(a), 1) +# define JL_ATOMIC_RELEASE(a) \ + _InterlockedExchange64(&(a), 0) +#else +# error "No atomic operations supported." +#endif + +#if 0 +#define JL_DEFINE_MUTEX(m) \ + uv_mutex_t m ## _mutex; \ + uint64_t m ## _thread_id = -1; + +#define JL_DEFINE_MUTEX_EXT(m) \ + extern uv_mutex_t m ## _mutex; \ + extern uint64_t m ## _thread_id; + +#define JL_LOCK(m) \ + int m ## locked = 0; \ + if (m ## _thread_id != uv_thread_self()) { \ + uv_mutex_lock(&m ## _mutex); \ + m ## locked = 1; \ + m ## _thread_id = uv_thread_self(); \ + } + +#define JL_UNLOCK(m) \ + if (m ## locked) { \ + m ## _thread_id = -1; \ + m ## locked = 0; \ + uv_mutex_unlock(&m ## _mutex); \ + } +#else +#define JL_DEFINE_MUTEX(m) \ + uint64_t volatile m ## _mutex = 0; \ + int32_t m ## _lock_count = 0; + +#define JL_DEFINE_MUTEX_EXT(m) \ + extern uint64_t volatile m ## _mutex; \ + extern int32_t m ## _lock_count; + +#define JL_LOCK(m) \ + if (m ## _mutex == uv_thread_self()) \ + ++m ## _lock_count; \ + else { \ + for (; ;) { \ + if (m ## _mutex == 0 && \ + JL_ATOMIC_COMPARE_AND_SWAP(m ## _mutex, 0, \ + uv_thread_self())) { \ + m ## _lock_count = 1; \ + break; \ + } \ + jl_cpu_pause(); \ + } \ + } + +#define JL_UNLOCK(m) \ + if (m ## _mutex == uv_thread_self()) { \ + --m ## _lock_count; \ + if (m ## _lock_count == 0) \ + JL_ATOMIC_COMPARE_AND_SWAP(m ## _mutex, uv_thread_self(), 0); \ + } +#endif + // async signal handling ------------------------------------------------------ #include @@ -1111,14 +1198,14 @@ STATIC_INLINE void *alloc_4w() { return allocobj(4*sizeof(void*)); } DLLEXPORT extern volatile sig_atomic_t jl_signal_pending; DLLEXPORT extern volatile sig_atomic_t jl_defer_signal; -#define JL_SIGATOMIC_BEGIN() (jl_defer_signal++) +#define JL_SIGATOMIC_BEGIN() (JL_ATOMIC_FETCH_AND_ADD(jl_defer_signal,1)) #define JL_SIGATOMIC_END() \ do { \ - jl_defer_signal--; \ - if (jl_defer_signal == 0 && jl_signal_pending != 0) { \ - jl_signal_pending = 0; \ - jl_throw(jl_interrupt_exception); \ - } \ + if (JL_ATOMIC_FETCH_AND_ADD(jl_defer_signal,-1) == 1 \ + && jl_signal_pending != 0) { \ + jl_signal_pending = 0; \ + jl_throw(jl_interrupt_exception); \ + } \ } while(0) DLLEXPORT void restore_signals(void); @@ -1165,9 +1252,17 @@ typedef struct _jl_task_t { jl_module_t *current_module; } jl_task_t; -extern DLLEXPORT jl_task_t * volatile jl_current_task; -extern DLLEXPORT jl_task_t *jl_root_task; -extern DLLEXPORT jl_value_t *jl_exception_in_transit; +typedef struct { + jl_task_t **pcurrent_task; + jl_task_t **proot_task; + jl_value_t **pexception_in_transit; + jl_value_t **ptask_arg_in_transit; +} jl_thread_task_state_t; + +extern DLLEXPORT __JL_THREAD jl_task_t *jl_current_task; +extern DLLEXPORT __JL_THREAD jl_task_t *jl_root_task; +extern DLLEXPORT __JL_THREAD jl_value_t *jl_exception_in_transit; +extern DLLEXPORT __JL_THREAD jl_value_t *jl_task_arg_in_transit; DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize); jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg); @@ -1233,7 +1328,6 @@ void jl_longjmp(jmp_buf _Buf,int _Value); for (i__ca=1, jl_eh_restore_state(&__eh); i__ca; i__ca=0) #endif - // I/O system ----------------------------------------------------------------- #define JL_STREAM uv_stream_t diff --git a/src/julia_internal.h b/src/julia_internal.h index a491498303742..9f450d5208294 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -88,12 +88,24 @@ void jl_init_primitives(void); void jl_init_codegen(void); void jl_init_intrinsic_functions(void); void jl_init_tasks(void *stack, size_t ssize); +void jl_init_root_task(void); void jl_init_serializer(void); + void _julia_init(JL_IMAGE_SEARCH rel); #ifdef COPY_STACKS -extern void *jl_stackbase; +extern __JL_THREAD void *jl_stackbase; +#ifndef ASM_COPY_STACKS +extern __JL_THREAD jl_jmp_buf jl_base_ctx; +#endif #endif +void jl_set_stackbase(void); +void jl_set_base_ctx(void); + +void jl_init_threading(void); +void jl_start_threads(void); +void jl_shutdown_threading(void); + void jl_dump_bitcode(char *fname); void jl_dump_objfile(char *fname, int jit_model); int32_t jl_get_llvm_gv(jl_value_t *p); @@ -107,6 +119,8 @@ jl_function_t *jl_get_specialization(jl_function_t *f, jl_tuple_t *types); jl_function_t *jl_module_get_initializer(jl_module_t *m); void jl_generate_fptr(jl_function_t *f); void jl_fptr_to_llvm(void *fptr, jl_lambda_info_t *lam, int specsig); +jl_function_t *jl_get_specialization(jl_function_t *f, jl_tuple_t *types); +jl_tuple_t *arg_type_tuple(jl_value_t **args, size_t nargs); // backtraces #ifdef _OS_WINDOWS_ diff --git a/src/options.h b/src/options.h index eba5ea1e4f23b..699784b8956e6 100644 --- a/src/options.h +++ b/src/options.h @@ -74,4 +74,21 @@ #define COPY_STACKS #endif +// threading options ---------------------------------------------------------- + +// controls for when threads sleep +#define THREAD_SLEEP_THRESHOLD_NAME "JULIA_THREAD_SLEEP_THRESHOLD" +#define DEFAULT_THREAD_SLEEP_THRESHOLD 1e9 // cycles (1e9==1sec@1GHz) + +// defaults for # threads +#define NUM_THREADS_NAME "JULIA_NUM_THREADS" +#define DEFAULT_NUM_THREADS 8 + +// affinitization behavior +#define MACHINE_EXCLUSIVE_NAME "JULIA_EXCLUSIVE" +#define DEFAULT_MACHINE_EXCLUSIVE 0 + +// number of memory pools for lock free pool_alloc +#define N_GC_THREADS 16 + #endif diff --git a/src/task.c b/src/task.c index ed9bd921f8005..49215c880c619 100644 --- a/src/task.c +++ b/src/task.c @@ -12,6 +12,7 @@ #include #include "julia.h" #include "julia_internal.h" +#include "threading.h" #ifdef __cplusplus extern "C" { @@ -134,36 +135,34 @@ static void _probe_arch(void) /* end probing code */ -/* - TODO: - - per-task storage (scheme-like parameters) - - stack growth -*/ - static jl_sym_t *done_sym; static jl_sym_t *failed_sym; static jl_sym_t *runnable_sym; extern size_t jl_page_size; jl_datatype_t *jl_task_type; -DLLEXPORT jl_task_t * volatile jl_current_task; -jl_task_t *jl_root_task; -jl_value_t * volatile jl_task_arg_in_transit; -jl_value_t *jl_exception_in_transit; + #ifdef JL_GC_MARKSWEEP -jl_gcframe_t *jl_pgcstack = NULL; +__JL_THREAD jl_gcframe_t *jl_pgcstack = NULL; #endif +DLLEXPORT __JL_THREAD jl_task_t *jl_current_task; +DLLEXPORT __JL_THREAD jl_task_t *jl_root_task; +DLLEXPORT __JL_THREAD jl_value_t *jl_task_arg_in_transit; +DLLEXPORT __JL_THREAD jl_value_t *jl_exception_in_transit; + +static void start_task(); + #ifdef COPY_STACKS -static jl_jmp_buf * volatile jl_jmp_target; +__JL_THREAD jl_jmp_buf * volatile jl_jmp_target; #if defined(_CPU_X86_64_) || defined(_CPU_X86_) #define ASM_COPY_STACKS #endif -void *jl_stackbase; +__JL_THREAD void *jl_stackbase; #ifndef ASM_COPY_STACKS -static jl_jmp_buf jl_base_ctx; // base context of stack +__JL_THREAD jl_jmp_buf jl_base_ctx; // base context of stack #endif #if defined(_OS_WINDOWS_) && !defined(_COMPILER_MINGW_) @@ -210,6 +209,18 @@ restore_stack(jl_task_t *t, jl_jmp_buf *where, char *p) memcpy(_x, t->stkbuf, t->ssize); jl_longjmp(*jl_jmp_target, 1); } + +void jl_switch_stack(jl_task_t *t, jl_jmp_buf *where) +{ + assert(t == jl_current_task); + if (t->stkbuf == NULL) { + start_task(); + // doesn't return + } + else { + restore_stack(t, where, NULL); + } +} #endif static jl_function_t *task_done_hook_func=NULL; @@ -225,12 +236,19 @@ static void NORETURN finish_task(jl_task_t *t, jl_value_t *resultval) #ifdef COPY_STACKS t->stkbuf = NULL; #endif - if (task_done_hook_func == NULL) { - task_done_hook_func = (jl_function_t*)jl_get_global(jl_base_module, - jl_symbol("task_done_hook")); + if (ti_tid == 0) { + // for now only thread 0 runs the task scheduler + if (task_done_hook_func == NULL) { + task_done_hook_func = (jl_function_t*)jl_get_global(jl_base_module, + jl_symbol("task_done_hook")); + } + if (task_done_hook_func != NULL) { + jl_apply(task_done_hook_func, (jl_value_t**)&t, 1); + } } - if (task_done_hook_func != NULL) { - jl_apply(task_done_hook_func, (jl_value_t**)&t, 1); + else { + // others return to thread loop + jl_switchto(jl_root_task, jl_nothing); } abort(); } @@ -250,32 +268,39 @@ NORETURN start_task() abort(); } +#ifdef COPY_STACKS +void jl_set_stackbase() +{ + char __stk; + jl_stackbase = (char*)(((uptrint_t)&__stk + sizeof(__stk))&-16); // also ensures stackbase is 16-byte aligned +} +#else +void jl_set_stackbase() { } +#endif + #ifndef ASM_COPY_STACKS #if defined(_OS_WINDOWS_) && !defined(_COMPILER_MINGW_) -static void __declspec(noinline) +void __declspec(noinline) #else -static void __attribute__((noinline)) +void __attribute__((noinline)) #endif -set_base_ctx(char *__stk) +jl_set_base_ctx() { if (jl_setjmp(jl_base_ctx, 1)) { start_task(); } } #else -void set_base_ctx(char *__stk) { } +void jl_set_base_ctx() { } #endif - DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel) { // keep this function small, since we want to keep the stack frame // leading up to this also quite small _julia_init(rel); -#ifdef COPY_STACKS - char __stk; - jl_stackbase = (char*)(((uptrint_t)&__stk + sizeof(__stk))&-16); // also ensures stackbase is 16-byte aligned - set_base_ctx(&__stk); // separate function, to record the size of a stack frame -#endif + + jl_set_stackbase(); + jl_set_base_ctx(); } #ifndef COPY_STACKS @@ -297,6 +322,11 @@ static void init_task(jl_task_t *t) } #endif +DLLEXPORT void jl_handle_stack_switch() +{ + jl_switch_stack(jl_current_task, jl_jmp_target); +} + static void ctx_switch(jl_task_t *t, jl_jmp_buf *where) { if (t == jl_current_task) @@ -746,6 +776,7 @@ DLLEXPORT void gdbbacktrace() void NORETURN throw_internal(jl_value_t *e) { assert(e != NULL); + jl_exception_in_transit = e; if (jl_current_task->eh != NULL) { jl_longjmp(jl_current_task->eh->eh_ctx, 1); @@ -869,6 +900,9 @@ jl_function_t *jl_unprotect_stack_func; void jl_init_tasks(void *stack, size_t ssize) { + (void)stack; // TODO make per-thread + (void)ssize; + _probe_arch(); jl_task_type = jl_new_datatype(jl_symbol("Task"), jl_any_type, @@ -895,12 +929,18 @@ void jl_init_tasks(void *stack, size_t ssize) failed_sym = jl_symbol("failed"); runnable_sym = jl_symbol("runnable"); + jl_unprotect_stack_func = jl_new_closure(jl_unprotect_stack, (jl_value_t*)jl_null, NULL); +} + +void jl_init_root_task(void) +{ jl_current_task = (jl_task_t*)allocobj(sizeof(jl_task_t)); jl_current_task->type = (jl_value_t*)jl_task_type; #ifdef COPY_STACKS jl_current_task->ssize = 0; // size of saved piece jl_current_task->bufsz = 0; #else + // TODO update for threads jl_current_task->stack = stack; jl_current_task->ssize = ssize; #endif @@ -924,7 +964,6 @@ void jl_init_tasks(void *stack, size_t ssize) jl_exception_in_transit = (jl_value_t*)jl_null; jl_task_arg_in_transit = (jl_value_t*)jl_null; - jl_unprotect_stack_func = jl_new_closure(jl_unprotect_stack, (jl_value_t*)jl_null, NULL); } #ifdef __cplusplus diff --git a/src/threadgroup.c b/src/threadgroup.c new file mode 100644 index 0000000000000..a83b34eff83ec --- /dev/null +++ b/src/threadgroup.c @@ -0,0 +1,236 @@ +/* +Copyright (c) 2014, Intel Corporation + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* + threading infrastructure + . threadgroup abstraction + . fork/join/barrier +*/ + + +#include "options.h" + +#include +#include +#include +#include "ia_misc.h" +#include "threadgroup.h" + + +int ti_threadgroup_create(uint8_t num_sockets, uint8_t num_cores, + uint8_t num_threads_per_core, + ti_threadgroup_t **newtg) +{ + int i; + ti_threadgroup_t *tg; + int num_threads = num_sockets * num_cores * num_threads_per_core; + char *cp; + + tg = (ti_threadgroup_t *)_mm_malloc(sizeof (ti_threadgroup_t), 64); + tg->tid_map = (int16_t *)_mm_malloc(num_threads * sizeof (int16_t), 64); + for (i = 0; i < num_threads; ++i) + tg->tid_map[i] = -1; + tg->num_sockets = num_sockets; + tg->num_cores = num_cores; + tg->num_threads_per_core = num_threads_per_core; + tg->num_threads = num_threads; + tg->added_threads = 0; + tg->thread_sense = (ti_thread_sense_t **) + _mm_malloc(num_threads * sizeof (ti_thread_sense_t *), 64); + for (i = 0; i < num_threads; i++) + tg->thread_sense[i] = NULL; + tg->group_sense = 0; + tg->forked = 0; + + pthread_mutex_init(&tg->alarm_lock, NULL); + pthread_cond_init(&tg->alarm, NULL); + + tg->sleep_threshold = DEFAULT_THREAD_SLEEP_THRESHOLD; + cp = getenv(THREAD_SLEEP_THRESHOLD_NAME); + if (cp) { + if (!strncasecmp(cp, "infinite", 8)) + tg->sleep_threshold = 0; + else + tg->sleep_threshold = (uint64_t)strtol(cp, NULL, 10); + } + + *newtg = tg; + return 0; +} + + +int ti_threadgroup_addthread(ti_threadgroup_t *tg, int16_t ext_tid, + int16_t *tgtid) +{ + if (ext_tid < 0 || ext_tid >= tg->num_threads) + return -1; + if (tg->tid_map[ext_tid] != -1) + return -2; + if (tg->added_threads == tg->num_threads) + return -3; + + tg->tid_map[ext_tid] = tg->added_threads++; + if (tgtid) *tgtid = tg->tid_map[ext_tid]; + + return 0; +} + + +int ti_threadgroup_initthread(ti_threadgroup_t *tg, int16_t ext_tid) +{ + ti_thread_sense_t *ts; + + if (ext_tid < 0 || ext_tid >= tg->num_threads) + return -1; + if (tg->thread_sense[tg->tid_map[ext_tid]] != NULL) + return -2; + if (tg->num_threads == 0) + return -3; + + ts = (ti_thread_sense_t *)_mm_malloc(sizeof (ti_thread_sense_t), 64); + ts->sense = 1; + tg->thread_sense[tg->tid_map[ext_tid]] = ts; + + return 0; +} + + +int ti_threadgroup_member(ti_threadgroup_t *tg, int16_t ext_tid, + int16_t *tgtid) +{ + if (ext_tid < 0 || ext_tid >= tg->num_threads) + return -1; + if (tg == NULL) { + if (tgtid) *tgtid = -1; + return -2; + } + if (tg->tid_map[ext_tid] == -1) { + if (tgtid) *tgtid = -1; + return -3; + } + if (tgtid) *tgtid = tg->tid_map[ext_tid]; + + return 0; +} + + +int ti_threadgroup_size(ti_threadgroup_t *tg, int16_t *tgsize) +{ + *tgsize = tg->num_threads; + return 0; +} + + +int ti_threadgroup_fork(ti_threadgroup_t *tg, int16_t ext_tid, + void **bcast_val) +{ + if (tg->tid_map[ext_tid] == 0) { + tg->envelope = bcast_val ? *bcast_val : NULL; + cpu_sfence(); + tg->forked = 1; + tg->group_sense = tg->thread_sense[0]->sense; + + // if it's possible that threads are sleeping, signal them + if (tg->sleep_threshold) { + pthread_mutex_lock(&tg->alarm_lock); + pthread_cond_broadcast(&tg->alarm); + pthread_mutex_unlock(&tg->alarm_lock); + } + } + else { + // spin up to threshold cycles (count sheep), then sleep + uint64_t spin_cycles, spin_start = rdtsc(); + while (tg->group_sense != + tg->thread_sense[tg->tid_map[ext_tid]]->sense) { + if (tg->sleep_threshold) { + spin_cycles = rdtsc() - spin_start; + if (spin_cycles >= tg->sleep_threshold) { + pthread_mutex_lock(&tg->alarm_lock); + if (tg->group_sense != + tg->thread_sense[tg->tid_map[ext_tid]]->sense) { + pthread_cond_wait(&tg->alarm, &tg->alarm_lock); + } + pthread_mutex_unlock(&tg->alarm_lock); + spin_start = rdtsc(); + continue; + } + } + cpu_pause(); + } + cpu_lfence(); + if (bcast_val) + *bcast_val = tg->envelope; + } + + return 0; +} + + +int ti_threadgroup_join(ti_threadgroup_t *tg, int16_t ext_tid) +{ + int i; + + tg->thread_sense[tg->tid_map[ext_tid]]->sense + = !tg->thread_sense[tg->tid_map[ext_tid]]->sense; + if (tg->tid_map[ext_tid] == 0) { + for (i = 1; i < tg->num_threads; ++i) { + while (tg->thread_sense[i]->sense == tg->group_sense) + cpu_pause(); + } + tg->forked = 0; + } + + return 0; +} + + +void ti_threadgroup_barrier(ti_threadgroup_t *tg, int16_t ext_tid) +{ + if (tg->tid_map[ext_tid] == 0 && !tg->forked) + return; + + ti_threadgroup_join(tg, ext_tid); + ti_threadgroup_fork(tg, ext_tid, NULL); +} + + +int ti_threadgroup_destroy(ti_threadgroup_t *tg) +{ + int i; + + pthread_mutex_destroy(&tg->alarm_lock); + pthread_cond_destroy(&tg->alarm); + + for (i = 0; i < tg->num_threads; i++) + _mm_free(tg->thread_sense[i]); + _mm_free(tg->thread_sense); + _mm_free(tg->tid_map); + _mm_free(tg); + + return 0; +} + diff --git a/src/threadgroup.h b/src/threadgroup.h new file mode 100644 index 0000000000000..5c752455f3f70 --- /dev/null +++ b/src/threadgroup.h @@ -0,0 +1,79 @@ +/* +Copyright (c) 2014, Intel Corporation + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef THREADGROUP_H +#define THREADGROUP_H + +#include +#include + + +// for the barrier +typedef struct { + volatile int sense; + +} ti_thread_sense_t; + + +// thread group +typedef struct { + int16_t *tid_map, num_threads, added_threads; + uint8_t num_sockets, num_cores, num_threads_per_core; + + // fork/join/barrier + uint8_t forked; + volatile uint8_t group_sense; + ti_thread_sense_t **thread_sense; + void *envelope; + + // to let threads sleep + pthread_mutex_t alarm_lock; + pthread_cond_t alarm; + uint64_t sleep_threshold; + +} ti_threadgroup_t; + + +int ti_threadgroup_create(uint8_t num_sockets, uint8_t num_cores, + uint8_t num_threads_per_core, + ti_threadgroup_t **newtg); +int ti_threadgroup_addthread(ti_threadgroup_t *tg, int16_t ext_tid, + int16_t *tgtid); +int ti_threadgroup_initthread(ti_threadgroup_t *tg, int16_t ext_tid); +int ti_threadgroup_member(ti_threadgroup_t *tg, int16_t ext_tid, + int16_t *tgtid); +int ti_threadgroup_size(ti_threadgroup_t *tg, int16_t *tgsize); +int ti_threadgroup_fork(ti_threadgroup_t *tg, int16_t ext_tid, + void **bcast_val); +int ti_threadgroup_join(ti_threadgroup_t *tg, int16_t ext_tid); +void ti_threadgroup_barrier(ti_threadgroup_t *tg, int16_t ext_tid); +int ti_threadgroup_destroy(ti_threadgroup_t *tg); + +extern ti_threadgroup_t *tgworld; + +#endif /* THREADGROUP_H */ + diff --git a/src/threading.c b/src/threading.c new file mode 100644 index 0000000000000..ab0624c17931b --- /dev/null +++ b/src/threading.c @@ -0,0 +1,468 @@ +/* +Copyright (c) 2014, Intel Corporation + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* + threading infrastructure + . thread and threadgroup creation + . thread function + . invoke Julia function from multiple threads +*/ + + +#include +#include +#include +#include +#include +#include +#include + +#include "julia.h" +#include "julia_internal.h" +#include "uv.h" + +#include "ia_misc.h" +#include "threadgroup.h" +#include "threading.h" + + +/* TODO: + . ugly mixture of uv_thread* and pthread*; fix with patch to libuv? + . fix interface to properly support thread groups + . add queue per thread for tasks + . add reduction; reduce values returned from thread function + . make code generation thread-safe and remove the lock +*/ + +// lock for code generation +JL_DEFINE_MUTEX(codegen); + +// thread ID +__JL_THREAD int16_t ti_tid = 0; + +// thread heap +__JL_THREAD struct _jl_thread_heap_t *jl_thread_heap; +struct _jl_thread_heap_t **jl_all_heaps; +jl_gcframe_t ***jl_all_pgcstacks; +jl_thread_task_state_t *jl_all_task_states; + +// only one thread group for now +ti_threadgroup_t *tgworld; + +// for broadcasting work to threads +ti_threadwork_t threadwork; + +DLLEXPORT int jl_max_threads; // # threads possible +DLLEXPORT int jl_n_threads; // # threads we're actually using + +#if PROFILE_JL_THREADING +double cpu_ghz; +uint64_t prep_ticks; +uint64_t *fork_ticks; +uint64_t *user_ticks; +uint64_t *join_ticks; +#endif + + +// create a thread and affinitize it if proc_num is specified +int ti_threadcreate(uint64_t *pthread_id, int proc_num, + void *(*thread_fun)(void *), void *thread_arg) +{ + pthread_attr_t attr; + pthread_attr_init(&attr); + +#ifdef _OS_LINUX + cpu_set_t cset; + if (proc_num >= 0) { + CPU_ZERO(&cset); + CPU_SET(proc_num, &cset); + pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cset); + } +#endif + + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + return pthread_create(pthread_id, &attr, thread_fun, thread_arg); +} + + +// set thread affinity +void ti_threadsetaffinity(uint64_t pthread_id, int proc_num) +{ +#ifdef _OS_LINUX + cpu_set_t cset; + + CPU_ZERO(&cset); + CPU_SET(proc_num, &cset); + pthread_setaffinity_np(pthread_id, sizeof(cpu_set_t), &cset); +#endif +} + + +struct _jl_thread_heap_t *jl_mk_thread_heap(void); + +// must be called by each thread at startup +void ti_initthread(int16_t tid) +{ + ti_tid = tid; + jl_pgcstack = NULL; + jl_all_pgcstacks[tid] = &jl_pgcstack; + jl_thread_heap = jl_mk_thread_heap(); + jl_all_heaps[tid] = jl_thread_heap; + + jl_all_task_states[tid].pcurrent_task = &jl_current_task; + jl_all_task_states[tid].proot_task = &jl_root_task; + jl_all_task_states[tid].pexception_in_transit = &jl_exception_in_transit; + jl_all_task_states[tid].ptask_arg_in_transit = &jl_task_arg_in_transit; +} + + +// all threads call this function to run user code +jl_value_t *ti_run_fun(jl_function_t *f, jl_tuple_t *args) +{ + JL_TRY { + jl_apply(f, &jl_tupleref(args,0), jl_tuple_len(args)); + } + JL_CATCH { + return jl_exception_in_transit; + } + return jl_nothing; +} + + +// thread function: used by all except the main thread +void *ti_threadfun(void *arg) +{ + ti_threadarg_t *ta = (ti_threadarg_t *)arg; + ti_threadgroup_t *tg; + ti_threadwork_t *work; + + // initialize this thread (set tid, create heap, etc.) + ti_initthread(ta->tid); + + // set up tasking + jl_init_root_task(); + jl_set_stackbase(); + jl_set_base_ctx(); + + // set the thread-local tid and wait for a thread group + while (ta->state == TI_THREAD_INIT) + cpu_pause(); + cpu_lfence(); + + // initialize this thread in the thread group + tg = ta->tg; + ti_threadgroup_initthread(tg, ti_tid); + + // free the thread argument here + free(ta); + + // work loop + for (; ;) { +#if PROFILE_JL_THREADING + uint64_t tstart = rdtsc(); +#endif + + ti_threadgroup_fork(tg, ti_tid, (void **)&work); + +#if PROFILE_JL_THREADING + uint64_t tfork = rdtsc(); + fork_ticks[ti_tid] += tfork - tstart; +#endif + + if (work) { + if (work->command == TI_THREADWORK_DONE) + break; + else if (work->command == TI_THREADWORK_RUN) + // TODO: return value? reduction? + ti_run_fun(work->fun, work->args); + } + +#if PROFILE_JL_THREADING + uint64_t tuser = rdtsc(); + user_ticks[ti_tid] += tuser - tfork; +#endif + + ti_threadgroup_join(tg, ti_tid); + +#if PROFILE_JL_THREADING + uint64_t tjoin = rdtsc(); + join_ticks[ti_tid] += tjoin - tuser; +#endif + + // TODO: + // nowait should skip the join, but confirm that fork is reentrant + } + + return NULL; +} + + +#if PROFILE_JL_THREADING +void ti_reset_timings(); +#endif + +// interface to Julia; sets up to make the runtime thread-safe +void jl_init_threading(void) +{ + char *cp; + + // how many threads available, usable + jl_max_threads = jl_cpu_cores(); + jl_n_threads = DEFAULT_NUM_THREADS; + cp = getenv(NUM_THREADS_NAME); + if (cp) { + jl_n_threads = (uint64_t)strtol(cp, NULL, 10); + } + if (jl_n_threads > jl_max_threads) + jl_n_threads = jl_max_threads; + + // set up space for per-thread heaps + jl_all_heaps = malloc(jl_n_threads * sizeof(void*)); + jl_all_pgcstacks = malloc(jl_n_threads * sizeof(void*)); + jl_all_task_states = malloc(jl_n_threads * sizeof(jl_thread_task_state_t)); + +#if PROFILE_JL_THREADING + // estimate CPU speed + uint64_t cpu_tim = rdtsc(); + sleep(1); + cpu_ghz = ((double)(rdtsc() - cpu_tim)) / 1e9; + + // set up space for profiling information + fork_ticks = (uint64_t *)_mm_malloc(jl_n_threads * sizeof (uint64_t), 64); + user_ticks = (uint64_t *)_mm_malloc(jl_n_threads * sizeof (uint64_t), 64); + join_ticks = (uint64_t *)_mm_malloc(jl_n_threads * sizeof (uint64_t), 64); + ti_reset_timings(); +#endif + + // initialize this master thread (set tid, create heap, etc.) + ti_initthread(0); +} + +void jl_start_threads(void) +{ + char *cp; + int i, exclusive; + uint64_t ptid; + ti_threadarg_t **targs; + + // do we have exclusive use of the machine? default is no + exclusive = DEFAULT_MACHINE_EXCLUSIVE; + cp = getenv(MACHINE_EXCLUSIVE_NAME); + if (cp) + exclusive = strtol(cp, NULL, 10); + + // exclusive use: affinitize threads, master thread on proc 0, rest + // according to a 'compact' policy + // non-exclusive: no affinity settings; let the kernel move threads about + if (exclusive) + ti_threadsetaffinity(uv_thread_self(), 0); + + // create threads + targs = malloc((jl_n_threads - 1) * sizeof (ti_threadarg_t *)); + for (i = 0; i < jl_n_threads - 1; ++i) { + targs[i] = (ti_threadarg_t *)malloc(sizeof (ti_threadarg_t)); + targs[i]->state = TI_THREAD_INIT; + targs[i]->tid = i + 1; + ti_threadcreate(&ptid, exclusive ? i+1 : -1, ti_threadfun, targs[i]); + } + + // set up the world thread group + ti_threadgroup_create(1, jl_n_threads, 1, &tgworld); + for (i = 0; i < jl_n_threads; ++i) + ti_threadgroup_addthread(tgworld, i, NULL); + ti_threadgroup_initthread(tgworld, ti_tid); + + // give the threads the world thread group; they will block waiting for fork + for (i = 0; i < jl_n_threads - 1; ++i) { + targs[i]->tg = tgworld; + cpu_sfence(); + targs[i]->state = TI_THREAD_WORK; + } + + // free the argument array; the threads will free their arguments + free(targs); +} + + +// TODO: is this needed? where/when/how to call it? +void jl_shutdown_threading(void) +{ + // stop the spinning threads by sending them a command + ti_threadwork_t *work = &threadwork; + + work->command = TI_THREADWORK_DONE; + ti_threadgroup_fork(tgworld, ti_tid, (void **)&work); + + sleep(1); + + // destroy the world thread group + ti_threadgroup_destroy(tgworld); + + // TODO: clean up and free the per-thread heaps + +#if PROFILE_JL_THREADING + _mm_free(join_ticks); + _mm_free(user_ticks); + _mm_free(fork_ticks); + fork_ticks = user_ticks = join_ticks = NULL; +#endif +} + + +// return calling thread's ID +int16_t jl_threadid() +{ + return ti_tid; +} + + +// return thread's thread group +void *jl_threadgroup() +{ + return (void *)tgworld; +} + + +// utility +void jl_cpu_pause() +{ + cpu_pause(); +} + + +// interface to user code: specialize and compile the user thread function +// and run it in all threads +jl_value_t *jl_threading_run(jl_function_t *f, jl_tuple_t *args) +{ +#if PROFILE_JL_THREADING + uint64_t tstart = rdtsc(); +#endif + + jl_tuple_t *argtypes = NULL; + jl_function_t *fun = NULL; + JL_GC_PUSH2(&argtypes, &fun); + argtypes = arg_type_tuple(&jl_tupleref(args, 0), jl_tuple_len(args)); + fun = jl_get_specialization(f, argtypes); + if (fun == NULL) + fun = f; + jl_compile(fun); + jl_generate_fptr(fun); + + threadwork.command = TI_THREADWORK_RUN; + threadwork.fun = fun; + threadwork.args = args; + threadwork.ret = jl_nothing; + +#if PROFILE_JL_THREADING + uint64_t tcompile = rdtsc(); + prep_ticks += (tcompile - tstart); +#endif + + // fork the world thread group + ti_threadwork_t *tw = (ti_threadwork_t *)&threadwork; + ti_threadgroup_fork(tgworld, ti_tid, (void **)&tw); + +#if PROFILE_JL_THREADING + uint64_t tfork = rdtsc(); + fork_ticks[ti_tid] += (tfork - tcompile); +#endif + + // this thread must do work too (TODO: reduction?) + tw->ret = ti_run_fun(fun, args); + +#if PROFILE_JL_THREADING + uint64_t trun = rdtsc(); + user_ticks[ti_tid] += (trun - tfork); +#endif + + // wait for completion (TODO: nowait?) + ti_threadgroup_join(tgworld, ti_tid); + +#if PROFILE_JL_THREADING + uint64_t tjoin = rdtsc(); + join_ticks[ti_tid] += (tjoin - trun); +#endif + + JL_GC_POP(); + + return tw->ret; +} + + +#if PROFILE_JL_THREADING + +void ti_reset_timings() +{ + int i; + prep_ticks = 0; + for (i = 0; i < jl_n_threads; i++) + fork_ticks[i] = user_ticks[i] = join_ticks[i] = 0; +} + +void ti_timings(uint64_t *times, uint64_t *min, uint64_t *max, uint64_t *avg) +{ + int i; + *min = UINT64_MAX; + *max = *avg = 0; + for (i = 0; i < jl_n_threads; i++) { + if (times[i] < *min) + *min = times[i]; + if (times[i] > *max) + *max = times[i]; + *avg += times[i]; + } + *avg /= jl_n_threads; +} + +#define TICKS_TO_SECS(t) (((double)(t)) / (cpu_ghz * 1e9)) + +void jl_threading_profile() +{ + if (!fork_ticks) return; + + printf("\nti profile:\n"); + printf("prep: %g (%lu)\n", TICKS_TO_SECS(prep_ticks), prep_ticks); + + uint64_t min, max, avg; + ti_timings(fork_ticks, &min, &max, &avg); + printf("fork: %g (%g - %g)\n", TICKS_TO_SECS(min), TICKS_TO_SECS(max), + TICKS_TO_SECS(avg)); + ti_timings(user_ticks, &min, &max, &avg); + printf("user: %g (%g - %g)\n", TICKS_TO_SECS(min), TICKS_TO_SECS(max), + TICKS_TO_SECS(avg)); + ti_timings(join_ticks, &min, &max, &avg); + printf("join: %g (%g - %g)\n", TICKS_TO_SECS(min), TICKS_TO_SECS(max), + TICKS_TO_SECS(avg)); +} + +#else + +void jl_threading_profile() +{ +} + +#endif diff --git a/src/threading.h b/src/threading.h new file mode 100644 index 0000000000000..2c615f4935531 --- /dev/null +++ b/src/threading.h @@ -0,0 +1,96 @@ +/* +Copyright (c) 2014, Intel Corporation + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef THREADING_H +#define THREADING_H + +#include +#include "julia.h" +#include "threadgroup.h" + +#define PROFILE_JL_THREADING 1 + + +// thread ID +extern __thread int16_t ti_tid; + +// GC +extern __JL_THREAD struct _jl_thread_heap_t *jl_thread_heap; +extern struct _jl_thread_heap_t **jl_all_heaps; +extern jl_gcframe_t ***jl_all_pgcstacks; +extern jl_thread_task_state_t *jl_all_task_states; + +extern DLLEXPORT int jl_n_threads; // # threads we're actually using + +// thread state +enum { + TI_THREAD_INIT, + TI_THREAD_WORK +}; + + +// passed to thread function +typedef struct { + int16_t volatile state; + int16_t tid; + ti_threadgroup_t *tg; + +} ti_threadarg_t; + + +// commands to thread function +enum { + TI_THREADWORK_DONE, + TI_THREADWORK_RUN +}; + + +// work command to thread function +typedef struct { + uint8_t command; + jl_function_t *fun; + jl_tuple_t *args; + jl_value_t *ret; + +} ti_threadwork_t; + + +// basic functions for thread creation +int ti_threadcreate(uint64_t *pthread_id, int proc_num, + void *(*thread_fun)(void *), void *thread_arg); +void ti_threadsetaffinity(uint64_t pthread_id, int proc_num); + +// thread function +void *ti_threadfun(void *arg); + +// helpers for thread function +void ti_initthread(int16_t tid); +jl_value_t *ti_runthread(jl_function_t *f, jl_tuple_t *args, size_t nargs); + + +#endif /* THREADING_H */ + diff --git a/test/perf/threads/laplace3d/README b/test/perf/threads/laplace3d/README new file mode 100644 index 0000000000000..a57205897ee0c --- /dev/null +++ b/test/perf/threads/laplace3d/README @@ -0,0 +1,19 @@ +Laplace 3D + +This is a simple 7 point stencil on a 3D grid. It is bandwidth-bound. +The C version gets 60 GB/s on a 2 socket SNB-EP. The code doesn't +handle corner cases at the grid edges, so each dimension must be 4n+2 +for SSE or 8n+2 for AVX. + +E.g.: + +$ ./laplace3d 258 258 258 1000 avx 0 + +The Matlab/Octave version is orders of magnitude slower than the C +and Julia versions, so it uses a smaller grid. + +Speedup with 16 threads on a 2 socket SNB-EP: + +C: X +Julia: Y + diff --git a/test/perf/threads/laplace3d/laplace3d.c b/test/perf/threads/laplace3d/laplace3d.c new file mode 100644 index 0000000000000..76392a1c43542 --- /dev/null +++ b/test/perf/threads/laplace3d/laplace3d.c @@ -0,0 +1,375 @@ +/* +Copyright (c) 2014, Intel Corporation + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* Laplace 3D + + orig: simple serial version + naive: simple parallelized version + auto: some ninja knowledge, using icc directives + sse/avx: ninja-optimized + + Requires Sandy Bridge and up. + + Note that the SSE/AVX versions do not handle boundary conditions + and thus each dimension must be 4n+2/8n+2. Try 258x258x258. + + 2014.08.06 anand.deshpande Initial code. + 2014.08.06 dhiraj.kalamkar Padding and streaming stores. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../src/ia_misc.h" + +void l3d_naive(int nx, int padded_nx, int ny, int nz, float *u1, float *u2); +void l3d_auto(int nx, int padded_nx, int ny, int nz, float *u1, float *u2); +void l3d_sse(int nx, int padded_nx, int ny, int nz, float *u1, float *u2); +void l3d_avx(int nx, int padded_nx, int ny, int nz, float *u1, float *u2); +void l3d_orig(int nx, int ny, int nz, float *u1, float *u2); + + +double cpughz() +{ + uint64_t t0 = rdtsc(); + sleep(1); + uint64_t onesec = rdtsc() - t0; + return onesec*1.0/1e9; +} + +int main(int argc, char **argv) +{ + int nx, padded_nx, ny, nz, iters, i, j, k, ind, p_ind, verify, + nthreads, pad_size; + float *u1, *u1_p, *u1_orig, *u2, *u2_p, *u2_orig, *foo, + error_tol = 0.00001; + double ghz; + void (*l3d)(int nx, int padded_nx, int ny, int nz, + float *u1, float *u2); + + if (argc != 7) { + fprintf(stderr, "Usage:\n" + " laplace3d <#iters> " + "\n"); + exit(-1); + } + + nx = strtol(argv[1], NULL, 10); + ny = strtol(argv[2], NULL, 10); + nz = strtol(argv[3], NULL, 10); + + padded_nx = ((nx + 0x7) & (~0x7)); + iters = strtol(argv[4], NULL, 10); + + if (strncasecmp(argv[5], "naive", 5) == 0) + l3d = l3d_naive; + else if (strncasecmp(argv[5], "auto", 4) == 0) + l3d = l3d_auto; + else if (strncasecmp(argv[5], "sse", 3) == 0) + l3d = l3d_sse; + else if (strncasecmp(argv[5], "avx", 3) == 0) + l3d = l3d_avx; + else { + fprintf(stderr, "don't recognize %s. naive, auto, sse, or avx?\n", + argv[5]); + exit(-1); + } + + verify = strtol(argv[6], NULL, 10); + + ghz = cpughz(); + nthreads = omp_get_max_threads(); + printf("machine speed is %g GHz, using %d threads\n", ghz, nthreads); + + printf("laplace3d: %d iterations on %dx%dx%d grid, " + "verification is %s\n", + iters, nx, ny, nz, verify ? "on" : "off"); + + /* pad for aligned access; non-naive only */ + if (strncasecmp(argv[5], "naive", 5) != 0) { + pad_size = (((1 + padded_nx + padded_nx * ny) + 0xF) & (~0xF)) - + (1 + padded_nx + padded_nx * ny); + printf("using padded_nx = %d, pad_size = %d\n", + padded_nx, pad_size); + + u1_p = (float *)_mm_malloc(sizeof (float) * + (padded_nx * ny * nz + pad_size), 64); + u2_p = (float *)_mm_malloc(sizeof (float) * + (padded_nx * ny * nz + pad_size), 64); + u1 = u1_p + pad_size; + u2 = u2_p + pad_size; + } + else { + u1_p = (float *)_mm_malloc(sizeof (float) * (nx * ny * nz), 64); + u2_p = (float *)_mm_malloc(sizeof (float) * (nx * ny * nz), 64); + u1 = u1_p; + u2 = u2_p; + padded_nx = nx; + } + u1_orig = (float *)_mm_malloc(sizeof (float) * nx * ny * nz, 64); + u2_orig = (float *)_mm_malloc(sizeof (float) * nx * ny * nz, 64); + + // initialize + #pragma omp parallel for private(k,j,i,ind,p_ind) + for (k = 0; k < nz; ++k) { + for (j = 0; j < ny; ++j) { + for (i = 0; i < nx; ++i) { + ind = i + j*nx + k*nx*ny; + p_ind = i + j*padded_nx + k*padded_nx*ny; + + if (i == 0 || i == nx - 1 + || j == 0 || j == ny - 1 + || k == 0 || k == nz - 1) { + // Dirichlet b.c.'s + u1[p_ind] = u1_orig[ind] = u2[p_ind] = 1.0f; + } + else { + u1[p_ind] = u1_orig[ind] = u2[p_ind] = 0.0f; + } + } + } + } + + // run optimized version + uint64_t t0 = rdtsc(); + for (i = 0; i < iters; ++i) { + l3d(nx, padded_nx, ny, nz, u1, u2); + foo = u1; u1 = u2; u2 = foo; + } + uint64_t gold = rdtsc() - t0; + double elapsed = gold / (ghz * 1e9); + + double grid_size = nx * ny * nz; + double gflops = grid_size * iters * 6.0 / 1e9; + double gflops_sec = gflops / elapsed; + + double traffic = grid_size * iters * 4 * 2.0 / 1e9; + double bw_realized = traffic / elapsed; + + printf("laplace3d completed in %.4lf seconds\n", elapsed); + printf("GFLOPs/sec: %.1f\n", gflops_sec); + printf("BW realized: %.1f\n", bw_realized); + + if (verify) { + // run serial version for verification + uint64_t st0 = rdtsc(); + for (i = 0; i < iters; ++i) { + l3d_orig(nx, ny, nz, u1_orig, u2_orig); + foo = u1_orig; u1_orig = u2_orig; u2_orig = foo; + } + uint64_t ser = rdtsc() - st0; + elapsed = ser / (ghz * 1e9); + gflops_sec = gflops / elapsed; + bw_realized = traffic / elapsed; + + printf("laplace3d_orig completed in %.2lf seconds\n", elapsed); + printf("GFLOPs/sec: %.1f\n", gflops_sec); + printf("BW realized: %.1f\n", bw_realized); + + // verify + for (k = 0; k < nz; ++k) { + for (j = 0; j < ny; ++j) { + for (i = 0; i < nx; ++i) { + ind = i + j*nx + k*nx*ny; + p_ind = i + j*padded_nx + k*padded_nx*ny; + + if (fabs(u1[p_ind] - u1_orig[ind]) > error_tol) { + printf("ERROR %f - %f [%d, %d, %d]\n", + u1[p_ind], u1_orig[ind], i, j, k); + goto done; + } + } + } + } + printf("verified, no error\n"); + } + + done: + _mm_free(u1_p); + _mm_free(u2_p); + _mm_free(u1_orig); + _mm_free(u2_orig); + + return 0; +} + +void l3d_naive(int nx, int padded_nx, int ny, int nz, float *u1, float *u2) +{ + int i, j, k, ind; + const float sixth = 1.0f/6.0f; + + /* compute on the grid */ + #pragma omp parallel for private(i,j,k,ind) + for (k = 1; k < nz-1; ++k) { + for (j = 1; j < ny-1; ++j) { + #pragma ivdep + for (i = 1; i < nx-1; ++i) { + ind = i + j*padded_nx + k*padded_nx*ny; + u2[ind] = + ( u1[ind-1 ] + u1[ind+1 ] + + u1[ind-padded_nx ] + u1[ind+padded_nx ] + + u1[ind-padded_nx*ny] + u1[ind+padded_nx*ny] ) * sixth; + } + } + } +} + +void l3d_auto(int nx, int padded_nx, int ny, int nz, float *u1, float *u2) +{ + int i, j, k, ind; + + float sixth = 1.0f/6.0f; + +#if defined(__INTEL_COMPILER) + __assume(padded_nx%8==0); + __assume_aligned(&u1[1],32); + __assume_aligned(&u2[1],32); +#elif defined(__GNUC__) + if (!(padded_nx%8==0)) + __builtin_unreachable(); +#if __has_builtin(__builtin_assume_aligned) + __builtin_assume_aligned(&u1[1],32); + __builtin_assume_aligned(&u2[1],32); +#endif +#endif + + /* compute on the grid */ + #pragma omp parallel for private(i,j,k,ind) + for (k = 1; k < nz-1; ++k) { + for (j = 1; j < ny-1; ++j) { + #pragma vector nontemporal(u2) + for (i = 1; i < nx-1; ++i) { + ind = i + j*padded_nx + k*padded_nx*ny; + u2[ind] = + ( u1[ind-1 ] + u1[ind+1 ] + + u1[ind-padded_nx ] + u1[ind+padded_nx ] + + u1[ind-padded_nx*ny] + u1[ind+padded_nx*ny] ) * sixth; + } + } + } +} + +void l3d_sse(int nx, int padded_nx, int ny, int nz, float *u1, float *u2) +{ + int i, j, k, ind; + + float fsixth = 1.0f/6.0f; + __m128 sixth = _mm_set_ps1(fsixth); + + /* compute on the grid */ + #pragma omp parallel for private(i,j,k,ind) + for (k = 1; k < nz-1; ++k) { + for (j = 1; j < ny-1; ++j) { + for (i = 1; i < nx-1; i += 4) { + ind = i + j*padded_nx + k*padded_nx*ny; + + __m128 pSrc1 = _mm_loadu_ps(&u1[ind-1]); + __m128 pSrc2 = _mm_loadu_ps(&u1[ind+1]); + __m128 pSrc3 = _mm_load_ps(&u1[ind-padded_nx]); + __m128 pSrc4 = _mm_load_ps(&u1[ind+padded_nx]); + __m128 pSrc5 = _mm_load_ps(&u1[ind-padded_nx*ny]); + __m128 pSrc6 = _mm_load_ps(&u1[ind+padded_nx*ny]); + + __m128 sum1 = _mm_add_ps(pSrc1, pSrc2); + __m128 sum2 = _mm_add_ps(pSrc3, pSrc4); + __m128 sum3 = _mm_add_ps(pSrc5, pSrc6); + __m128 sum4 = _mm_add_ps(sum1, sum2); + __m128 vsum = _mm_add_ps(sum3, sum4); + + vsum = _mm_mul_ps(vsum, sixth); + + _mm_stream_ps(&u2[ind], vsum); + } + } + } +} + +void l3d_avx(int nx, int padded_nx, int ny, int nz, float *u1, float *u2) +{ + int i, j, k, ind; + + float fsixth = 1.0f/6.0f; + __m256 sixth = _mm256_set1_ps(fsixth); + + /* compute on the grid */ + #pragma omp parallel for private(i,j,k,ind) + for (k = 1; k < nz-1; ++k) { + for (j = 1; j < ny-1; ++j) { + for (i = 1; i < nx-1; i += 8) { + ind = i + j*padded_nx + k*padded_nx*ny; + + __m256 pSrc1 = _mm256_loadu_ps(&u1[ind-1]); + __m256 pSrc2 = _mm256_loadu_ps(&u1[ind+1]); + __m256 pSrc3 = _mm256_load_ps(&u1[ind-padded_nx]); + __m256 pSrc4 = _mm256_load_ps(&u1[ind+padded_nx]); + __m256 pSrc5 = _mm256_load_ps(&u1[ind-padded_nx*ny]); + __m256 pSrc6 = _mm256_load_ps(&u1[ind+padded_nx*ny]); + + __m256 sum1 = _mm256_add_ps(pSrc1, pSrc2); + __m256 sum2 = _mm256_add_ps(pSrc3, pSrc4); + __m256 sum3 = _mm256_add_ps(pSrc5, pSrc6); + __m256 sum4 = _mm256_add_ps(sum1, sum2); + __m256 vsum = _mm256_add_ps(sum3, sum4); + + vsum = _mm256_mul_ps(vsum, sixth); + + _mm256_stream_ps(&u2[ind], vsum); + } + } + } +} + +void l3d_orig(int nx, int ny, int nz, float *u1, float *u2) +{ + int i, j, k, ind; + const float sixth = 1.0f/6.0f; + + for (k = 0; k < nz; ++k) { + for (j = 0; j < ny; ++j) { + for (i = 0; i < nx; ++i) { + ind = i + j*nx + k*nx*ny; + + if (i == 0 || i == nx - 1 + || j == 0 || j == ny - 1 + || k == 0 || k == nz - 1) { + u2[ind] = u1[ind]; // Dirichlet b.c.'s + } + else { + u2[ind] = ( u1[ind-1 ] + u1[ind+1 ] + + u1[ind-nx ] + u1[ind+nx ] + + u1[ind-nx*ny] + u1[ind+nx*ny] ) * sixth; + } + } + } + } +} + diff --git a/test/perf/threads/laplace3d/laplace3d.jl b/test/perf/threads/laplace3d/laplace3d.jl new file mode 100644 index 0000000000000..5d153c1e329c6 --- /dev/null +++ b/test/perf/threads/laplace3d/laplace3d.jl @@ -0,0 +1,133 @@ +using Base.Cartesian +using Base.Threading + +const sixth = 1.0f0/6.0f0 +const error_tol = 0.00001 + +function stencil3d(u::Array{Float32,3}, k_1::Int64, k_2::Int64, k_3::Int64) + return (u[k_1-1, k_2, k_3 ] + u[k_1+1, k_2, k_3] + + u[k_1, k_2-1, k_3 ] + u[k_1, k_2+1, k_3] + + u[k_1, k_2, k_3-1] + u[k_1, k_2, k_3+1]) * sixth +end + +function l3d_orig(u1::Array{Float32,3}, u3::Array{Float32,3}, + nx::Int64, ny::Int64, nz::Int64) + @nloops 3 k u1 begin + if @nany 3 d->(k_d == 1 || k_d == size(u1, d)) + @inbounds (@nref 3 u3 k) = (@nref 3 u1 k) + else + @inbounds (@nref 3 u3 k) = stencil3d(u1, k_1, k_2, k_3) + end + end +end + +function l3d_threadfun(u1, u3, nx, ny, nz) + tid = threadid() + tnz, rem = divrem(nz-2, nthreads()) + z_start = 2 + ((tid-1) * tnz) + z_end = z_start + tnz - 1 + if tid <= rem + z_start = z_start + tid - 1 + z_end = z_end + tid + else + z_start = z_start + rem + z_end = z_end + rem + end + + for k_3 = z_start:z_end + for k_2 = 2:ny-1 + @simd for k_1 = 2:nx-1 + @inbounds u3[k_1, k_2, k_3] = stencil3d(u1, k_1, k_2, k_3) + end + end + end +end + +function l3d_threadblock(u1, u3, nx, ny, nz) + @threads all begin + tid = threadid() + tnz, rem = divrem(nz-2, nthreads()) + z_start = 2 + ((tid-1) * tnz) + z_end = z_start + tnz - 1 + if tid <= rem + z_start = z_start + tid - 1 + z_end = z_end + tid + else + z_start = z_start + rem + z_end = z_end + rem + end + + for k_3 = z_start:z_end + for k_2 = 2:ny-1 + @simd for k_1 = 2:nx-1 + @inbounds u3[k_1, k_2, k_3] = stencil3d(u1, k_1, k_2, k_3) + end + end + end + end +end + +function l3d_threadfor(u1, u3, nx, ny, nz) + @threads all for k_3=2:nz-1 + for k_2 = 2:ny-1 + @simd for k_1 = 2:nx-1 + @inbounds u3[k_1, k_2, k_3] = stencil3d(u1, k_1, k_2, k_3) + end + end + end +end + +precompile(l3d_threadfun, (Array{Float32,3}, Array{Float32,3}, Int64, Int64, Int64)) +precompile(l3d_threadblock, (Array{Float32,3}, Array{Float32,3}, Int64, Int64, Int64)) +precompile(l3d_threadfor, (Array{Float32,3}, Array{Float32,3}, Int64, Int64, Int64)) + +function laplace3d(nx=258, ny=258, nz=258; iters=100, verify=false) + u1 = Array(Float32, nx, ny, nz) + u3 = Array(Float32, nx, ny, nz) + @nloops 3 k u1 begin + if @nany 3 d->(k_d == 1 || k_d == size(u1, d)) + (@nref 3 u3 k) = (@nref 3 u1 k) = 1.0 + else + (@nref 3 u1 k) = 0.0 + end + end + @time for n in 1:iters + # @threads all l3d_threadfun(u1, u3, nx, ny, nz) + # l3d_threadblock(u1, u3, nx, ny, nz) + # l3d_threadfor(u1, u3, nx, ny, nz) + ccall(:jl_threading_run, Void, (Any, Any), l3d_threadfun, (u1, u3, nx, ny, nz)) + foo = u1 + u1 = u3 + u3 = foo + end + if verify + u1_orig = Array(Float32, nx, ny, nz) + u3_orig = Array(Float32, nx, ny, nz) + @nloops 3 k u1_orig begin + if @nany 3 d->(k_d == 1 || k_d == size(u1_orig, d)) + (@nref 3 u3_orig k) = (@nref 3 u1_orig k) = 1.0 + else + (@nref 3 u1_orig k) = 0.0 + end + end + @time for n in 1:iters + l3d_orig(u1_orig, u3_orig, nx, ny, nz) + foo = u1_orig + u1_orig = u3_orig + u3_orig = foo + end + @nloops 3 k u1 begin + if abs((@nref 3 u1 k) - (@nref 3 u1_orig k)) > error_tol + error(@sprintf("Verify error: %f - %f [%d, %d, %d]\n", + (@nref 3 u1 k), (@nref 3 u1_orig k), k_1, k_2, k_3)) + end + end + println("verification succeeded") + end +end + +gc() + +laplace3d(verify=true) +#laplace3d(iters=1000, verify=false) + diff --git a/test/perf/threads/laplace3d/laplace3d.m b/test/perf/threads/laplace3d/laplace3d.m new file mode 100644 index 0000000000000..f0431e02a1bd2 --- /dev/null +++ b/test/perf/threads/laplace3d/laplace3d.m @@ -0,0 +1,36 @@ +nx = 34; +ny = 34; +nz = 34; +niters = 100; +sixth = 1.0/6.0; + +u1 = zeros(nx, ny, nz); +u2 = zeros(nx, ny, nz); + +for k3 = 1:nz + for k2 = 1:ny + for k1 = 1:nx + if k1==1 || k1==nx || k2==1 || k2==ny || k3==1 || k3==nz + u1(k1,k2,k3) = 1.0; + u2(k1,k2,k3) = 1.0; + endif + end + end +end + +tic() +for n = 1:niters + for k3 = 2:nz-1 + for k2 = 2:ny-1 + for k1 = 2:nx-1 + s = (u1(k1-1,k2,k3) + u1(k1+1,k2,k3) + u1(k1,k2-1,k3) + u1(k1,k2+1,k3) + u1(k1,k2,k3-1) + u1(k1,k2,k3+1)) * sixth; + u2(k1,k2,k3) = s; + end + end + end + foo = u1; + u1 = u2; + u2 = foo; +end +toc() + diff --git a/test/runtests.jl b/test/runtests.jl index e2d6ae4f42782..34373b29c9294 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -11,7 +11,7 @@ testnames = [ "sysinfo", "rounding", "ranges", "mod2pi", "euler", "show", "lineedit", "replcompletions", "repl", "sets", "test", "goto", "llvmcall", "grisu", "nullable", "meta", "profile", - "libgit2", "docs", "base64", "parser" + "libgit2", "docs", "base64", "parser", "threads" ] if isdir(joinpath(JULIA_HOME, Base.DOCDIR, "examples")) diff --git a/test/threads.jl b/test/threads.jl new file mode 100644 index 0000000000000..b924a0da32fcd --- /dev/null +++ b/test/threads.jl @@ -0,0 +1,49 @@ +using Base.Test +using Base.Threading + +println("Threading tests") + +expected = [1:nthreads()] + +# test 1 +arr = zeros(Int16, nthreads()) + +function foo(A) + tid = threadid() + A[tid] = tid +end + +@time @threads all foo(arr) + +@test arr == expected + + +# test 2 +arr = zeros(Int16, nthreads()) + +function bar(A) + @threads all for i = 1:nthreads() + tid = threadid() + A[i] = tid + end +end + +@time bar(arr) + +@test arr == expected + + +# test 3 +arr = zeros(Int16, nthreads()) + +function baz(A) + @threads all begin + tid = threadid() + A[tid] = tid + end +end + +@time baz(arr) + +@test arr == expected + From 6cfe327ff8e34e7ac6ebea00145ed2cf82854060 Mon Sep 17 00:00:00 2001 From: kpamnany Date: Tue, 27 Jan 2015 15:14:12 +0530 Subject: [PATCH 0002/1938] Fix for the parallel for form. --- base/threading.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/threading.jl b/base/threading.jl index cbea2ebbab83f..5af33a846deeb 100644 --- a/base/threading.jl +++ b/base/threading.jl @@ -16,7 +16,7 @@ function _threadsfor(forexpr) function $fun() tid = threadid() # divide loop iterations among threads - len, rem = divrem($ll-$lf+1, nthreads()) + len, rem = divrem($(esc(ll))-$(esc(lf))+1, nthreads()) # not enough iterations for all the threads? if len == 0 if tid > rem @@ -25,7 +25,7 @@ function _threadsfor(forexpr) len, rem = 1, 0 end # compute this thread's range - f = $lf + ((tid-1) * len) + f = $(esc(lf)) + ((tid-1) * len) l = f + len - 1 # distribute remaining iterations evenly if rem > 0 From 8c9820508a32ab467d2d1b8228ea55b95072b82e Mon Sep 17 00:00:00 2001 From: "Arch D. Robison" Date: Fri, 13 Feb 2015 09:09:05 -0600 Subject: [PATCH 0003/1938] Change LLVM_GIT_URL_LLVM to use "https:" instead of "git@", since some firewalls cannot deal with the latter. --- Make.user | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Make.user b/Make.user index 69a5cadd04efa..eb5fe551583a0 100644 --- a/Make.user +++ b/Make.user @@ -2,4 +2,4 @@ CC = clang CFLAGS = -ftls-model=global-dynamic CXX = clang++ LLVM_VER = svn -LLVM_GIT_URL_LLVM = git@github.com:JuliaLang/llvm.git -b jl/thread-local-storage +LLVM_GIT_URL_LLVM = https://github.com/JuliaLang/llvm.git -b jl/thread-local-storage From 65c9d8984c72976bf1776802f06bc500b1066383 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 18 Mar 2015 15:07:31 -0400 Subject: [PATCH 0004/1938] inlining improvement that helps #10527 this avoids inserting temporary variable copies of local variables unless they might be assigned by an inner function. Conflicts: base/inference.jl --- base/inference.jl | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 1dfc6d3c3b0c7..ade3fd7ab30ba 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -13,6 +13,7 @@ type StaticVarInfo sp::Tuple # static parameters tuple cenv::ObjectIdDict # types of closed vars vars::Array{Any,1} # names of args and locals + vinfo::Array{Any,1} # variable properties label_counter::Int # index of the current highest label for this function end @@ -75,6 +76,14 @@ end is_local(sv::StaticVarInfo, s::Symbol) = contains_is(sv.vars, s) is_closed(sv::StaticVarInfo, s::Symbol) = haskey(sv.cenv, s) +function is_assigned_inner(sv::StaticVarInfo, s::Symbol) + for vi in sv.vinfo + if vi[1] === s + return (vi[3]&4) != 0 + end + end + return false +end is_global(sv::StaticVarInfo, s::Symbol) = !is_local(sv,s) && !is_closed(sv,s) && !is_static_parameter(sv,s) @@ -1412,7 +1421,7 @@ function typeinf(linfo::LambdaStaticData,atypes::Tuple,sparams::Tuple, def, cop) # types of closed vars cenv = ObjectIdDict() - for vi = ((ast.args[2][3])::Array{Any,1}) + for vi in (ast.args[2][3])::Array{Any,1} vi::Array{Any,1} vname = vi[1] vtype = vi[2] @@ -1430,7 +1439,7 @@ function typeinf(linfo::LambdaStaticData,atypes::Tuple,sparams::Tuple, def, cop) s[1][vname] = vtype end end - sv = StaticVarInfo(sparams, cenv, vars, label_counter(body)) + sv = StaticVarInfo(sparams, cenv, vars, vinflist, label_counter(body)) frame.sv = sv recpts = IntSet() # statements that depend recursively on our value @@ -1990,15 +1999,17 @@ end function effect_free(e::ANY, sv, allow_volatile::Bool) if isa(e,SymbolNode) allow_volatile && return true - if is_global(sv, (e::SymbolNode).name) + if is_assigned_inner(sv, (e::SymbolNode).name) || is_global(sv, (e::SymbolNode).name) return false end + return true end if isa(e,Symbol) allow_volatile && return true - if is_global(sv, e::Symbol) + if is_assigned_inner(sv, e::Symbol) || is_global(sv, e::Symbol) return false end + return true end if isa(e,Number) || isa(e,AbstractString) || isa(e,TopNode) || isa(e,QuoteNode) || isa(e,Type) || isa(e,Tuple) From 0afad357ebedaa007560757debffca1d55531697 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 18 Mar 2015 15:39:50 -0400 Subject: [PATCH 0005/1938] fix merge mistake in fix for #10527 --- base/inference.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/inference.jl b/base/inference.jl index ade3fd7ab30ba..a4505d03ca155 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1428,7 +1428,8 @@ function typeinf(linfo::LambdaStaticData,atypes::Tuple,sparams::Tuple, def, cop) cenv[vname] = vtype s[1][vname] = vtype end - for vi = ((ast.args[2][2])::Array{Any,1}) + vinflist = (ast.args[2][2])::Array{Any,1} + for vi in vinflist vi::Array{Any,1} if (vi[3]&4)!=0 # variables assigned by inner functions are treated like From 6c7e175864cac64111cebf65f13b17d85c377b33 Mon Sep 17 00:00:00 2001 From: "Arch D. Robison" Date: Fri, 27 Mar 2015 13:06:36 -0500 Subject: [PATCH 0006/1938] Cosmetic change to bring closer to master: __JL_THREAD -> JL_THREAD --- src/julia.h | 14 +++++++------- src/julia_internal.h | 4 ++-- src/task.c | 16 ++++++++-------- src/threading.c | 4 ++-- src/threading.h | 2 +- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/julia.h b/src/julia.h index 8254f59d69132..a94cdded9c034 100644 --- a/src/julia.h +++ b/src/julia.h @@ -21,12 +21,12 @@ extern "C" { # else # define MAX_ALIGN sizeof(void*) # endif -# define __JL_THREAD __thread +# define JL_THREAD __thread #else # define jl_jmp_buf jmp_buf # include //for _resetstkoflw # define MAX_ALIGN 8 -# define __JL_THREAD __declspec(thread) +# define JL_THREAD __declspec(thread) #endif #ifdef _P64 @@ -1023,7 +1023,7 @@ typedef struct _jl_gcframe_t { // jl_value_t *x=NULL, *y=NULL; JL_GC_PUSH(&x, &y); // x = f(); y = g(); foo(x, y) -extern DLLEXPORT __JL_THREAD jl_gcframe_t *jl_pgcstack; +extern DLLEXPORT JL_THREAD jl_gcframe_t *jl_pgcstack; #define JL_GC_PUSH(...) \ void *__gc_stkf[] = {(void*)((VA_NARG(__VA_ARGS__)<<1)|1), jl_pgcstack, \ @@ -1259,10 +1259,10 @@ typedef struct { jl_value_t **ptask_arg_in_transit; } jl_thread_task_state_t; -extern DLLEXPORT __JL_THREAD jl_task_t *jl_current_task; -extern DLLEXPORT __JL_THREAD jl_task_t *jl_root_task; -extern DLLEXPORT __JL_THREAD jl_value_t *jl_exception_in_transit; -extern DLLEXPORT __JL_THREAD jl_value_t *jl_task_arg_in_transit; +extern DLLEXPORT JL_THREAD jl_task_t *jl_current_task; +extern DLLEXPORT JL_THREAD jl_task_t *jl_root_task; +extern DLLEXPORT JL_THREAD jl_value_t *jl_exception_in_transit; +extern DLLEXPORT JL_THREAD jl_value_t *jl_task_arg_in_transit; DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize); jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg); diff --git a/src/julia_internal.h b/src/julia_internal.h index 9f450d5208294..fb1f1907926e1 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -93,9 +93,9 @@ void jl_init_serializer(void); void _julia_init(JL_IMAGE_SEARCH rel); #ifdef COPY_STACKS -extern __JL_THREAD void *jl_stackbase; +extern JL_THREAD void *jl_stackbase; #ifndef ASM_COPY_STACKS -extern __JL_THREAD jl_jmp_buf jl_base_ctx; +extern JL_THREAD jl_jmp_buf jl_base_ctx; #endif #endif diff --git a/src/task.c b/src/task.c index 49215c880c619..e02fbd0e407c7 100644 --- a/src/task.c +++ b/src/task.c @@ -143,26 +143,26 @@ extern size_t jl_page_size; jl_datatype_t *jl_task_type; #ifdef JL_GC_MARKSWEEP -__JL_THREAD jl_gcframe_t *jl_pgcstack = NULL; +JL_THREAD jl_gcframe_t *jl_pgcstack = NULL; #endif -DLLEXPORT __JL_THREAD jl_task_t *jl_current_task; -DLLEXPORT __JL_THREAD jl_task_t *jl_root_task; -DLLEXPORT __JL_THREAD jl_value_t *jl_task_arg_in_transit; -DLLEXPORT __JL_THREAD jl_value_t *jl_exception_in_transit; +DLLEXPORT JL_THREAD jl_task_t *jl_current_task; +DLLEXPORT JL_THREAD jl_task_t *jl_root_task; +DLLEXPORT JL_THREAD jl_value_t *jl_task_arg_in_transit; +DLLEXPORT JL_THREAD jl_value_t *jl_exception_in_transit; static void start_task(); #ifdef COPY_STACKS -__JL_THREAD jl_jmp_buf * volatile jl_jmp_target; +JL_THREAD jl_jmp_buf * volatile jl_jmp_target; #if defined(_CPU_X86_64_) || defined(_CPU_X86_) #define ASM_COPY_STACKS #endif -__JL_THREAD void *jl_stackbase; +JL_THREAD void *jl_stackbase; #ifndef ASM_COPY_STACKS -__JL_THREAD jl_jmp_buf jl_base_ctx; // base context of stack +JL_THREAD jl_jmp_buf jl_base_ctx; // base context of stack #endif #if defined(_OS_WINDOWS_) && !defined(_COMPILER_MINGW_) diff --git a/src/threading.c b/src/threading.c index ab0624c17931b..00885fec92f0f 100644 --- a/src/threading.c +++ b/src/threading.c @@ -62,10 +62,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. JL_DEFINE_MUTEX(codegen); // thread ID -__JL_THREAD int16_t ti_tid = 0; +JL_THREAD int16_t ti_tid = 0; // thread heap -__JL_THREAD struct _jl_thread_heap_t *jl_thread_heap; +JL_THREAD struct _jl_thread_heap_t *jl_thread_heap; struct _jl_thread_heap_t **jl_all_heaps; jl_gcframe_t ***jl_all_pgcstacks; jl_thread_task_state_t *jl_all_task_states; diff --git a/src/threading.h b/src/threading.h index 2c615f4935531..7b6b6dbf2ef85 100644 --- a/src/threading.h +++ b/src/threading.h @@ -39,7 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. extern __thread int16_t ti_tid; // GC -extern __JL_THREAD struct _jl_thread_heap_t *jl_thread_heap; +extern JL_THREAD struct _jl_thread_heap_t *jl_thread_heap; extern struct _jl_thread_heap_t **jl_all_heaps; extern jl_gcframe_t ***jl_all_pgcstacks; extern jl_thread_task_state_t *jl_all_task_states; From 1db03d221ba7159adcaf78fce04353ce4f40b440 Mon Sep 17 00:00:00 2001 From: "Arch D. Robison" Date: Fri, 27 Mar 2015 14:13:13 -0500 Subject: [PATCH 0007/1938] More cosmetic changes to bring closer to master. --- src/julia.h | 185 +++++++++++++++++++++++++----------------------- src/threading.c | 2 +- 2 files changed, 99 insertions(+), 88 deletions(-) diff --git a/src/julia.h b/src/julia.h index a94cdded9c034..08ef149a22b1c 100644 --- a/src/julia.h +++ b/src/julia.h @@ -21,12 +21,10 @@ extern "C" { # else # define MAX_ALIGN sizeof(void*) # endif -# define JL_THREAD __thread #else # define jl_jmp_buf jmp_buf # include //for _resetstkoflw # define MAX_ALIGN 8 -# define JL_THREAD __declspec(thread) #endif #ifdef _P64 @@ -53,6 +51,104 @@ extern "C" { #define JL_ATTRIBUTE_ALIGN_PTRSIZE(x) #endif +// threading ------------------------------------------------------------------ + +// WARNING: Threading support is incomplete. Changing the 1 to a 0 will break Julia. +// Nonetheless, we define JL_THREAD and use it to give advanced notice to maintainers +// of what eventual threading support will change. +#if 0 +// Definition for compiling non-thread-safe Julia. +# define JL_THREAD +#elif !defined(_OS_WINDOWS_) +// Definition for compiling Julia on platforms with GCC __thread. +# define JL_THREAD __thread +#else +// Definition for compiling Julia on Windows +# define JL_THREAD __declspec(thread) +#endif + +DLLEXPORT int16_t jl_threadid(void); +DLLEXPORT void *jl_threadgroup(void); +DLLEXPORT void jl_cpu_pause(void); +DLLEXPORT void jl_threading_profile(); + +#if __GNUC__ +# define JL_ATOMIC_FETCH_AND_ADD(a,b) \ + __sync_fetch_and_add(&(a), (b)) +# define JL_ATOMIC_COMPARE_AND_SWAP(a,b,c) \ + __sync_bool_compare_and_swap(&(a), (b), (c)) +# define JL_ATOMIC_TEST_AND_SET(a) \ + __sync_lock_test_and_set(&(a), 1) +# define JL_ATOMIC_RELEASE(a) \ + __sync_lock_release(&(a)) +#elif _WIN32 +# define JL_ATOMIC_FETCH_AND_ADD(a,b) \ + _InterlockedExchangeAdd((volatile LONG *)&(a), (b)) +# define JL_ATOMIC_COMPARE_AND_SWAP(a,b,c) \ + _InterlockedCompareExchange64(&(a), (c), (b)) +# define JL_ATOMIC_TEST_AND_SET(a) \ + _InterlockedExchange64(&(a), 1) +# define JL_ATOMIC_RELEASE(a) \ + _InterlockedExchange64(&(a), 0) +#else +# error "No atomic operations supported." +#endif + +#if 0 +#define JL_DEFINE_MUTEX(m) \ + uv_mutex_t m ## _mutex; \ + uint64_t m ## _thread_id = -1; + +#define JL_DEFINE_MUTEX_EXT(m) \ + extern uv_mutex_t m ## _mutex; \ + extern uint64_t m ## _thread_id; + +#define JL_LOCK(m) \ + int m ## locked = 0; \ + if (m ## _thread_id != uv_thread_self()) { \ + uv_mutex_lock(&m ## _mutex); \ + m ## locked = 1; \ + m ## _thread_id = uv_thread_self(); \ + } + +#define JL_UNLOCK(m) \ + if (m ## locked) { \ + m ## _thread_id = -1; \ + m ## locked = 0; \ + uv_mutex_unlock(&m ## _mutex); \ + } +#else +#define JL_DEFINE_MUTEX(m) \ + uint64_t volatile m ## _mutex = 0; \ + int32_t m ## _lock_count = 0; + +#define JL_DEFINE_MUTEX_EXT(m) \ + extern uint64_t volatile m ## _mutex; \ + extern int32_t m ## _lock_count; + +#define JL_LOCK(m) \ + if (m ## _mutex == uv_thread_self()) \ + ++m ## _lock_count; \ + else { \ + for (; ;) { \ + if (m ## _mutex == 0 && \ + JL_ATOMIC_COMPARE_AND_SWAP(m ## _mutex, 0, \ + uv_thread_self())) { \ + m ## _lock_count = 1; \ + break; \ + } \ + jl_cpu_pause(); \ + } \ + } + +#define JL_UNLOCK(m) \ + if (m ## _mutex == uv_thread_self()) { \ + --m ## _lock_count; \ + if (m ## _lock_count == 0) \ + JL_ATOMIC_COMPARE_AND_SWAP(m ## _mutex, uv_thread_self(), 0); \ + } +#endif + // core data types ------------------------------------------------------------ #ifdef OVERLAP_TUPLE_LEN @@ -1106,91 +1202,6 @@ STATIC_INLINE void *alloc_4w() { return allocobj(4*sizeof(void*)); } #define allocobj(nb) malloc(nb) #endif -// threading ------------------------------------------------------------------ - -DLLEXPORT int16_t jl_threadid(void); -DLLEXPORT void *jl_threadgroup(void); -DLLEXPORT void jl_cpu_pause(void); -DLLEXPORT jl_value_t *jl_threading_run(jl_function_t *fun, jl_tuple_t *args); -DLLEXPORT void jl_threading_profile(); - -#if __GNUC__ -# define JL_ATOMIC_FETCH_AND_ADD(a,b) \ - __sync_fetch_and_add(&(a), (b)) -# define JL_ATOMIC_COMPARE_AND_SWAP(a,b,c) \ - __sync_bool_compare_and_swap(&(a), (b), (c)) -# define JL_ATOMIC_TEST_AND_SET(a) \ - __sync_lock_test_and_set(&(a), 1) -# define JL_ATOMIC_RELEASE(a) \ - __sync_lock_release(&(a)) -#elif _WIN32 -# define JL_ATOMIC_FETCH_AND_ADD(a,b) \ - _InterlockedExchangeAdd((volatile LONG *)&(a), (b)) -# define JL_ATOMIC_COMPARE_AND_SWAP(a,b,c) \ - _InterlockedCompareExchange64(&(a), (c), (b)) -# define JL_ATOMIC_TEST_AND_SET(a) \ - _InterlockedExchange64(&(a), 1) -# define JL_ATOMIC_RELEASE(a) \ - _InterlockedExchange64(&(a), 0) -#else -# error "No atomic operations supported." -#endif - -#if 0 -#define JL_DEFINE_MUTEX(m) \ - uv_mutex_t m ## _mutex; \ - uint64_t m ## _thread_id = -1; - -#define JL_DEFINE_MUTEX_EXT(m) \ - extern uv_mutex_t m ## _mutex; \ - extern uint64_t m ## _thread_id; - -#define JL_LOCK(m) \ - int m ## locked = 0; \ - if (m ## _thread_id != uv_thread_self()) { \ - uv_mutex_lock(&m ## _mutex); \ - m ## locked = 1; \ - m ## _thread_id = uv_thread_self(); \ - } - -#define JL_UNLOCK(m) \ - if (m ## locked) { \ - m ## _thread_id = -1; \ - m ## locked = 0; \ - uv_mutex_unlock(&m ## _mutex); \ - } -#else -#define JL_DEFINE_MUTEX(m) \ - uint64_t volatile m ## _mutex = 0; \ - int32_t m ## _lock_count = 0; - -#define JL_DEFINE_MUTEX_EXT(m) \ - extern uint64_t volatile m ## _mutex; \ - extern int32_t m ## _lock_count; - -#define JL_LOCK(m) \ - if (m ## _mutex == uv_thread_self()) \ - ++m ## _lock_count; \ - else { \ - for (; ;) { \ - if (m ## _mutex == 0 && \ - JL_ATOMIC_COMPARE_AND_SWAP(m ## _mutex, 0, \ - uv_thread_self())) { \ - m ## _lock_count = 1; \ - break; \ - } \ - jl_cpu_pause(); \ - } \ - } - -#define JL_UNLOCK(m) \ - if (m ## _mutex == uv_thread_self()) { \ - --m ## _lock_count; \ - if (m ## _lock_count == 0) \ - JL_ATOMIC_COMPARE_AND_SWAP(m ## _mutex, uv_thread_self(), 0); \ - } -#endif - // async signal handling ------------------------------------------------------ #include diff --git a/src/threading.c b/src/threading.c index 00885fec92f0f..161946269a28d 100644 --- a/src/threading.c +++ b/src/threading.c @@ -356,7 +356,7 @@ void jl_cpu_pause() // interface to user code: specialize and compile the user thread function // and run it in all threads -jl_value_t *jl_threading_run(jl_function_t *f, jl_tuple_t *args) +DLLEXPORT jl_value_t *jl_threading_run(jl_function_t *f, jl_tuple_t *args) { #if PROFILE_JL_THREADING uint64_t tstart = rdtsc(); From 4ee059e7eb6a77cedc531ab353e524bdd0c312ed Mon Sep 17 00:00:00 2001 From: "Arch D. Robison" Date: Wed, 1 Apr 2015 14:26:52 -0500 Subject: [PATCH 0008/1938] Bring tasking up to version in master branch. --- src/init.c | 4 +- src/julia.h | 9 ++- src/julia_internal.h | 11 +-- src/support/dtypes.h | 8 ++ src/task.c | 176 ++++++++++++++++++------------------------- src/threading.c | 6 +- 6 files changed, 95 insertions(+), 119 deletions(-) diff --git a/src/init.c b/src/init.c index 49c8f062ac6cf..1bf06eed8702c 100644 --- a/src/init.c +++ b/src/init.c @@ -991,8 +991,8 @@ void _julia_init(JL_IMAGE_SEARCH rel) #endif jl_init_frontend(); jl_init_types(); - jl_init_tasks(jl_stack_lo, jl_stack_hi-jl_stack_lo); - jl_init_root_task(); + jl_init_tasks(); + jl_init_root_task(jl_stack_lo, jl_stack_hi-jl_stack_lo); jl_init_codegen(); jl_start_threads(); diff --git a/src/julia.h b/src/julia.h index 08ef149a22b1c..19936b6842794 100644 --- a/src/julia.h +++ b/src/julia.h @@ -519,6 +519,7 @@ extern jl_sym_t *arrow_sym; extern jl_sym_t *ldots_sym; #else #define jl_typeof(v) (((jl_value_t*)(v))->type) #endif +#define jl_set_typeof(v,t) ((v)->type = (jl_value_t*)(t)) #define jl_typeis(v,t) (jl_typeof(v)==(jl_value_t*)(t)) #ifdef OVERLAP_TUPLE_LEN @@ -1264,16 +1265,16 @@ typedef struct _jl_task_t { } jl_task_t; typedef struct { - jl_task_t **pcurrent_task; + jl_task_t * volatile *pcurrent_task; jl_task_t **proot_task; jl_value_t **pexception_in_transit; - jl_value_t **ptask_arg_in_transit; + jl_value_t * volatile *ptask_arg_in_transit; } jl_thread_task_state_t; -extern DLLEXPORT JL_THREAD jl_task_t *jl_current_task; +extern DLLEXPORT JL_THREAD jl_task_t *volatile jl_current_task; extern DLLEXPORT JL_THREAD jl_task_t *jl_root_task; extern DLLEXPORT JL_THREAD jl_value_t *jl_exception_in_transit; -extern DLLEXPORT JL_THREAD jl_value_t *jl_task_arg_in_transit; +extern DLLEXPORT JL_THREAD jl_value_t * volatile jl_task_arg_in_transit; DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize); jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg); diff --git a/src/julia_internal.h b/src/julia_internal.h index fb1f1907926e1..01ee450440f54 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -87,20 +87,17 @@ void jl_init_frontend(void); void jl_init_primitives(void); void jl_init_codegen(void); void jl_init_intrinsic_functions(void); -void jl_init_tasks(void *stack, size_t ssize); -void jl_init_root_task(void); +void jl_init_tasks(void); +void jl_init_root_task(void *stack, size_t ssize); void jl_init_serializer(void); void _julia_init(JL_IMAGE_SEARCH rel); #ifdef COPY_STACKS extern JL_THREAD void *jl_stackbase; -#ifndef ASM_COPY_STACKS -extern JL_THREAD jl_jmp_buf jl_base_ctx; -#endif #endif -void jl_set_stackbase(void); -void jl_set_base_ctx(void); +void jl_set_stackbase(char *__stk); +void jl_set_base_ctx(char *__stk); void jl_init_threading(void); void jl_start_threads(void); diff --git a/src/support/dtypes.h b/src/support/dtypes.h index 11a288bc7ea07..74d36007140c6 100644 --- a/src/support/dtypes.h +++ b/src/support/dtypes.h @@ -131,6 +131,14 @@ # define INLINE inline #endif +#if defined(_OS_WINDOWS_) && !defined(_COMPILER_MINGW_) +# define NOINLINE __declspec(noinline) +# define NOINLINE_DECL(f) __declspec(noinline) f +#else +# define NOINLINE __attribute__((noinline)) +# define NOINLINE_DECL(f) f __attribute__((noinline)) +#endif + typedef int bool_t; typedef unsigned char byte_t; /* 1 byte */ diff --git a/src/task.c b/src/task.c index e02fbd0e407c7..f9ce786b60c82 100644 --- a/src/task.c +++ b/src/task.c @@ -7,7 +7,6 @@ #include #include #include -//#include #include #include #include "julia.h" @@ -40,16 +39,16 @@ static int _stack_grows_up; static size_t _frame_offset; struct _probe_data { - intptr_t low_bound; /* below probe on stack */ - intptr_t probe_local; /* local to probe on stack */ - intptr_t high_bound; /* above probe on stack */ - intptr_t prior_local; /* value of probe_local from earlier call */ + intptr_t low_bound; /* below probe on stack */ + intptr_t probe_local; /* local to probe on stack */ + intptr_t high_bound; /* above probe on stack */ + intptr_t prior_local; /* value of probe_local from earlier call */ - jl_jmp_buf probe_env; /* saved environment of probe */ - jl_jmp_buf probe_sameAR; /* second environment saved by same call */ - jl_jmp_buf probe_samePC; /* environment saved on previous call */ + jl_jmp_buf probe_env; /* saved environment of probe */ + jl_jmp_buf probe_sameAR; /* second environment saved by same call */ + jl_jmp_buf probe_samePC; /* environment saved on previous call */ - jl_jmp_buf * ref_probe; /* switches between probes */ + jl_jmp_buf * ref_probe; /* switches between probes */ }; static void boundhigh(struct _probe_data *p) @@ -75,11 +74,7 @@ static void boundlow(struct _probe_data *p) } // we need this function to exist so we can measure its stack frame! -#if defined(_OS_WINDOWS_) && !defined(_COMPILER_MINGW_) -static void __declspec(noinline) fill(struct _probe_data *p); -#else -static void fill(struct _probe_data *p) __attribute__ ((noinline)); -#endif +static void NOINLINE_DECL(fill(struct _probe_data *p)); static void fill(struct _probe_data *p) { @@ -135,26 +130,30 @@ static void _probe_arch(void) /* end probing code */ +/* + TODO: + - per-task storage (scheme-like parameters) + - stack growth +*/ + static jl_sym_t *done_sym; static jl_sym_t *failed_sym; static jl_sym_t *runnable_sym; extern size_t jl_page_size; jl_datatype_t *jl_task_type; - +DLLEXPORT JL_THREAD jl_task_t * volatile jl_current_task; +JL_THREAD jl_task_t *jl_root_task; +JL_THREAD jl_value_t * volatile jl_task_arg_in_transit; +JL_THREAD jl_value_t *jl_exception_in_transit; #ifdef JL_GC_MARKSWEEP JL_THREAD jl_gcframe_t *jl_pgcstack = NULL; #endif -DLLEXPORT JL_THREAD jl_task_t *jl_current_task; -DLLEXPORT JL_THREAD jl_task_t *jl_root_task; -DLLEXPORT JL_THREAD jl_value_t *jl_task_arg_in_transit; -DLLEXPORT JL_THREAD jl_value_t *jl_exception_in_transit; - static void start_task(); #ifdef COPY_STACKS -JL_THREAD jl_jmp_buf * volatile jl_jmp_target; +static JL_THREAD jl_jmp_buf * volatile jl_jmp_target; #if defined(_CPU_X86_64_) || defined(_CPU_X86_) #define ASM_COPY_STACKS @@ -162,15 +161,10 @@ JL_THREAD jl_jmp_buf * volatile jl_jmp_target; JL_THREAD void *jl_stackbase; #ifndef ASM_COPY_STACKS -JL_THREAD jl_jmp_buf jl_base_ctx; // base context of stack +static JL_THREAD jl_jmp_buf jl_base_ctx; // base context of stack #endif -#if defined(_OS_WINDOWS_) && !defined(_COMPILER_MINGW_) -static void __declspec(noinline) -#else -static void __attribute__((noinline)) -#endif -save_stack(jl_task_t *t) +static void NOINLINE save_stack(jl_task_t *t) { if (t->state == done_sym || t->state == failed_sym) return; @@ -189,12 +183,7 @@ save_stack(jl_task_t *t) memcpy(buf, (char*)&_x, nb); } -#if defined(_OS_WINDOWS_) && !defined(_COMPILER_MINGW_) -void __declspec(noinline) -#else -void __attribute__((noinline)) -#endif -restore_stack(jl_task_t *t, jl_jmp_buf *where, char *p) +void NOINLINE restore_stack(jl_task_t *t, jl_jmp_buf *where, char *p) { char *_x = (char*)jl_stackbase - t->ssize; if (!p) { @@ -209,18 +198,6 @@ restore_stack(jl_task_t *t, jl_jmp_buf *where, char *p) memcpy(_x, t->stkbuf, t->ssize); jl_longjmp(*jl_jmp_target, 1); } - -void jl_switch_stack(jl_task_t *t, jl_jmp_buf *where) -{ - assert(t == jl_current_task); - if (t->stkbuf == NULL) { - start_task(); - // doesn't return - } - else { - restore_stack(t, where, NULL); - } -} #endif static jl_function_t *task_done_hook_func=NULL; @@ -236,29 +213,23 @@ static void NORETURN finish_task(jl_task_t *t, jl_value_t *resultval) #ifdef COPY_STACKS t->stkbuf = NULL; #endif - if (ti_tid == 0) { - // for now only thread 0 runs the task scheduler - if (task_done_hook_func == NULL) { - task_done_hook_func = (jl_function_t*)jl_get_global(jl_base_module, - jl_symbol("task_done_hook")); - } - if (task_done_hook_func != NULL) { - jl_apply(task_done_hook_func, (jl_value_t**)&t, 1); - } - } - else { - // others return to thread loop + if (ti_tid != 0) { + // For now, only thread 0 runs the task scheduler. + // The others return to the thread loop jl_switchto(jl_root_task, jl_nothing); + abort(); + } + if (task_done_hook_func == NULL) { + task_done_hook_func = (jl_function_t*)jl_get_global(jl_base_module, + jl_symbol("task_done_hook")); + } + if (task_done_hook_func != NULL) { + jl_apply(task_done_hook_func, (jl_value_t**)&t, 1); } abort(); } -#if defined(_OS_WINDOWS_) && !defined(_COMPILER_MINGW_) -static void __declspec(noinline) -#else -static void __attribute__((noinline)) -#endif -NORETURN start_task() +static void NOINLINE NORETURN start_task() { // this runs the first time we switch to a task jl_task_t *t = jl_current_task; @@ -269,38 +240,35 @@ NORETURN start_task() } #ifdef COPY_STACKS -void jl_set_stackbase() +void jl_set_stackbase(char *__stk) { - char __stk; jl_stackbase = (char*)(((uptrint_t)&__stk + sizeof(__stk))&-16); // also ensures stackbase is 16-byte aligned } #else -void jl_set_stackbase() { } +void jl_set_stackbase(char *__stk) { } #endif #ifndef ASM_COPY_STACKS -#if defined(_OS_WINDOWS_) && !defined(_COMPILER_MINGW_) -void __declspec(noinline) -#else -void __attribute__((noinline)) -#endif -jl_set_base_ctx() +void NOINLINE jl_set_base_ctx(char *__stk) { if (jl_setjmp(jl_base_ctx, 1)) { start_task(); } } #else -void jl_set_base_ctx() { } +void jl_set_base_ctx(char *__stk) { } #endif + DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel) { // keep this function small, since we want to keep the stack frame // leading up to this also quite small _julia_init(rel); - - jl_set_stackbase(); - jl_set_base_ctx(); +#ifdef COPY_STACKS + char __stk; + jl_stackbase = (char*)(((uptrint_t)&__stk + sizeof(__stk))&-16); // also ensures stackbase is 16-byte aligned + jl_set_base_ctx(&__stk); // separate function, to record the size of a stack frame +#endif } #ifndef COPY_STACKS @@ -322,11 +290,6 @@ static void init_task(jl_task_t *t) } #endif -DLLEXPORT void jl_handle_stack_switch() -{ - jl_switch_stack(jl_current_task, jl_jmp_target); -} - static void ctx_switch(jl_task_t *t, jl_jmp_buf *where) { if (t == jl_current_task) @@ -699,14 +662,18 @@ static void record_backtrace(void) static jl_value_t *array_ptr_void_type = NULL; DLLEXPORT jl_value_t *jl_backtrace_from_here(void) { - if (array_ptr_void_type == NULL) - array_ptr_void_type = jl_apply_type((jl_value_t*)jl_array_type, - jl_tuple2(jl_voidpointer_type, - jl_box_long(1))); - jl_array_t *bt = jl_alloc_array_1d(array_ptr_void_type, MAX_BT_SIZE); + jl_tuple_t *tp = NULL; + jl_array_t *bt = NULL; + JL_GC_PUSH2(&tp, &bt); + if (array_ptr_void_type == NULL) { + tp = jl_tuple2(jl_voidpointer_type, jl_box_long(1)); + array_ptr_void_type = jl_apply_type((jl_value_t*)jl_array_type, tp); + } + bt = jl_alloc_array_1d(array_ptr_void_type, MAX_BT_SIZE); size_t n = rec_backtrace((ptrint_t*)jl_array_data(bt), MAX_BT_SIZE); if (n < MAX_BT_SIZE) jl_array_del_end(bt, MAX_BT_SIZE-n); + JL_GC_POP(); return (jl_value_t*)bt; } @@ -732,12 +699,16 @@ DLLEXPORT jl_value_t *jl_lookup_code_address(void *ip, int skipC) DLLEXPORT jl_value_t *jl_get_backtrace(void) { - if (array_ptr_void_type == NULL) - array_ptr_void_type = jl_apply_type((jl_value_t*)jl_array_type, - jl_tuple2(jl_voidpointer_type, - jl_box_long(1))); - jl_array_t *bt = jl_alloc_array_1d(array_ptr_void_type, bt_size); + jl_tuple_t *tp = NULL; + jl_array_t *bt = NULL; + JL_GC_PUSH2(&tp, &bt); + if (array_ptr_void_type == NULL) { + tp = jl_tuple2(jl_voidpointer_type, jl_box_long(1)); + array_ptr_void_type = jl_apply_type((jl_value_t*)jl_array_type, tp); + } + bt = jl_alloc_array_1d(array_ptr_void_type, bt_size); memcpy(bt->data, bt_data, bt_size*sizeof(void*)); + JL_GC_POP(); return (jl_value_t*)bt; } @@ -776,23 +747,23 @@ DLLEXPORT void gdbbacktrace() void NORETURN throw_internal(jl_value_t *e) { assert(e != NULL); - jl_exception_in_transit = e; if (jl_current_task->eh != NULL) { jl_longjmp(jl_current_task->eh->eh_ctx, 1); } else { if (jl_current_task == jl_root_task) { - JL_PRINTF(JL_STDERR, "fatal: error thrown and no exception handler available.\n"); + jl_printf(JL_STDERR, "fatal: error thrown and no exception handler available.\n"); jl_static_show(JL_STDERR, e); - JL_PRINTF(JL_STDERR, "\n"); + jl_printf(JL_STDERR, "\n"); + jlbacktrace(); jl_exit(1); } jl_current_task->exception = e; finish_task(jl_current_task, e); assert(0); } - jl_exit(1); + assert(0); } // record backtrace and raise an error @@ -822,7 +793,7 @@ DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize) { size_t pagesz = jl_page_size; jl_task_t *t = (jl_task_t*)allocobj(sizeof(jl_task_t)); - t->type = (jl_value_t*)jl_task_type; + jl_set_typeof(t, jl_task_type); ssize = LLT_ALIGN(ssize, pagesz); t->ssize = ssize; t->current_module = NULL; @@ -898,11 +869,9 @@ DLLEXPORT jl_value_t *jl_get_current_task(void) jl_function_t *jl_unprotect_stack_func; -void jl_init_tasks(void *stack, size_t ssize) +// Do one-time initializations for task system +void jl_init_tasks(void) { - (void)stack; // TODO make per-thread - (void)ssize; - _probe_arch(); jl_task_type = jl_new_datatype(jl_symbol("Task"), jl_any_type, @@ -932,10 +901,11 @@ void jl_init_tasks(void *stack, size_t ssize) jl_unprotect_stack_func = jl_new_closure(jl_unprotect_stack, (jl_value_t*)jl_null, NULL); } -void jl_init_root_task(void) +// Initialize a root task using the given stack. +void jl_init_root_task(void *stack, size_t ssize) { jl_current_task = (jl_task_t*)allocobj(sizeof(jl_task_t)); - jl_current_task->type = (jl_value_t*)jl_task_type; + jl_set_typeof(jl_current_task, jl_task_type); #ifdef COPY_STACKS jl_current_task->ssize = 0; // size of saved piece jl_current_task->bufsz = 0; diff --git a/src/threading.c b/src/threading.c index 161946269a28d..26df7b8a7e3ff 100644 --- a/src/threading.c +++ b/src/threading.c @@ -164,9 +164,9 @@ void *ti_threadfun(void *arg) ti_initthread(ta->tid); // set up tasking - jl_init_root_task(); - jl_set_stackbase(); - jl_set_base_ctx(); + jl_init_root_task(0,0); + jl_set_stackbase((char*)&arg); + jl_set_base_ctx((char*)&arg); // set the thread-local tid and wait for a thread group while (ta->state == TI_THREAD_INIT) From 766c60870575bf64208c785622279f761dd4af7e Mon Sep 17 00:00:00 2001 From: "Arch D. Robison" Date: Wed, 8 Apr 2015 15:25:14 -0500 Subject: [PATCH 0009/1938] Fix Relocation and CodeModel. --- src/codegen.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 000709b192746..d143a623d13b8 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5621,8 +5621,8 @@ extern "C" void jl_init_codegen(void) jl_TargetMachine->getTargetCPU(), jl_TargetMachine->getTargetFeatureString(), jl_TargetMachine->Options, - Reloc::Default, - CodeModel::JITDefault, + Reloc::PIC_, + CodeModel::Default, #ifdef DISABLE_OPT CodeGenOpt::None // -O3 #else From 6311538e49d1c94b341cb861dc52b900f0ae6d71 Mon Sep 17 00:00:00 2001 From: "Arch D. Robison" Date: Fri, 10 Apr 2015 15:25:18 -0500 Subject: [PATCH 0010/1938] Use https://github.com/JuliaLang/llvm.git -b kf/tlsrebase5 --- Make.user | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Make.user b/Make.user index eb5fe551583a0..20b685c13a019 100644 --- a/Make.user +++ b/Make.user @@ -2,4 +2,7 @@ CC = clang CFLAGS = -ftls-model=global-dynamic CXX = clang++ LLVM_VER = svn -LLVM_GIT_URL_LLVM = https://github.com/JuliaLang/llvm.git -b jl/thread-local-storage +LLVM_GIT_URL_LLVM = https://github.com/JuliaLang/llvm.git -b kf/tlsrebase5 + +LLVM_DEBUG = 1 +LLVM_ASSERTIONS = 1 From 7aadf02e13800af74cb50d4a363cd8c8e4d0724f Mon Sep 17 00:00:00 2001 From: "Arch D. Robison" Date: Mon, 13 Apr 2015 14:54:40 -0500 Subject: [PATCH 0011/1938] Resolve merge of base/inference.jl --- base/inference.jl | 8 -------- 1 file changed, 8 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 1736d7cad60bb..f645054b88a75 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1563,11 +1563,7 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::Tuple, sparams::Tuple cenv[vname] = vtype s[1][vname] = VarState(vtype,false) end -<<<<<<< HEAD - vinflist = (ast.args[2][2])::Array{Any,1} -======= vinflist = ast.args[2][2]::Array{Any,1} ->>>>>>> c8c967c761f0463fad6b84ca36d18832cefa5092 for vi in vinflist vi::Array{Any,1} if (vi[3]&4)!=0 @@ -1579,16 +1575,12 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::Tuple, sparams::Tuple s[1][vname] = VarState(vtype,false) end end -<<<<<<< HEAD - sv = StaticVarInfo(sparams, cenv, vars, vinflist, label_counter(body)) -======= gensym_uses = find_gensym_uses(body) gensym_init = Any[ NF for i = 1:length(gensym_uses) ] gensym_types = copy(gensym_init) sv = StaticVarInfo(sparams, cenv, vars, gensym_types, vinflist, length(labels), ObjectIdDict()) ->>>>>>> c8c967c761f0463fad6b84ca36d18832cefa5092 frame.sv = sv recpts = IntSet() # statements that depend recursively on our value From fbfd8e98916503dd50d18341c13b9df6ac8e4627 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 13 Apr 2015 16:38:34 -0400 Subject: [PATCH 0012/1938] Some whitespace fixes --- src/gf.c | 2 +- src/julia.h | 4 ++-- test/perf/threads/laplace3d/laplace3d.c | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/gf.c b/src/gf.c index 50e6bb0298d84..01eb68bdff465 100644 --- a/src/gf.c +++ b/src/gf.c @@ -926,7 +926,7 @@ static jl_function_t *cache_method(jl_methtable_t *mt, jl_tuple_t *type, jl_type_infer(newmeth->linfo, type, method->linfo); } JL_GC_POP(); - JL_UNLOCK(codegen) + JL_UNLOCK(codegen) return newmeth; } diff --git a/src/julia.h b/src/julia.h index 9495a7da0296c..68547680cfbe9 100644 --- a/src/julia.h +++ b/src/julia.h @@ -79,7 +79,7 @@ DLLEXPORT void jl_threading_profile(); # define JL_ATOMIC_FETCH_AND_ADD(a,b) \ __sync_fetch_and_add(&(a), (b)) # define JL_ATOMIC_COMPARE_AND_SWAP(a,b,c) \ - __sync_bool_compare_and_swap(&(a), (b), (c)) + __sync_bool_compare_and_swap(&(a), (b), (c)) # define JL_ATOMIC_TEST_AND_SET(a) \ __sync_lock_test_and_set(&(a), 1) # define JL_ATOMIC_RELEASE(a) \ @@ -88,7 +88,7 @@ DLLEXPORT void jl_threading_profile(); # define JL_ATOMIC_FETCH_AND_ADD(a,b) \ _InterlockedExchangeAdd((volatile LONG *)&(a), (b)) # define JL_ATOMIC_COMPARE_AND_SWAP(a,b,c) \ - _InterlockedCompareExchange64(&(a), (c), (b)) + _InterlockedCompareExchange64(&(a), (c), (b)) # define JL_ATOMIC_TEST_AND_SET(a) \ _InterlockedExchange64(&(a), 1) # define JL_ATOMIC_RELEASE(a) \ diff --git a/test/perf/threads/laplace3d/laplace3d.c b/test/perf/threads/laplace3d/laplace3d.c index 76392a1c43542..7e69b47935f8b 100644 --- a/test/perf/threads/laplace3d/laplace3d.c +++ b/test/perf/threads/laplace3d/laplace3d.c @@ -145,12 +145,12 @@ int main(int argc, char **argv) for (i = 0; i < nx; ++i) { ind = i + j*nx + k*nx*ny; p_ind = i + j*padded_nx + k*padded_nx*ny; - + if (i == 0 || i == nx - 1 || j == 0 || j == ny - 1 || k == 0 || k == nz - 1) { // Dirichlet b.c.'s - u1[p_ind] = u1_orig[ind] = u2[p_ind] = 1.0f; + u1[p_ind] = u1_orig[ind] = u2[p_ind] = 1.0f; } else { u1[p_ind] = u1_orig[ind] = u2[p_ind] = 0.0f; @@ -298,13 +298,13 @@ void l3d_sse(int nx, int padded_nx, int ny, int nz, float *u1, float *u2) __m128 pSrc4 = _mm_load_ps(&u1[ind+padded_nx]); __m128 pSrc5 = _mm_load_ps(&u1[ind-padded_nx*ny]); __m128 pSrc6 = _mm_load_ps(&u1[ind+padded_nx*ny]); - + __m128 sum1 = _mm_add_ps(pSrc1, pSrc2); __m128 sum2 = _mm_add_ps(pSrc3, pSrc4); __m128 sum3 = _mm_add_ps(pSrc5, pSrc6); __m128 sum4 = _mm_add_ps(sum1, sum2); __m128 vsum = _mm_add_ps(sum3, sum4); - + vsum = _mm_mul_ps(vsum, sixth); _mm_stream_ps(&u2[ind], vsum); @@ -333,13 +333,13 @@ void l3d_avx(int nx, int padded_nx, int ny, int nz, float *u1, float *u2) __m256 pSrc4 = _mm256_load_ps(&u1[ind+padded_nx]); __m256 pSrc5 = _mm256_load_ps(&u1[ind-padded_nx*ny]); __m256 pSrc6 = _mm256_load_ps(&u1[ind+padded_nx*ny]); - + __m256 sum1 = _mm256_add_ps(pSrc1, pSrc2); __m256 sum2 = _mm256_add_ps(pSrc3, pSrc4); __m256 sum3 = _mm256_add_ps(pSrc5, pSrc6); __m256 sum4 = _mm256_add_ps(sum1, sum2); __m256 vsum = _mm256_add_ps(sum3, sum4); - + vsum = _mm256_mul_ps(vsum, sixth); _mm256_stream_ps(&u2[ind], vsum); From 41911d96d86397168ade1f39914a8d7ed1653558 Mon Sep 17 00:00:00 2001 From: Mauro Werder Date: Tue, 14 Jul 2015 21:02:00 +0100 Subject: [PATCH 0013/1938] Big overhaul of "Scope" manual section [ci skip] --- doc/manual/functions.rst | 1 + doc/manual/modules.rst | 14 +- doc/manual/noteworthy-differences.rst | 2 + doc/manual/variables-and-scoping.rst | 477 ++++++++++++++++---------- 4 files changed, 309 insertions(+), 185 deletions(-) diff --git a/doc/manual/functions.rst b/doc/manual/functions.rst index 8c7166edde293..b5bfe588cf244 100644 --- a/doc/manual/functions.rst +++ b/doc/manual/functions.rst @@ -495,6 +495,7 @@ tuple, explicitly after a semicolon. For example, ``plot(x, y; ``plot(x, y, width=2)``. This is useful in situations where the keyword name is computed at runtime. +.. _man-evaluation-scope-default-values: Evaluation Scope of Default Values ---------------------------------- diff --git a/doc/manual/modules.rst b/doc/manual/modules.rst index 09e2acb909653..3c3f2581d6ffc 100644 --- a/doc/manual/modules.rst +++ b/doc/manual/modules.rst @@ -6,12 +6,14 @@ .. index:: module, baremodule, using, import, export, importall -Modules in Julia are separate global variable workspaces. They are -delimited syntactically, inside ``module Name ... end``. Modules allow -you to create top-level definitions without worrying about name conflicts -when your code is used together with somebody else's. Within a module, you -can control which names from other modules are visible (via importing), -and specify which of your names are intended to be public (via exporting). +Modules in Julia are separate variable workspaces, i.e. they introduce +a new global scope. They are delimited syntactically, inside ``module +Name ... end``. Modules allow you to create top-level definitions (aka +global variables) without worrying about name conflicts when your code +is used together with somebody else's. Within a module, you can +control which names from other modules are visible (via importing), +and specify which of your names are intended to be public (via +exporting). The following example demonstrates the major features of modules. It is not meant to be run, but is shown for illustrative purposes:: diff --git a/doc/manual/noteworthy-differences.rst b/doc/manual/noteworthy-differences.rst index 0926b57c35ec6..730cc34acd98e 100644 --- a/doc/manual/noteworthy-differences.rst +++ b/doc/manual/noteworthy-differences.rst @@ -91,6 +91,8 @@ some noteworthy differences that may trip up Julia users accustomed to MATLAB: ``ans`` is not set when Julia code is run in non-interactive mode. - Julia's ``type``\ s do not support dynamically adding fields at runtime, unlike MATLAB's ``class``\ es. Instead, use a :obj:`Dict`. +- In Julia each module has its own global scope/namespace, whereas in + Matlab there is just one global scope. Noteworthy differences from R ----------------------------- diff --git a/doc/manual/variables-and-scoping.rst b/doc/manual/variables-and-scoping.rst index 8310b631f458a..4a408115ddf5e 100644 --- a/doc/manual/variables-and-scoping.rst +++ b/doc/manual/variables-and-scoping.rst @@ -6,234 +6,347 @@ Scope of Variables ******************** -The *scope* of a variable is the region of code within which a variable -is visible. Variable scoping helps avoid variable naming conflicts. The -concept is intuitive: two functions can both have arguments called ``x`` -without the two ``x``'s referring to the same thing. Similarly there are -many other cases where different blocks of code can use the same name -without referring to the same thing. The rules for when the same -variable name does or doesn't refer to the same thing are called scope -rules; this section spells them out in detail. +The *scope* of a variable is the region of code within which a +variable is visible. Variable scoping helps avoid variable naming +conflicts. The concept is intuitive: two functions can both have +arguments called ``x`` without the two ``x``'s referring to the same +thing. Similarly there are many other cases where different blocks of +code can use the same name without referring to the same thing. The +rules for when the same variable name does or doesn't refer to the +same thing are called scope rules; this section spells them out in +detail. Certain constructs in the language introduce *scope blocks*, which are regions of code that are eligible to be the scope of some set of -variables. The scope of a variable cannot be an arbitrary set of source -lines; instead, it will always line up with one of these blocks. -The constructs introducing such blocks are: - -- ``function`` bodies (:ref:`either syntax `) -- ``while`` loops -- ``for`` loops -- ``try`` blocks -- ``catch`` blocks -- ``finally`` blocks -- ``let`` blocks -- ``type`` blocks. - -Notably missing from this list are -:ref:`begin blocks ` and :ref:`if blocks `, which do -*not* introduce new scope blocks. - -Certain constructs introduce new variables into the current innermost -scope. When a variable is introduced into a scope, it is also inherited -by all inner scopes unless one of those inner scopes explicitly -overrides it. +variables. The scope of a variable cannot be an arbitrary set of +source lines; instead, it will always line up with one of these +blocks. There are two main types of scopes in Julia, *global scope* +and *local scope*, the latter can be nested. The constructs +introducing scope blocks are: + ++--------------------------------+----------------------------------------------------------------------------------+ +| Scope name | block/construct introducing this kind of scope | ++================================+==================================================================================+ +| :ref:`global ` | | module, baremodule, at interactive prompt (REPL) | ++--------------------------------+------------------------------+---------------------------------------------------+ +| :ref:`local ` | :ref:`soft ` | | for, while, list-comprehensions, | +| | | try-catch-finally, let | +| +------------------------------+---------------------------------------------------+ +| | :ref:`hard `| | functions (either syntax, anonymous & do-blocks)| +| | | | type, immutable, macro | ++--------------------------------+------------------------------+---------------------------------------------------+ + +Notably missing from this table are :ref:`begin blocks +` and :ref:`if blocks +`, which do *not* introduce new scope +blocks. All three types of scopes follow somewhat different rules +which will be explained below as well as some extra rules for +certain blocks. Julia uses `lexical scoping `_, meaning that a function's scope does not inherit from its caller's scope, but from the scope in which the function was defined. -For example, in the following code the ``x`` inside ``foo`` is found -in the global scope (and if no global variable ``x`` existed, an -undefined variable error would be raised):: +For example, in the following code the ``x`` inside ``foo`` is refers +to the ``x`` in the global scope of its module ``Bar``:: - function foo() - x + module Bar + x = 1 + foo() = x end - function bar() - x = 1 - foo() - end +and not a ``x`` in the scope where ``foo`` is used:: - x = 2 + julia> import Bar - julia> bar() - 2 + julia> x = -1; -If ``foo`` is instead defined inside ``bar``, then it accesses -the local ``x`` present in that function:: + julia> Bar.foo() + 1 - function bar() - function foo() - x - end - x = 1 - foo() +Thus *lexical scope* means that the scope of variables can be inferred +from the source code alone. + +.. _man-global: + +Global Scope +------------ + +*Each module introduces a new global scope*, separate from the global +scope of all other modules; there is no all-encompassing global scope. +Modules can introduce variables of other modules into their scope +through the :ref:`using or import ` statements or through +qualified access using the dot-notation, i.e. each module is a +so-called *namespace*. Note that variable bindings can only be +changed within their global scope and not from an outside module. :: + + module A + a = 1 # a global in A's scope end - x = 2 + module B + # b = a # would error as B's global scope is separate from A's + module C + c = 2 + end + b = C.c # can access the namespace of a nested global scope + # through a qualified access + import A # makes module A available + d = A.a + # A.a = 2 # would error with: "ERROR: type Module has no field a" + end - julia> bar() - 1 +Note that the interactive prompt (aka REPL) is in the global scope of +the module ``Main``. -The constructs that introduce new variables into the current scope -are as follows: - -- A declaration ``local x`` or ``const x`` introduces a new local variable. -- A declaration ``global x`` makes ``x`` in the current scope and inner - scopes refer to the global variable of that name. -- A function's arguments are introduced as new local variables into the - function's body scope. -- An assignment ``x = y`` introduces a new local variable ``x`` only if - ``x`` is neither declared global nor introduced as local - by any enclosing scope before *or after* the current line of code. - -In the following example, there is only one ``x`` assigned both inside -and outside the ``for`` loop:: - - function foo(n) - x = 0 - for i = 1:n - x = x + 1 - end - x +.. _man-local-scope: + +Local Scope +----------- + +A local scope *usually* inherits all the variables from its parent +scope, both for reading and writing. There are two subtypes of local +scopes, hard and soft, with slightly different rules concerning what +variables are inherited. Unlike global scopes, local scopes are not +namespaces, thus variables in an inner scope cannot be retrieved from +the parent scope through some sort of qualified access. + +The following rules and examples pertain to both hard and soft local +scopes. A newly introduced variable in a local scope does not +back-propagate to its parent scope. For example, here the ``z`` is not +introduced into the top-level scope:: + + for i=1:10 + z = i end - julia> foo(10) - 10 + julia> z + ERROR: UndefVarError: z not defined + +(Note, in this and all following examples it is assumed that their +top-level is a global scope with a clean workspace, for instance a +newly started REPL.) -In the next example, the loop has a separate ``x`` and the function -always returns zero:: +Inside a local scope a variable can be forced to be a local variable +using the ``local`` keyword:: - function foo(n) - x = 0 - for i = 1:n + x = 0 + for i=1:10 local x - x = i - end - x + x = i + 1 end - julia> foo(10) + julia> x 0 -In this example, an ``x`` exists only inside the loop, and the function -encounters an undefined variable error on its last line (unless there is -a global variable ``x``):: +Inside a local scope a new global variable can be defined using the +keyword ``global``:: - function foo(n) - for i = 1:n - x = i - end - x + for i=1:10 + global z + z = i end - julia> foo(10) - in foo: x not defined - -A variable that is not assigned to or otherwise introduced locally -defaults to global, so this function would return the value of the -global ``x`` if there were such a variable, or produce an error if no such -global existed. As a consequence, the only way to assign to a global -variable inside a non-top-level scope is to explicitly declare the -variable as global within some scope, since otherwise the assignment -would introduce a new local rather than assigning to the global. This -rule works out well in practice, since the vast majority of variables -assigned inside functions are intended to be local variables, and using -global variables should be the exception rather than the rule, -and assigning new values to them even more so. - -One last example shows that an outer assignment introducing ``x`` need -not come before an inner usage:: - - function foo(n) - f = y -> n + x + y - x = 1 - f(2) + julia> z + 10 + +.. + However, there is no keyword to introduce a new local variable into a + parent local scope. + +The location of both the ``local`` and ``global`` keywords within the +scope block is irrelevant. The following is equivalent to the last +example (although stylistically worse):: + + for i=1:10 + z = i + global z end - julia> foo(10) - 13 + julia> z + 10 -This behavior may seem slightly odd for a normal variable, but allows -for named functions — which are just normal variables holding function -objects — to be used before they are defined. This allows functions to -be defined in whatever order is intuitive and convenient, rather than -forcing bottom up ordering or requiring forward declarations, both of -which one typically sees in C programs. As an example, here is an -inefficient, mutually recursive way to test if positive integers are -even or odd:: +Multiple global or local definitions can be on one line and can also +be paired with assignments:: - even(n) = n == 0 ? true : odd(n-1) - odd(n) = n == 0 ? false : even(n-1) + for i=1:10 + global x=i, y, z + local a=4, b , c=1 + end - julia> even(3) - false - julia> odd(3) - true +.. _man-soft-scope: -Julia provides built-in, efficient functions to test for oddness and evenness -called :func:`iseven` and :func:`isodd` so the above definitions should only be -taken as examples. +Soft Local Scope +^^^^^^^^^^^^^^^^ -Since functions can be used before they are defined, as long as they are -defined by the time they are actually called, no syntax for forward -declarations is necessary, and definitions can be ordered arbitrarily. + In a soft local scope, all variables are inherited from its parent + scope unless a variable is specifically marked with the keyword + ``local``. -At the interactive prompt, variable scope works the same way as anywhere -else. The prompt behaves as if there is scope block wrapped around -everything you type, except that this scope block is identified with the -global scope. This is especially evident in the case of assignments: +Soft local scopes are introduced by for-loops, while-loops, +list-comprehensions, try-catch-finally-blocks, and let-blocks. There +are some extra rules for :ref:`let-blocks ` and for +:ref:`for-loops and list-comprehensions `. -.. doctest:: +In the following example the ``x`` and ``y`` refer always to the same +variables as the soft local scope inherits both read and write +variables:: - julia> for i = 1:1; y = 10; end + x,y = 0, 1 + for i = 1:10 + x = i + y + 1 + end - julia> y - ERROR: UndefVarError: y not defined + julia> x + 11 + +Within soft scopes, the `global` keyword is never necessary, although +allowed. The only case when it would change the semantics is +(currently) a syntax error:: + + x = 1 + let + local x = 2 + let + global x = 3 + end + end - julia> y = 0 - 0 + # ERROR: syntax: `global #1#x`: #1#x is local variable in the enclosing scope - julia> for i = 1:1; y = 10; end +.. _man-hard-scope: - julia> y - 10 +Hard Local Scope +^^^^^^^^^^^^^^^^ -In the former case, ``y`` only exists inside of the ``for`` loop. In the -latter case, an outer ``y`` has been introduced and so is inherited -within the loop. Due to the special identification of the prompt's scope -block with the global scope, it is not necessary to declare ``global y`` -inside the loop. However, in code not entered into the interactive -prompt this declaration would be necessary in order to modify a global -variable. +Hard local scopes are introduced by function definitions (in all their +forms), type & immutable-blocks and macro-definitions. -Multiple variables can be declared global using the following syntax:: + In a hard local scope, all variables are inherited from its parent + scope unless: + - an assignment would result in a modified *global* variable, or + - a variable is specifically marked with the keyword ``local``. + +Thus global variables are only inherited for reading but not for +writing:: + + x,y = 1,2 function foo() - global x=1, y="bar", z=3 + x = 2 # assignment introduces a new local + return x + y # y refers to the global end julia> foo() - 3 + 4 julia> x 1 - julia> y - "bar" +An explicit ``global`` is needed to assign to a global variable:: - julia> z - 3 + x = 1 + function foo() + global x = 2 + end + foo() + + julia> x + 2 + +However, nested functions can modify the parent's *local* variables:: + + x,y = 1,2 + function foo() + x = 2 # introduces a new local + function bar() + x = 10 # modifies the parent's x + return x+y # y is global + end + return bar() + x # 12 + 10 (x is modified in call of bar()) + end + + julia> foo() + 22 # x,y unchanged + +The distinction between inheriting global and local variables for +assignment can lead to some slight differences between functions +defined in local vs. global scopes. Consider the modification of the +last example by moving ``bar`` to the global scope:: + + x,y = 1,2 + function bar() + x = 10 # local + return x+y + end + function foo() + x = 2 # local + return bar() + x # 12 + 2 (x is not modified) + end + + julia> foo() + 14 # as x is not modified anymore + # x,y unchanged + +Note that above subtlety does not pertain to type and macro +definitions as they can only appear at the global scope. +There are special scoping rules concerning the evaluation of default +and keyword function arguments which are described in the +:ref:`Function section `. + + +An assignment introducing a variable used inside a function, type or +macro definition need not come before its inner usage: + +.. doctest:: + + julia> f = y -> x + y + (anonymous function) + + julia> f(3) + ERROR: UndefVarError: x not defined + in anonymous at none:1 + + julia> x = 1 + 1 + + julia> f(3) + 4 + +This behavior may seem slightly odd for a normal variable, but allows +for named functions — which are just normal variables holding function +objects — to be used before they are defined. This allows functions to +be defined in whatever order is intuitive and convenient, rather than +forcing bottom up ordering or requiring forward declarations, as long +as they are defined by the time they are actually called. As an +example, here is an inefficient, mutually recursive way to test if +positive integers are even or odd:: + + even(n) = n == 0 ? true : odd(n-1) + odd(n) = n == 0 ? false : even(n-1) + + julia> even(3) + false + + julia> odd(3) + true + +Julia provides built-in, efficient functions to test for oddness and evenness +called :func:`iseven` and :func:`isodd` so the above definitions should only be +taken as examples. + +.. _man-let-blocks: + +Let Blocks +^^^^^^^^^^ -The ``let`` statement provides a different way to introduce variables. Unlike assignments to local variables, ``let`` statements allocate new -variable bindings each time they run. An assignment modifies an existing -value location, and ``let`` creates new locations. This difference is -usually not important, and is only detectable in the case of variables -that outlive their scope via closures. The ``let`` syntax accepts a -comma-separated series of assignments and variable names:: +variable bindings each time they run. An assignment modifies an +existing value location, and ``let`` creates new locations. This +difference is usually not important, and is only detectable in the +case of variables that outlive their scope via closures. The ``let`` +syntax accepts a comma-separated series of assignments and variable +names:: let var1 = value1, var2, var3 = value3 code @@ -245,7 +358,7 @@ has been introduced. Therefore it makes sense to write something like ``let x = x`` since the two ``x`` variables are distinct and have separate storage. Here is an example where the behavior of ``let`` is needed:: - Fs = cell(2) + Fs = Array(Any,2) i = 1 while i <= 2 Fs[i] = ()->i @@ -263,7 +376,7 @@ However, it is always the same variable ``i``, so the two closures behave identically. We can use ``let`` to create a new binding for ``i``:: - Fs = cell(2) + Fs = Array(Any,2) i = 1 while i <= 2 let i = i @@ -296,15 +409,21 @@ block without creating any new bindings: Since ``let`` introduces a new scope block, the inner local ``x`` is a different variable than the outer local ``x``. + +.. _man-for-loops-scope: + For Loops and Comprehensions ----------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + -``for`` loops and :ref:`comprehensions ` have a special -additional behavior: any new variables introduced in their body scopes are -freshly allocated for each loop iteration. Therefore these constructs are -similar to ``while`` loops with ``let`` blocks inside:: +``for`` loops and :ref:`comprehensions ` have the +following behavior: any new variables introduced in their body scopes +are freshly allocated for each loop iteration. This is in contrast to +``while`` loops which reuse the variables for all +iterations. Therefore these constructs are similar to ``while`` loops +with ``let`` blocks inside:: - Fs = cell(2) + Fs = Array(Any,2) for i = 1:2 Fs[i] = ()->i end @@ -315,7 +434,7 @@ similar to ``while`` loops with ``let`` blocks inside:: julia> Fs[2]() 2 -``for`` loops will reuse existing variables for iteration:: +``for`` loops will reuse existing variables for its iteration variable:: i = 0 for i = 1:3 @@ -327,7 +446,7 @@ iteration variables:: x = 0 [ x for x=1:3 ] - x # here still equal to 0 + X # here still equal to 0 Constants --------- From abdfc24ac1a7acd67b474a1848758589d4613065 Mon Sep 17 00:00:00 2001 From: Mauro Werder Date: Thu, 16 Jul 2015 14:55:25 +0200 Subject: [PATCH 0014/1938] Some updates after a mailing-list conversation [ci skip] --- doc/manual/variables-and-scoping.rst | 65 ++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 13 deletions(-) diff --git a/doc/manual/variables-and-scoping.rst b/doc/manual/variables-and-scoping.rst index 4a408115ddf5e..802d706b1bf54 100644 --- a/doc/manual/variables-and-scoping.rst +++ b/doc/manual/variables-and-scoping.rst @@ -24,6 +24,8 @@ blocks. There are two main types of scopes in Julia, *global scope* and *local scope*, the latter can be nested. The constructs introducing scope blocks are: +.. _man-scope-table: + +--------------------------------+----------------------------------------------------------------------------------+ | Scope name | block/construct introducing this kind of scope | +================================+==================================================================================+ @@ -32,7 +34,7 @@ introducing scope blocks are: | :ref:`local ` | :ref:`soft ` | | for, while, list-comprehensions, | | | | try-catch-finally, let | | +------------------------------+---------------------------------------------------+ -| | :ref:`hard `| | functions (either syntax, anonymous & do-blocks)| +| | :ref:`hard ` | | functions (either syntax, anonymous & do-blocks)| | | | | type, immutable, macro | +--------------------------------+------------------------------+---------------------------------------------------+ @@ -103,12 +105,14 @@ the module ``Main``. Local Scope ----------- -A local scope *usually* inherits all the variables from its parent -scope, both for reading and writing. There are two subtypes of local -scopes, hard and soft, with slightly different rules concerning what -variables are inherited. Unlike global scopes, local scopes are not -namespaces, thus variables in an inner scope cannot be retrieved from -the parent scope through some sort of qualified access. +A new local scope is introduced by most code-blocks, see above +:ref:`table ` for a complete list. A local scope +*usually* inherits all the variables from its parent scope, both for +reading and writing. There are two subtypes of local scopes, hard and +soft, with slightly different rules concerning what variables are +inherited. Unlike global scopes, local scopes are not namespaces, +thus variables in an inner scope cannot be retrieved from the parent +scope through some sort of qualified access. The following rules and examples pertain to both hard and soft local scopes. A newly introduced variable in a local scope does not @@ -204,7 +208,6 @@ Within soft scopes, the `global` keyword is never necessary, although allowed. The only case when it would change the semantics is (currently) a syntax error:: - x = 1 let local x = 2 let @@ -212,7 +215,7 @@ allowed. The only case when it would change the semantics is end end - # ERROR: syntax: `global #1#x`: #1#x is local variable in the enclosing scope + # ERROR: syntax: `global x`: x is local variable in the enclosing scope .. _man-hard-scope: @@ -254,7 +257,9 @@ An explicit ``global`` is needed to assign to a global variable:: julia> x 2 -However, nested functions can modify the parent's *local* variables:: +Note that *nested functions* can behave differently to functions +defined in the global scope as they can modify their parent scope's +*local* variables:: x,y = 1,2 function foo() @@ -267,7 +272,7 @@ However, nested functions can modify the parent's *local* variables:: end julia> foo() - 22 # x,y unchanged + 22 # (x,y unchanged) The distinction between inheriting global and local variables for assignment can lead to some slight differences between functions @@ -285,8 +290,8 @@ last example by moving ``bar`` to the global scope:: end julia> foo() - 14 # as x is not modified anymore - # x,y unchanged + 14 # as x is not modified anymore. + # (x,y unchanged) Note that above subtlety does not pertain to type and macro definitions as they can only appear at the global scope. @@ -335,6 +340,40 @@ Julia provides built-in, efficient functions to test for oddness and evenness called :func:`iseven` and :func:`isodd` so the above definitions should only be taken as examples. +Hard vs. Soft Local Scope +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Blocks which introduce a soft local scope, such as loops, are +generally used to manipulate the variables in their parent scope. +Thus their default is to fully access all variables in their parent +scope. + +Conversely, the code inside blocks which introduce a hard local scope +(function, type and macro definitions) can be executed at any place in +a program. Remotely changing the state of global variables in other +modules should be done with care and thus this is an opt-in feature +requiring the ``global`` keyword. + +The reason to allow *modifying local* variables of parent scopes in +nested functions is to allow constructing `closures +`_ +which have a private state, for instance the ``state`` variable in the +following example:: + + let + state = 0 + global counter + counter() = state += 1 + end + + julia> counter() + 1 + + julia> counter() + 2 + +See also the closures in the examples in the next two sections. + .. _man-let-blocks: Let Blocks From 02660d20bcd9edb345d38f9516616fd3e81bf255 Mon Sep 17 00:00:00 2001 From: matthieugomez Date: Sun, 2 Aug 2015 14:35:04 -0400 Subject: [PATCH 0015/1938] Move discussion about abstract types in fields from FAQ to performances [av skip] --- doc/manual/faq.rst | 300 ------------------------------- doc/manual/performance-tips.rst | 310 ++++++++++++++++++++++++++++++-- 2 files changed, 300 insertions(+), 310 deletions(-) diff --git a/doc/manual/faq.rst b/doc/manual/faq.rst index f98a04ae76d29..7db3f6e1982e0 100644 --- a/doc/manual/faq.rst +++ b/doc/manual/faq.rst @@ -447,306 +447,6 @@ in the future, we could consider defaulting to checked integer arithmetic in Julia, but for now, we have to live with the possibility of overflow. -.. _man-abstract-fields: - -How do "abstract" or ambiguous fields in types interact with the compiler? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Types can be declared without specifying the types of their fields: - -.. doctest:: - - julia> type MyAmbiguousType - a - end - -This allows ``a`` to be of any type. This can often be useful, but it -does have a downside: for objects of type ``MyAmbiguousType``, the -compiler will not be able to generate high-performance code. The -reason is that the compiler uses the types of objects, not their -values, to determine how to build code. Unfortunately, very little can -be inferred about an object of type ``MyAmbiguousType``: - -.. doctest:: - - julia> b = MyAmbiguousType("Hello") - MyAmbiguousType("Hello") - - julia> c = MyAmbiguousType(17) - MyAmbiguousType(17) - - julia> typeof(b) - MyAmbiguousType - - julia> typeof(c) - MyAmbiguousType - -``b`` and ``c`` have the same type, yet their underlying -representation of data in memory is very different. Even if you stored -just numeric values in field ``a``, the fact that the memory -representation of a ``UInt8`` differs from a ``Float64`` also means -that the CPU needs to handle them using two different kinds of -instructions. Since the required information is not available in the -type, such decisions have to be made at run-time. This slows -performance. - -You can do better by declaring the type of ``a``. Here, we are focused -on the case where ``a`` might be any one of several types, in which -case the natural solution is to use parameters. For example: - -.. doctest:: - - julia> type MyType{T<:AbstractFloat} - a::T - end - -This is a better choice than - -.. doctest:: - - julia> type MyStillAmbiguousType - a::AbstractFloat - end - -because the first version specifies the type of ``a`` from the type of -the wrapper object. For example: - -.. doctest:: - - julia> m = MyType(3.2) - MyType{Float64}(3.2) - - julia> t = MyStillAmbiguousType(3.2) - MyStillAmbiguousType(3.2) - - julia> typeof(m) - MyType{Float64} - - julia> typeof(t) - MyStillAmbiguousType - -The type of field ``a`` can be readily determined from the type of -``m``, but not from the type of ``t``. Indeed, in ``t`` it's possible -to change the type of field ``a``: - -.. doctest:: - - julia> typeof(t.a) - Float64 - - julia> t.a = 4.5f0 - 4.5f0 - - julia> typeof(t.a) - Float32 - -In contrast, once ``m`` is constructed, the type of ``m.a`` cannot -change: - -.. doctest:: - - julia> m.a = 4.5f0 - 4.5 - - julia> typeof(m.a) - Float64 - -The fact that the type of ``m.a`` is known from ``m``'s type---coupled -with the fact that its type cannot change mid-function---allows the -compiler to generate highly-optimized code for objects like ``m`` but -not for objects like ``t``. - -Of course, all of this is true only if we construct ``m`` with a -concrete type. We can break this by explicitly constructing it with -an abstract type: - -.. doctest:: - - julia> m = MyType{AbstractFloat}(3.2) - MyType{AbstractFloat}(3.2) - - julia> typeof(m.a) - Float64 - - julia> m.a = 4.5f0 - 4.5f0 - - julia> typeof(m.a) - Float32 - -For all practical purposes, such objects behave identically to those -of ``MyStillAmbiguousType``. - -It's quite instructive to compare the sheer amount code generated for -a simple function -:: - - func(m::MyType) = m.a+1 - -using -:: - - code_llvm(func,(MyType{Float64},)) - code_llvm(func,(MyType{AbstractFloat},)) - code_llvm(func,(MyType,)) - -For reasons of length the results are not shown here, but you may wish -to try this yourself. Because the type is fully-specified in the first -case, the compiler doesn't need to generate any code to resolve the -type at run-time. This results in shorter and faster code. - - -.. _man-abstract-container-type: - -How should I declare "abstract container type" fields? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The same best practices that apply in the `previous section -<#man-abstract-fields>`_ also work for container types: - -.. doctest:: - - julia> type MySimpleContainer{A<:AbstractVector} - a::A - end - - julia> type MyAmbiguousContainer{T} - a::AbstractVector{T} - end - -For example: - -.. doctest:: - - julia> c = MySimpleContainer(1:3); - - julia> typeof(c) - MySimpleContainer{UnitRange{Int64}} - - julia> c = MySimpleContainer([1:3;]); - - julia> typeof(c) - MySimpleContainer{Array{Int64,1}} - - julia> b = MyAmbiguousContainer(1:3); - - julia> typeof(b) - MyAmbiguousContainer{Int64} - - julia> b = MyAmbiguousContainer([1:3;]); - - julia> typeof(b) - MyAmbiguousContainer{Int64} - -For ``MySimpleContainer``, the object is fully-specified by its type -and parameters, so the compiler can generate optimized functions. In -most instances, this will probably suffice. - -While the compiler can now do its job perfectly well, there are cases -where *you* might wish that your code could do different things -depending on the *element type* of ``a``. Usually the best way to -achieve this is to wrap your specific operation (here, ``foo``) in a -separate function:: - - function sumfoo(c::MySimpleContainer) - s = 0 - for x in c.a - s += foo(x) - end - s - end - - foo(x::Integer) = x - foo(x::AbstractFloat) = round(x) - -This keeps things simple, while allowing the compiler to generate -optimized code in all cases. - -However, there are cases where you may need to declare different -versions of the outer function for different element types of -``a``. You could do it like this:: - - function myfun{T<:AbstractFloat}(c::MySimpleContainer{Vector{T}}) - ... - end - function myfun{T<:Integer}(c::MySimpleContainer{Vector{T}}) - ... - end - -This works fine for ``Vector{T}``, but we'd also have to write -explicit versions for ``UnitRange{T}`` or other abstract types. To -prevent such tedium, you can use two parameters in the declaration of -``MyContainer``:: - - type MyContainer{T, A<:AbstractVector} - a::A - end - MyContainer(v::AbstractVector) = MyContainer{eltype(v), typeof(v)}(v) - - julia> b = MyContainer(1.3:5); - - julia> typeof(b) - MyContainer{Float64,UnitRange{Float64}} - -Note the somewhat surprising fact that ``T`` doesn't appear in the -declaration of field ``a``, a point that we'll return to in a moment. -With this approach, one can write functions such as:: - - function myfunc{T<:Integer, A<:AbstractArray}(c::MyContainer{T,A}) - return c.a[1]+1 - end - # Note: because we can only define MyContainer for - # A<:AbstractArray, and any unspecified parameters are arbitrary, - # the previous could have been written more succinctly as - # function myfunc{T<:Integer}(c::MyContainer{T}) - - function myfunc{T<:AbstractFloat}(c::MyContainer{T}) - return c.a[1]+2 - end - - function myfunc{T<:Integer}(c::MyContainer{T,Vector{T}}) - return c.a[1]+3 - end - - julia> myfunc(MyContainer(1:3)) - 2 - - julia> myfunc(MyContainer(1.0:3)) - 3.0 - - julia> myfunc(MyContainer([1:3])) - 4 - -As you can see, with this approach it's possible to specialize on both -the element type ``T`` and the array type ``A``. - -However, there's one remaining hole: we haven't enforced that ``A`` -has element type ``T``, so it's perfectly possible to construct an -object like this:: - - julia> b = MyContainer{Int64, UnitRange{Float64}}(1.3:5); - - julia> typeof(b) - MyContainer{Int64,UnitRange{Float64}} - -To prevent this, we can add an inner constructor:: - - type MyBetterContainer{T<:Real, A<:AbstractVector} - a::A - - MyBetterContainer(v::AbstractVector{T}) = new(v) - end - MyBetterContainer(v::AbstractVector) = MyBetterContainer{eltype(v),typeof(v)}(v) - - - julia> b = MyBetterContainer(1.3:5); - - julia> typeof(b) - MyBetterContainer{Float64,UnitRange{Float64}} - - julia> b = MyBetterContainer{Int64, UnitRange{Float64}}(1.3:5); - ERROR: no method MyBetterContainer(UnitRange{Float64},) - -The inner constructor requires that the element type of ``A`` be ``T``. -- _man-packages: diff --git a/doc/manual/performance-tips.rst b/doc/manual/performance-tips.rst index df033f3182fc8..8ca9d704c7add 100644 --- a/doc/manual/performance-tips.rst +++ b/doc/manual/performance-tips.rst @@ -166,6 +166,10 @@ that can be manipulated efficiently. See also the discussion under :ref:`man-parametric-types`. + + + + Type declarations ----------------- @@ -176,25 +180,311 @@ arguments, local variables, and expressions. However, there are a few specific instances where declarations are helpful. -Declare specific types for fields of composite types -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _man-abstract-fields: +Avoid fields with abstract type +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Types can be declared without specifying the types of their fields: + +.. doctest:: + + julia> type MyAmbiguousType + a + end + +This allows ``a`` to be of any type. This can often be useful, but it +does have a downside: for objects of type ``MyAmbiguousType``, the +compiler will not be able to generate high-performance code. The +reason is that the compiler uses the types of objects, not their +values, to determine how to build code. Unfortunately, very little can +be inferred about an object of type ``MyAmbiguousType``: + +.. doctest:: + + julia> b = MyAmbiguousType("Hello") + MyAmbiguousType("Hello") + + julia> c = MyAmbiguousType(17) + MyAmbiguousType(17) + + julia> typeof(b) + MyAmbiguousType + + julia> typeof(c) + MyAmbiguousType + +``b`` and ``c`` have the same type, yet their underlying +representation of data in memory is very different. Even if you stored +just numeric values in field ``a``, the fact that the memory +representation of a ``UInt8`` differs from a ``Float64`` also means +that the CPU needs to handle them using two different kinds of +instructions. Since the required information is not available in the +type, such decisions have to be made at run-time. This slows +performance. + +You can do better by declaring the type of ``a``. Here, we are focused +on the case where ``a`` might be any one of several types, in which +case the natural solution is to use parameters. For example: + +.. doctest:: + + julia> type MyType{T<:AbstractFloat} + a::T + end + +This is a better choice than + +.. doctest:: + + julia> type MyStillAmbiguousType + a::AbstractFloat + end + +because the first version specifies the type of ``a`` from the type of +the wrapper object. For example: + +.. doctest:: + + julia> m = MyType(3.2) + MyType{Float64}(3.2) + + julia> t = MyStillAmbiguousType(3.2) + MyStillAmbiguousType(3.2) + + julia> typeof(m) + MyType{Float64} + + julia> typeof(t) + MyStillAmbiguousType + +The type of field ``a`` can be readily determined from the type of +``m``, but not from the type of ``t``. Indeed, in ``t`` it's possible +to change the type of field ``a``: + +.. doctest:: + + julia> typeof(t.a) + Float64 + + julia> t.a = 4.5f0 + 4.5f0 + + julia> typeof(t.a) + Float32 + +In contrast, once ``m`` is constructed, the type of ``m.a`` cannot +change: + +.. doctest:: + + julia> m.a = 4.5f0 + 4.5 + + julia> typeof(m.a) + Float64 + +The fact that the type of ``m.a`` is known from ``m``'s type---coupled +with the fact that its type cannot change mid-function---allows the +compiler to generate highly-optimized code for objects like ``m`` but +not for objects like ``t``. + +Of course, all of this is true only if we construct ``m`` with a +concrete type. We can break this by explicitly constructing it with +an abstract type: + +.. doctest:: + + julia> m = MyType{AbstractFloat}(3.2) + MyType{AbstractFloat}(3.2) + + julia> typeof(m.a) + Float64 + + julia> m.a = 4.5f0 + 4.5f0 + + julia> typeof(m.a) + Float32 + +For all practical purposes, such objects behave identically to those +of ``MyStillAmbiguousType``. + +It's quite instructive to compare the sheer amount code generated for +a simple function +:: + + func(m::MyType) = m.a+1 + +using +:: + + code_llvm(func,(MyType{Float64},)) + code_llvm(func,(MyType{AbstractFloat},)) + code_llvm(func,(MyType,)) -Given a user-defined type like the following:: +For reasons of length the results are not shown here, but you may wish +to try this yourself. Because the type is fully-specified in the first +case, the compiler doesn't need to generate any code to resolve the +type at run-time. This results in shorter and faster code. - type Foo - field + +.. _man-abstract-container-type: + +Avoid fields with abstract containers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The same best practices also work for container types: + +.. doctest:: + + julia> type MySimpleContainer{A<:AbstractVector} + a::A + end + + julia> type MyAmbiguousContainer{T} + a::AbstractVector{T} + end + +For example: + +.. doctest:: + + julia> c = MySimpleContainer(1:3); + + julia> typeof(c) + MySimpleContainer{UnitRange{Int64}} + + julia> c = MySimpleContainer([1:3;]); + + julia> typeof(c) + MySimpleContainer{Array{Int64,1}} + + julia> b = MyAmbiguousContainer(1:3); + + julia> typeof(b) + MyAmbiguousContainer{Int64} + + julia> b = MyAmbiguousContainer([1:3;]); + + julia> typeof(b) + MyAmbiguousContainer{Int64} + +For ``MySimpleContainer``, the object is fully-specified by its type +and parameters, so the compiler can generate optimized functions. In +most instances, this will probably suffice. + +While the compiler can now do its job perfectly well, there are cases +where *you* might wish that your code could do different things +depending on the *element type* of ``a``. Usually the best way to +achieve this is to wrap your specific operation (here, ``foo``) in a +separate function:: + + function sumfoo(c::MySimpleContainer) + s = 0 + for x in c.a + s += foo(x) + end + s + end + + foo(x::Integer) = x + foo(x::AbstractFloat) = round(x) + +This keeps things simple, while allowing the compiler to generate +optimized code in all cases. + +However, there are cases where you may need to declare different +versions of the outer function for different element types of +``a``. You could do it like this:: + + function myfun{T<:AbstractFloat}(c::MySimpleContainer{Vector{T}}) + ... + end + function myfun{T<:Integer}(c::MySimpleContainer{Vector{T}}) + ... + end + +This works fine for ``Vector{T}``, but we'd also have to write +explicit versions for ``UnitRange{T}`` or other abstract types. To +prevent such tedium, you can use two parameters in the declaration of +``MyContainer``:: + + type MyContainer{T, A<:AbstractVector} + a::A + end + MyContainer(v::AbstractVector) = MyContainer{eltype(v), typeof(v)}(v) + + julia> b = MyContainer(1.3:5); + + julia> typeof(b) + MyContainer{Float64,UnitRange{Float64}} + +Note the somewhat surprising fact that ``T`` doesn't appear in the +declaration of field ``a``, a point that we'll return to in a moment. +With this approach, one can write functions such as:: + + function myfunc{T<:Integer, A<:AbstractArray}(c::MyContainer{T,A}) + return c.a[1]+1 + end + # Note: because we can only define MyContainer for + # A<:AbstractArray, and any unspecified parameters are arbitrary, + # the previous could have been written more succinctly as + # function myfunc{T<:Integer}(c::MyContainer{T}) + + function myfunc{T<:AbstractFloat}(c::MyContainer{T}) + return c.a[1]+2 + end + + function myfunc{T<:Integer}(c::MyContainer{T,Vector{T}}) + return c.a[1]+3 end -the compiler will not generally know the type of ``foo.field``, since it -might be modified at any time to refer to a value of a different type. -It will help to declare the most specific type possible, such as -``field::Float64`` or ``field::Array{Int64,1}``. + julia> myfunc(MyContainer(1:3)) + 2 + + julia> myfunc(MyContainer(1.0:3)) + 3.0 + + julia> myfunc(MyContainer([1:3])) + 4 + +As you can see, with this approach it's possible to specialize on both +the element type ``T`` and the array type ``A``. + +However, there's one remaining hole: we haven't enforced that ``A`` +has element type ``T``, so it's perfectly possible to construct an +object like this:: + + julia> b = MyContainer{Int64, UnitRange{Float64}}(1.3:5); + + julia> typeof(b) + MyContainer{Int64,UnitRange{Float64}} + +To prevent this, we can add an inner constructor:: + + type MyBetterContainer{T<:Real, A<:AbstractVector} + a::A + + MyBetterContainer(v::AbstractVector{T}) = new(v) + end + MyBetterContainer(v::AbstractVector) = MyBetterContainer{eltype(v),typeof(v)}(v) + + + julia> b = MyBetterContainer(1.3:5); + + julia> typeof(b) + MyBetterContainer{Float64,UnitRange{Float64}} + + julia> b = MyBetterContainer{Int64, UnitRange{Float64}}(1.3:5); + ERROR: no method MyBetterContainer(UnitRange{Float64},) + +The inner constructor requires that the element type of ``A`` be ``T``. Annotate values taken from untyped locations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ It is often convenient to work with data structures that may contain -values of any type, such as the original ``Foo`` type above, or cell +values of any type, such as cell arrays (arrays of type ``Array{Any}``). But, if you're using one of these structures and happen to know the type of an element, it helps to share this knowledge with the compiler:: From 4a2d2ccedb0a3992c75d31e803c9256672d6c27e Mon Sep 17 00:00:00 2001 From: Kiran Pamnany Date: Mon, 13 Apr 2015 15:01:09 +0530 Subject: [PATCH 0016/1938] Affinity fix for threads. Multi-threaded workloads. --- src/threading.c | 4 +- test/perf/threads/laplace3d/README | 9 +- test/perf/threads/laplace3d/laplace3d.jl | 22 ++-- .../{laplace3d.m => laplace3d_devec.m} | 18 +-- test/perf/threads/laplace3d/laplace3d_vec.m | 34 ++++++ test/perf/threads/stockcorr/pstockcorr.jl | 113 ++++++++++++++++++ test/perf/threads/stockcorr/pstockcorr.m | 45 +++++++ 7 files changed, 217 insertions(+), 28 deletions(-) rename test/perf/threads/laplace3d/{laplace3d.m => laplace3d_devec.m} (84%) create mode 100644 test/perf/threads/laplace3d/laplace3d_vec.m create mode 100644 test/perf/threads/stockcorr/pstockcorr.jl create mode 100755 test/perf/threads/stockcorr/pstockcorr.m diff --git a/src/threading.c b/src/threading.c index 6c9c5c5900823..de5aebe3c445b 100644 --- a/src/threading.c +++ b/src/threading.c @@ -94,7 +94,7 @@ int ti_threadcreate(uint64_t *pthread_id, int proc_num, pthread_attr_t attr; pthread_attr_init(&attr); -#ifdef _OS_LINUX +#ifdef _OS_LINUX_ cpu_set_t cset; if (proc_num >= 0) { CPU_ZERO(&cset); @@ -111,7 +111,7 @@ int ti_threadcreate(uint64_t *pthread_id, int proc_num, // set thread affinity void ti_threadsetaffinity(uint64_t pthread_id, int proc_num) { -#ifdef _OS_LINUX +#ifdef _OS_LINUX_ cpu_set_t cset; CPU_ZERO(&cset); diff --git a/test/perf/threads/laplace3d/README b/test/perf/threads/laplace3d/README index a57205897ee0c..67553fc52ecae 100644 --- a/test/perf/threads/laplace3d/README +++ b/test/perf/threads/laplace3d/README @@ -9,11 +9,6 @@ E.g.: $ ./laplace3d 258 258 258 1000 avx 0 -The Matlab/Octave version is orders of magnitude slower than the C -and Julia versions, so it uses a smaller grid. - -Speedup with 16 threads on a 2 socket SNB-EP: - -C: X -Julia: Y +The Matlab/Octave versions are enough slower than the C and Julia +versions that they use only 1/10 the iterations. diff --git a/test/perf/threads/laplace3d/laplace3d.jl b/test/perf/threads/laplace3d/laplace3d.jl index 5d153c1e329c6..9a6e1998f1664 100644 --- a/test/perf/threads/laplace3d/laplace3d.jl +++ b/test/perf/threads/laplace3d/laplace3d.jl @@ -1,3 +1,5 @@ +## 3D Laplace equation + using Base.Cartesian using Base.Threading @@ -21,6 +23,7 @@ function l3d_orig(u1::Array{Float32,3}, u3::Array{Float32,3}, end end +## @threads 'call' form function l3d_threadfun(u1, u3, nx, ny, nz) tid = threadid() tnz, rem = divrem(nz-2, nthreads()) @@ -43,6 +46,7 @@ function l3d_threadfun(u1, u3, nx, ny, nz) end end +## @threads 'block' form function l3d_threadblock(u1, u3, nx, ny, nz) @threads all begin tid = threadid() @@ -67,6 +71,7 @@ function l3d_threadblock(u1, u3, nx, ny, nz) end end +## @threads 'for' form function l3d_threadfor(u1, u3, nx, ny, nz) @threads all for k_3=2:nz-1 for k_2 = 2:ny-1 @@ -77,11 +82,8 @@ function l3d_threadfor(u1, u3, nx, ny, nz) end end -precompile(l3d_threadfun, (Array{Float32,3}, Array{Float32,3}, Int64, Int64, Int64)) -precompile(l3d_threadblock, (Array{Float32,3}, Array{Float32,3}, Int64, Int64, Int64)) -precompile(l3d_threadfor, (Array{Float32,3}, Array{Float32,3}, Int64, Int64, Int64)) - -function laplace3d(nx=258, ny=258, nz=258; iters=100, verify=false) +## initialize and run +function laplace3d(nx=290, ny=290, nz=290; iters=1000, verify=false) u1 = Array(Float32, nx, ny, nz) u3 = Array(Float32, nx, ny, nz) @nloops 3 k u1 begin @@ -92,10 +94,10 @@ function laplace3d(nx=258, ny=258, nz=258; iters=100, verify=false) end end @time for n in 1:iters + l3d_threadfor(u1, u3, nx, ny, nz) # @threads all l3d_threadfun(u1, u3, nx, ny, nz) # l3d_threadblock(u1, u3, nx, ny, nz) - # l3d_threadfor(u1, u3, nx, ny, nz) - ccall(:jl_threading_run, Void, (Any, Any), l3d_threadfun, (u1, u3, nx, ny, nz)) + # ccall(:jl_threading_run, Void, (Any, Any), l3d_threadfun, (u1, u3, nx, ny, nz)) foo = u1 u1 = u3 u3 = foo @@ -126,8 +128,6 @@ function laplace3d(nx=258, ny=258, nz=258; iters=100, verify=false) end end -gc() - -laplace3d(verify=true) -#laplace3d(iters=1000, verify=false) +@time laplace3d() +#ccall(:jl_threading_profile, Void, ()) diff --git a/test/perf/threads/laplace3d/laplace3d.m b/test/perf/threads/laplace3d/laplace3d_devec.m similarity index 84% rename from test/perf/threads/laplace3d/laplace3d.m rename to test/perf/threads/laplace3d/laplace3d_devec.m index f0431e02a1bd2..38473d9f04025 100644 --- a/test/perf/threads/laplace3d/laplace3d.m +++ b/test/perf/threads/laplace3d/laplace3d_devec.m @@ -1,7 +1,9 @@ -nx = 34; -ny = 34; -nz = 34; -niters = 100; +% 3D Laplace equation solver + +nx = 290; +ny = 290; +nz = 290; +iters = 100; sixth = 1.0/6.0; u1 = zeros(nx, ny, nz); @@ -13,13 +15,13 @@ if k1==1 || k1==nx || k2==1 || k2==ny || k3==1 || k3==nz u1(k1,k2,k3) = 1.0; u2(k1,k2,k3) = 1.0; - endif + end end end end -tic() -for n = 1:niters +tic +for n = 1:iters for k3 = 2:nz-1 for k2 = 2:ny-1 for k1 = 2:nx-1 @@ -32,5 +34,5 @@ u1 = u2; u2 = foo; end -toc() +toc diff --git a/test/perf/threads/laplace3d/laplace3d_vec.m b/test/perf/threads/laplace3d/laplace3d_vec.m new file mode 100644 index 0000000000000..43767dc352774 --- /dev/null +++ b/test/perf/threads/laplace3d/laplace3d_vec.m @@ -0,0 +1,34 @@ +% 3D Laplace equation solver + +nx = 290; +ny = 290; +nz = 290; +iters = 100; +sixth = 1.0/6.0; + +u1 = zeros(nx, ny, nz); +u2 = zeros(nx, ny, nx); + +%Boundary conditions +u1(1,:,:) = 1.0; +u1(nx,:,:) = 1.0; +u1(:,1,:) = 1.0; +u1(:,ny,:) = 1.0; +u1(:,:,1) = 1.0; +u1(:,:,nz) = 1.0; +u2 = u1; + +i = 2:nx-1; +j = 2:ny-1; +k = 2:nz-1; + +%Main +tic +for n = 1:iters + u2(i,j,k) = (u1(i-1,j,k) + u1(i+1,j,k) + u1(i,j-1,k) + u1(i,j+1,k) + u1(i,j,k-1) + u1(i,j,k+1))*sixth; + foo = u1; + u1 = u2; + u2 = foo; +end +toc + diff --git a/test/perf/threads/stockcorr/pstockcorr.jl b/test/perf/threads/stockcorr/pstockcorr.jl new file mode 100644 index 0000000000000..91dcd826f492c --- /dev/null +++ b/test/perf/threads/stockcorr/pstockcorr.jl @@ -0,0 +1,113 @@ +## Test case from Issue #445 +## +## Added threaded implementation (2015-04-06) + +using Base.Threading + +#STOCKCORR - The original, unoptimised code that simulates two correlated assets +function stockcorr() + + ## Correlated asset information + CurrentPrice = [78. 102.] # Initial Prices of the two stocks + Corr = [1. 0.4; 0.4 1.] # Correlation Matrix + T = 500 # Number of days to simulate = 2years = 500days + n = 1000000 # Number of simulations + dt = 1/250 # Time step (1year = 250days) + Div=[0.01 0.01] # Dividend + Vol=[0.2 0.3] # Volatility + + ## Market Information + r = 0.03 # Risk-free rate + + ## Define storages + SimulPriceA = zeros(T,n) # Simulated Price of Asset A + SimulPriceA[1,:] = CurrentPrice[1] + SimulPriceB = zeros(T,n) # Simulated Price of Asset B + SimulPriceB[1,:] = CurrentPrice[2] + + ## Generating the paths of stock prices by Geometric Brownian Motion + UpperTriangle=chol(Corr) # UpperTriangle Matrix by Cholesky decomposition + + for i = 1:n + Wiener = randn(T-1,2) + CorrWiener = Wiener*UpperTriangle + for j = 2:T + SimulPriceA[j,i] = SimulPriceA[j-1,i]*exp((r-Div[1]-Vol[1]^2/2)*dt+Vol[1]*sqrt(dt)*CorrWiener[j-1,1]) + SimulPriceB[j,i] = SimulPriceB[j-1,i]*exp((r-Div[2]-Vol[2]^2/2)*dt+Vol[2]*sqrt(dt)*CorrWiener[j-1,2]) + end + end + + return (SimulPriceA, SimulPriceB) +end + +# Run paths in parallel +# NOTE: this has to be in its own function due to #10718 +function runpath!(n, Wiener, CorrWiener, SA, SB, T, UpperTriangle, k11, k12, k21, k22, rngs) + @threads all for i = 1:n + #for i = 1:n + randn!(rngs[threadid()], Wiener) + #randn!(rngs[1], Wiener) + A_mul_B!(CorrWiener, Wiener, UpperTriangle) + @simd for j = 2:T + @inbounds SA[j, i] = SA[j-1, i] * exp(k11 + k12*CorrWiener[j-1, 1]) + @inbounds SB[j, i] = SB[j-1, i] * exp(k21 + k22*CorrWiener[j-1, 2]) + end + end +end + +#precompile(runpath!, (Int64, Array{Float64,2}, Array{Float64,2}, Array{Float64,2}, Int64, Array{Float64, 2}, Float64, Float64, Float64, Float64)) + +# Threaded version +function pstockcorr() + + ## Correlated asset information + const CurrentPrice = [78. 102.] # Initial Prices of the two stocks + const Corr = [1. 0.4; 0.4 1.] # Correlation Matrix + const T = 500 # Number of days to simulate = 2years = 500days + const n = 1000000 # Number of simulations + const dt = 1/250 # Time step (1year = 250days) + const Div=[0.01 0.01] # Dividend + const Vol=[0.2 0.3] # Volatility + + ## Market Information + const r = 0.03 # Risk-free rate + + ## Define storages + SimulPriceA = zeros(T,n) # Simulated Price of Asset A + SimulPriceA[1,:] = CurrentPrice[1] + SimulPriceB = zeros(T,n) # Simulated Price of Asset B + SimulPriceB[1,:] = CurrentPrice[2] + + ## Generating the paths of stock prices by Geometric Brownian Motion + const UpperTriangle = full(chol(Corr)) # UpperTriangle Matrix by Cholesky decomposition + + # Optimization: pre-allocate these for performance + # NOTE: the new GC will hopefully fix this, but currently GC time + # kills performance if we don't do in-place computations + Wiener = Array(Float64, T-1, 2) + CorrWiener = Array(Float64, T-1, 2) + + # Runtime requirement: need per-thread RNG since it stores state + rngs = [MersenneTwister(777) for x in 1:nthreads()] + + # Optimization: pre-computable factors + # NOTE: this should be automatically hoisted out of the loop + # NOTE: applied this to the MATLAB version as well + k11 = (r-Div[1]-Vol[1]^2/2)*dt + k12 = Vol[1]*sqrt(dt) + k21 = (r-Div[2]-Vol[2]^2/2)*dt + k22 = Vol[2]*sqrt(dt) + + # NOTE: this reduces reported GC allocation to 576 bytes (due to?) + # and gives a slight boost to performance but we shouldn't need it... + runpath!(1, Wiener, CorrWiener, SimulPriceA, SimulPriceB, T, UpperTriangle, k11, k12, k21, k22, rngs) + + # run + @time runpath!(n, Wiener, CorrWiener, SimulPriceA, SimulPriceB, T, UpperTriangle, k11, k12, k21, k22, rngs) + + return (SimulPriceA, SimulPriceB) +end + +@time pstockcorr() +#ccall(:jl_threading_profile, Void, ()) + diff --git a/test/perf/threads/stockcorr/pstockcorr.m b/test/perf/threads/stockcorr/pstockcorr.m new file mode 100755 index 0000000000000..76d211813ca9c --- /dev/null +++ b/test/perf/threads/stockcorr/pstockcorr.m @@ -0,0 +1,45 @@ +%% Simulate two correlated assets +%% Manually hoisted some computations outside the loop + +%% Correlated asset information +CurrentPrice = [78 102]; %Initial Prices of the two stocks +Corr = [1 0.4; 0.4 1]; %Correlation Matrix +T = 500; %Number of days to simulate = 2years = 500days +n = 1000000; %Number of simulations +dt = 1/250; %Time step (1year = 250days) +Div=[0.01 0.01]; %Dividend +Vol=[0.2 0.3]; %Volatility + +%%Market Information +r = 0.03; %Risk-free rate + +%% Define storages +SimulPriceA=zeros(T,n); %Simulated Price of Asset A +SimulPriceA(1,:)=CurrentPrice(1); +SimulPriceB=zeros(T,n); %Simulated Price of Asset B +SimulPriceB(1,:)=CurrentPrice(2); + +%% Generating the paths of stock prices by Geometric Brownian Motion +UpperTriangle=chol(Corr); %UpperTriangle Matrix by Cholesky decomposition + +%% Pre-calculating factors +k11 = (r - Div(1) - Vol(1)^2/2)*dt; +k12 = Vol(1) * sqrt(dt); +k21 = (r - Div(2) - Vol(2)^2/2)*dt; +k22 = Vol(2) * sqrt(dt); + +tic +for i=1:n + Wiener=randn(T-1,2); + CorrWiener=Wiener*UpperTriangle; + for j=2:T + SimulPriceA(j,i)=SimulPriceA(j-1,i)*exp(k11+k12*CorrWiener(j-1,1)); + SimulPriceB(j,i)=SimulPriceB(j-1,i)*exp(k21+k22*CorrWiener(j-1,2)); + end +end +toc + +%% Plot the distribution of final prices +% Comment this section out if doing timings +%subplot(1,2,1);hist(SimulPriceA(end,:),100); +%subplot(1,2,2);hist(SimulPriceB(end,:),100); From 0f17e69d93341f642385b2cf19e7947bd584a8fc Mon Sep 17 00:00:00 2001 From: Kiran Pamnany Date: Fri, 5 Jun 2015 20:39:36 +0530 Subject: [PATCH 0017/1938] Lattice-Boltzmann 3D workload. --- test/perf/threads/lbm3d/README | 9 + test/perf/threads/lbm3d/circshift.jl | 108 ++++++++++++ test/perf/threads/lbm3d/lbm3d.jl | 215 +++++++++++++++++++++++ test/perf/threads/lbm3d/lbm3d.m | 90 ++++++++++ test/perf/threads/stockcorr/pstockcorr.m | 0 5 files changed, 422 insertions(+) create mode 100644 test/perf/threads/lbm3d/README create mode 100644 test/perf/threads/lbm3d/circshift.jl create mode 100644 test/perf/threads/lbm3d/lbm3d.jl create mode 100644 test/perf/threads/lbm3d/lbm3d.m mode change 100755 => 100644 test/perf/threads/stockcorr/pstockcorr.m diff --git a/test/perf/threads/lbm3d/README b/test/perf/threads/lbm3d/README new file mode 100644 index 0000000000000..c7ce696941bad --- /dev/null +++ b/test/perf/threads/lbm3d/README @@ -0,0 +1,9 @@ +3D Lattice Boltzmann (BGK) model of a fluid (http://exolete.com/lbm/) + +The Matlab/Octave version is from the website. The Julia version is +rewritten in devectorized form with threading directives applied to +the parallelizable code. + +The circshift! implementation needs work to support arbitrary +dimensioned grids. + diff --git a/test/perf/threads/lbm3d/circshift.jl b/test/perf/threads/lbm3d/circshift.jl new file mode 100644 index 0000000000000..17a58166cd71c --- /dev/null +++ b/test/perf/threads/lbm3d/circshift.jl @@ -0,0 +1,108 @@ +# An in-place version of circshift. Needs generalization -- currently +# works only on a 3D hyperplane of a 4D array and supports only unit +# shifts. Code needs refactoring to reduce duplication. + +using Match + +# a is the 4D array; index specifies which 3D hyperplane; vec +# specifies the shifts (only 1 and -1) +function circshift3d1!(a,index,vec) + nx = size(a,1) + ny = size(a,2) + nz = size(a,3) + for i = 1 : length(vec) + @match vec[i] begin + 1 => shift1!(a,nx,ny,nz,index,i) + -1 => shiftm1!(a,nx,ny,nz,index,i) + end + end +end + +function shift1!(a,nx,ny,nz,index,i) + @match i begin + 1 => shifti!(a,nx,ny,nz,index) + 2 => shiftj!(a,nx,ny,nz,index) + 3 => shiftk!(a,nx,ny,nz,index) + end +end + +function shiftm1!(a,nx,ny,nz,index,i) + @match i begin + 1 => shiftmi!(a,nx,ny,nz,index) + 2 => shiftmj!(a,nx,ny,nz,index) + 3 => shiftmk!(a,nx,ny,nz,index) + end +end + +function shifti!(a,nx,ny,nz,index) + for k = 1:nz + for j = 1:ny + t = a[end,j,k,index] + for i = nx-1:-1:1 + a[i+1,j,k,index] = a[i,j,k,index] + end + a[1,j,k,index] = t + end + end +end + +function shiftj!(a,nx,ny,nz,index) + for i = 1:nx + for k = 1:nz + t = a[i,end,k,index] + for j = ny-1:-1:1 + a[i,j+1,k,index] = a[i,j,k,index] + end + a[i,1,k,index] = t + end + end +end + +function shiftk!(a,nx,ny,nz,index) + for i = 1:nx + for j = 1:ny + t = a[i,j,end,index] + for k = nz-1:-1:1 + a[i,j,k+1,index] = a[i,j,k,index] + end + a[i,j,1,index] = t + end + end +end + +function shiftmi!(a,nx,ny,nz,index) + for k = 1:nz + for j = 1:ny + t = a[1,j,k,index] + for i = 1:nx-1 + a[i,j,k,index] = a[i+1,j,k,index] + end + a[end,j,k,index] = t + end + end +end + +function shiftmj!(a,nx,ny,nz,index) + for k = 1:nz + for i = 1:nx + t = a[i,1,k,index] + for j = 1:ny-1 + a[i,j,k,index] = a[i,j+1,k,index] + end + a[i,end,k,index] = t + end + end +end + +function shiftmk!(a,nx,ny,nz,index) + for i = 1:nx + for j = 1:ny + t = a[i,j,1,index] + for k = 1:nz-1 + a[i,j,k,index] = a[i,j,k+1,index] + end + a[i,j,end,index] = t + end + end +end + diff --git a/test/perf/threads/lbm3d/lbm3d.jl b/test/perf/threads/lbm3d/lbm3d.jl new file mode 100644 index 0000000000000..ffaee05cd8eb3 --- /dev/null +++ b/test/perf/threads/lbm3d/lbm3d.jl @@ -0,0 +1,215 @@ +# 3D Lattice Boltzmann (BGK) model of a fluid. +# (http://exolete.com/lbm/) +# D3Q19 model. At each timestep, particle densities propagate +# outwards in the directions indicated in the figure. An +# equivalent 'equilibrium' density is found, and the densities +# relax towards that state, in a proportion governed by omega. +# Iain Haslam, March 2006. +# +# Julia version: devectorized and threaded. The in-place +# circshift implementation is needed because of issue #10317. +# Despite that, GC time is ~22% of runtime. relax! and +# calc_equi! are broken out because of issue #10718. The +# threaded call form is used and the for loop form is is +# commented out in both because of issue #10527. + +#using PyPlot +using Base.Threading + +include("circshift.jl") + +const t1 = 1/3 +const t2 = 1/18 +const t3 = 1/36 + +const prop_shifts = tuple([ 0 0 1], [ 0 0 -1], [ 0 1 0], + [ 0 -1 0], [ 1 0 0], [-1 0 0], + [ 1 1 0], [ 1 -1 0], [-1 1 0], + [-1 -1 0], [ 1 0 1], [ 1 0 -1], + [-1 0 1], [-1 0 -1], [ 0 1 1], + [ 0 1 -1], [ 0 -1 1], [ 0 -1 -1]); + +const fourths = tuple([ 6, 8, 9,12,13], [ 7,10,11,14,15], + [ 4, 8,10,16,17], [ 5, 9,11,18,19], + [ 2,12,14,16,18], [ 3,13,15,17,19]) + + +function relax!(F, UX, UY, UZ, nx, ny, nz, deltaU, t1D, t2D, t3D, sSQU) + tid = threadid() + outerrange = Base.splitrange(nx, nthreads()) + for i = outerrange[tid] + #@threads all for i = 1:nx + for j = 1:ny + for k = 1:nz + density = 0.0 + for l = 1:size(F,4) + density = density + F[i,j,k,l] + end + fs = Array(Float64, 6) + for l = 1:6 + fs[l] = 0.0 + for m = 1:5 + fs[l] = fs[l] + F[i,j,k,fourths[l][m]] + end + end + UX[i,j,k] = (fs[1] - fs[2]) / density + UY[i,j,k] = (fs[3] - fs[4]) / density + UZ[i,j,k] = (fs[5] - fs[6]) / density + + if i == 1 + UX[i,j,k] = UX[i,j,k] + deltaU # Increase inlet pressure + end + + t1D[i,j,k] = t1 * density + t2D[i,j,k] = t2 * density + t3D[i,j,k] = t3 * density + sSQU[i,j,k] = 3/2 * (UX[i,j,k]^2 + UY[i,j,k]^2 + UZ[i,j,k]^2) + end + end + end +end + + +function calc_equi!(F, FEQ, t1D, t2D, t3D, U, UX, UY, UZ, sSQU, nx, ny, nz, omega) + tid = threadid() + outerrange = Base.splitrange(nx, nthreads()) + for i = outerrange[tid] + #@threads all for i = 1:nx + #tid = threadid() + for j = 1:ny + for k = 1:nz + + FEQ[i,j,k,1] = t1D[i,j,k,1] * (1 - sSQU[i,j,k,1]) + + # nearest neighbors + FEQ[i,j,k,2] = t2D[i,j,k,1] * (1 + 3*UZ[i,j,k,1] + 9/2*UZ[i,j,k,1]^2 - sSQU[i,j,k,1]) + FEQ[i,j,k,3] = t2D[i,j,k,1] * (1 - 3*UZ[i,j,k,1] + 9/2*UZ[i,j,k,1]^2 - sSQU[i,j,k,1]) + FEQ[i,j,k,4] = t2D[i,j,k,1] * (1 + 3*UY[i,j,k,1] + 9/2*UY[i,j,k,1]^2 - sSQU[i,j,k,1]) + FEQ[i,j,k,5] = t2D[i,j,k,1] * (1 - 3*UY[i,j,k,1] + 9/2*UY[i,j,k,1]^2 - sSQU[i,j,k,1]) + FEQ[i,j,k,6] = t2D[i,j,k,1] * (1 + 3*UX[i,j,k,1] + 9/2*UX[i,j,k,1]^2 - sSQU[i,j,k,1]) + FEQ[i,j,k,7] = t2D[i,j,k,1] * (1 - 3*UX[i,j,k,1] + 9/2*UX[i,j,k,1]^2 - sSQU[i,j,k,1]) + + U[1,tid] = UX[i,j,k,1] + UY[i,j,k,1] + U[2,tid] = UX[i,j,k,1] - UY[i,j,k,1] + U[3,tid] = -UX[i,j,k,1] + UY[i,j,k,1] + U[4,tid] = -U[1,tid] + U[5,tid] = UX[i,j,k,1] + UZ[i,j,k,1] + U[6,tid] = UX[i,j,k,1] - UZ[i,j,k,1] + U[7,tid] = -U[6,tid] + U[8,tid] = -U[5,tid] + U[9,tid] = UY[i,j,k,1] + UZ[i,j,k,1] + U[10,tid] = UY[i,j,k,1] - UZ[i,j,k,1] + U[11,tid] = -U[10,tid] + U[12,tid] = -U[9,tid] + + # next-nearest neighbors + for l = 1:12 + FEQ[i,j,k,l+7] = t3D[i,j,k,1] * (1 + 3*U[l,tid] + 9/2*(U[l,tid]^2) - sSQU[i,j,k,1]) + end + + for l = 1:19 + F[i,j,k,l] = omega * FEQ[i,j,k,l] + (1 - omega) * F[i,j,k,l] + end + + end + end + end +end + +precompile(calc_equi!, (Array{Float64,4}, Array{Float64,4}, Array{Float64,3}, Array{Float64,3}, Array{Float64,3}, Array{Float64,2}, Array{Float64,3}, Array{Float64,3}, Array{Float64,3}, Array{Float64,3}, Int64, Int64, Int64, Float64)) + +function lbm3d(n) + const nx = n + const ny = nx + const nz = nx + const omega = 1.0 + const density = 1.0 + + tprop = 0 + trelax = 0 + tequi = 0 + + F = repeat([density/19], outer=[nx, ny, nz, 19]) + FEQ = F + matsize = nx*ny*nz + + CI = [0:matsize:matsize*19;]' + + BOUND = Array(Float64,nx,ny,nz) + + for i=1:nx, j=1:ny, k=1:nz + BOUND[i,j,k] = ((i-5)^2 + (j-6)^2 + (k-7)^2) < 6 + end + + BOUND[:,:,1] = 1 + BOUND[:,1,:] = 1 + + ON = find(BOUND); # matrix offset of each Occupied Node + + TO_REFLECT = [ON+CI[2] ON+CI[3] ON+CI[4] ON+CI[5] ON+CI[6] ON+CI[7] ON+CI[8] ON+CI[9] ON+CI[10] ON+CI[11] ON+CI[12] ON+CI[13] ON+CI[14] ON+CI[15] ON+CI[16] ON+CI[17] ON+CI[18] ON+CI[19]] + REFLECTED = [ON+CI[3] ON+CI[2] ON+CI[5] ON+CI[4] ON+CI[7] ON+CI[6] ON+CI[11] ON+CI[10] ON+CI[9] ON+CI[8] ON+CI[15] ON+CI[14] ON+CI[13] ON+CI[12] ON+CI[19] ON+CI[18] ON+CI[17] ON+CI[16]] + + UX = Array(Float64,nx,ny,nz) + UY = Array(Float64,nx,ny,nz) + UZ = Array(Float64,nx,ny,nz) + U = Array(Float64,12,nthreads()) + t1D = Array(Float64,nx,ny,nz) + t2D = Array(Float64,nx,ny,nz) + t3D = Array(Float64,nx,ny,nz) + sSQU = Array(Float64,nx,ny,nz) + + avu = 1 + prevavu = 1 + ts = 0 + deltaU = 1e-7 + numactivenodes = sum(1-BOUND) + + @time while (ts < 4000 && (1e-10 < abs((prevavu-avu)/avu))) || ts < 100 + tic() + # Propagate -- nearest and next-nearest neighbors + for i = 2:19 + circshift3d1!(F, i, prop_shifts[i-1]) + end + tprop = tprop + toq() + + # Densities bouncing back at next timestep + BOUNCEDBACK = F[TO_REFLECT] + + tic() + + # Relax; calculate equilibrium state (FEQ) with equivalent speed and density to F + @threads all relax!(F, UX, UY, UZ, nx, ny, nz, deltaU, t1D, t2D, t3D, sSQU) + for o in ON + UX[o] = UY[o] = UZ[o] = t1D[o] = t2D[o] = t3D[o] = sSQU[o] = 0.0 + end + + trelax = trelax + toq() + tic() + + # Calculate equilibrium distribution: stationary + @threads all calc_equi!(F, FEQ, t1D, t2D, t3D, U, UX, UY, UZ, sSQU, nx, ny, nz, omega) + + tequi = tequi + toq() + + F[REFLECTED] = BOUNCEDBACK + + prevavu = avu + avu = sum(UX) / numactivenodes + ts = ts + 1 + #println(avu) + end + + println("ts: $(ts)") + println("propagation time: $tprop") + println("relaxation time: $trelax") + println("equilibrium time: $tequi") + + #zcut=5 + #quiver(UY[:,:,zcut], UX[:,:,zcut]) + #xlabel("y"); ylabel("x") + #title("Flow field at z = $(zcut)), after $(ts)") +end + +@time lbm3d(36) +#ccall(:jl_threading_profile, Void, ()) + diff --git a/test/perf/threads/lbm3d/lbm3d.m b/test/perf/threads/lbm3d/lbm3d.m new file mode 100644 index 0000000000000..56295ce6fece5 --- /dev/null +++ b/test/perf/threads/lbm3d/lbm3d.m @@ -0,0 +1,90 @@ +% 3D Lattice Boltzmann (BGK) model of a fluid. +% (http://exolete.com/lbm/) +% D3Q19 model. At each timestep, particle densities propagate +% outwards in the directions indicated in the figure. An +% equivalent 'equilibrium' density is found, and the densities +% relax towards that state, in a proportion governed by omega. +% Iain Haslam, March 2006. +nx=24;ny=nx;nz=nx; omega=1.0; density=1.0;t1=1/3; t2=1/18; t3=1/36; +F=repmat(density/19,[nx ny nz 19]); FEQ=F; matsize=nx*ny*nz; +CI=[0:matsize:matsize*19]; +BOUND=zeros(nx,ny,nz); +for i=1:nx, for j=1:ny, for k=1:nz + BOUND(i,j,k)=((i-5)^2+(j-6)^2+(k-7)^2)<6; +end, end, end +BOUND(:,:,1)=1;BOUND(:,1,:)=1; +ON=find(BOUND); %matrix offset of each Occupied Node +TO_REFLECT=[ON+CI(2) ON+CI(3) ON+CI(4) ON+CI(5) ON+CI(6) ON+CI(7) ON+CI(8) ... + ON+CI(9) ON+CI(10) ON+CI(11) ON+CI(12) ON+CI(13) ON+CI(14) ON+CI(15) ... + ON+CI(16) ON+CI(17) ON+CI(18) ON+CI(19)]; +REFLECTED=[ON+CI(3) ON+CI(2) ON+CI(5) ON+CI(4) ON+CI(7) ON+CI(6) ON+CI(11) ... + ON+CI(10) ON+CI(9) ON+CI(8) ON+CI(15) ON+CI(14) ON+CI(13) ON+CI(12) ... + ON+CI(19) ON+CI(18) ON+CI(17) ON+CI(16)]; +avu=1; prevavu=1; ts=0; deltaU=1e-7; numactivenodes=sum(sum(sum(1-BOUND))); +tic; +while (ts<4000 & 1e-10 Date: Fri, 19 Jun 2015 01:04:09 +0530 Subject: [PATCH 0018/1938] Minor updates & fixes to multithreaded workloads. --- test/perf/threads/lbm3d/lbm3d.m | 2 +- test/perf/threads/stockcorr/pstockcorr.jl | 8 ++-- .../{pstockcorr.m => stockcorr_devec.m} | 3 +- test/perf/threads/stockcorr/stockcorr_vec.m | 42 +++++++++++++++++++ 4 files changed, 48 insertions(+), 7 deletions(-) rename test/perf/threads/stockcorr/{pstockcorr.m => stockcorr_devec.m} (93%) create mode 100644 test/perf/threads/stockcorr/stockcorr_vec.m diff --git a/test/perf/threads/lbm3d/lbm3d.m b/test/perf/threads/lbm3d/lbm3d.m index 56295ce6fece5..9aa077bf9309d 100644 --- a/test/perf/threads/lbm3d/lbm3d.m +++ b/test/perf/threads/lbm3d/lbm3d.m @@ -5,7 +5,7 @@ % equivalent 'equilibrium' density is found, and the densities % relax towards that state, in a proportion governed by omega. % Iain Haslam, March 2006. -nx=24;ny=nx;nz=nx; omega=1.0; density=1.0;t1=1/3; t2=1/18; t3=1/36; +nx=36;ny=nx;nz=nx; omega=1.0; density=1.0;t1=1/3; t2=1/18; t3=1/36; F=repmat(density/19,[nx ny nz 19]); FEQ=F; matsize=nx*ny*nz; CI=[0:matsize:matsize*19]; BOUND=zeros(nx,ny,nz); diff --git a/test/perf/threads/stockcorr/pstockcorr.jl b/test/perf/threads/stockcorr/pstockcorr.jl index 91dcd826f492c..03091d2c230b0 100644 --- a/test/perf/threads/stockcorr/pstockcorr.jl +++ b/test/perf/threads/stockcorr/pstockcorr.jl @@ -58,13 +58,12 @@ end #precompile(runpath!, (Int64, Array{Float64,2}, Array{Float64,2}, Array{Float64,2}, Int64, Array{Float64, 2}, Float64, Float64, Float64, Float64)) # Threaded version -function pstockcorr() +function pstockcorr(n) ## Correlated asset information const CurrentPrice = [78. 102.] # Initial Prices of the two stocks const Corr = [1. 0.4; 0.4 1.] # Correlation Matrix const T = 500 # Number of days to simulate = 2years = 500days - const n = 1000000 # Number of simulations const dt = 1/250 # Time step (1year = 250days) const Div=[0.01 0.01] # Dividend const Vol=[0.2 0.3] # Volatility @@ -99,8 +98,7 @@ function pstockcorr() k22 = Vol[2]*sqrt(dt) # NOTE: this reduces reported GC allocation to 576 bytes (due to?) - # and gives a slight boost to performance but we shouldn't need it... - runpath!(1, Wiener, CorrWiener, SimulPriceA, SimulPriceB, T, UpperTriangle, k11, k12, k21, k22, rngs) + #runpath!(1, Wiener, CorrWiener, SimulPriceA, SimulPriceB, T, UpperTriangle, k11, k12, k21, k22, rngs) # run @time runpath!(n, Wiener, CorrWiener, SimulPriceA, SimulPriceB, T, UpperTriangle, k11, k12, k21, k22, rngs) @@ -108,6 +106,6 @@ function pstockcorr() return (SimulPriceA, SimulPriceB) end -@time pstockcorr() +@time pstockcorr(1000000) #ccall(:jl_threading_profile, Void, ()) diff --git a/test/perf/threads/stockcorr/pstockcorr.m b/test/perf/threads/stockcorr/stockcorr_devec.m similarity index 93% rename from test/perf/threads/stockcorr/pstockcorr.m rename to test/perf/threads/stockcorr/stockcorr_devec.m index 76d211813ca9c..39987ea28c356 100644 --- a/test/perf/threads/stockcorr/pstockcorr.m +++ b/test/perf/threads/stockcorr/stockcorr_devec.m @@ -1,5 +1,6 @@ %% Simulate two correlated assets -%% Manually hoisted some computations outside the loop +%% Manually hoisted some computations outside the loop; see ../../kernel/ +%% for the original version %% Correlated asset information CurrentPrice = [78 102]; %Initial Prices of the two stocks diff --git a/test/perf/threads/stockcorr/stockcorr_vec.m b/test/perf/threads/stockcorr/stockcorr_vec.m new file mode 100644 index 0000000000000..dce3c8159aa7b --- /dev/null +++ b/test/perf/threads/stockcorr/stockcorr_vec.m @@ -0,0 +1,42 @@ +%% Simulate two correlated assets +%% Vectorized version (from http://www.walkingrandomly.com/?p=3604) + +%% Correlated asset information +CurrentPrice = [78 102]; %Initial Prices of the two stocks +Corr = [1 0.4; 0.4 1]; %Correlation Matrix +T = 500; %Number of days to simulate = 2years = 500days +Div=[0.01 0.01]; %Dividend +Vol=[0.2 0.3]; %Volatility + +%% Market Information +r = 0.03; %Risk-free rate + +%% Simulation parameters +n=1000000; %Number of simulation +dt=1/250; %Time step (1year = 250days) + +%% Define storages +SimulPrices=repmat(CurrentPrice,n,1); +CorrWiener = zeros(T-1,2,n); + +%% Generating the paths of stock prices by Geometric Brownian Motion +UpperTriangle=chol(Corr); %UpperTriangle Matrix by Cholesky decomposition + +tic; +for i=1:n + CorrWiener(:,:,i)=randn(T-1,2)*UpperTriangle; +end +Volr = repmat(Vol,[T-1,1,n]); +Divr = repmat(Div,[T-1,1,n]); + +%% do simulation +sim = cumprod(exp((r-Divr-Volr.^2./2).*dt+Volr.*sqrt(dt).*CorrWiener)); +%get just the final prices +SimulPrices = SimulPrices.*reshape(sim(end,:,:),2,n)'; +toc; + +%% Plot the distribution of final prices +% Comment this section out if doing timings +%subplot(1,2,1);hist(SimulPrices(:,1),100); +%subplot(1,2,2);hist(SimulPrices(:,2),100); + From 4573cdba247584ceeb89cc6ba5ba630640196ff6 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Sun, 28 Jun 2015 09:56:22 -0400 Subject: [PATCH 0019/1938] Seed MT differently on each proc --- test/perf/threads/stockcorr/pstockcorr.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/perf/threads/stockcorr/pstockcorr.jl b/test/perf/threads/stockcorr/pstockcorr.jl index 03091d2c230b0..a494ec7513360 100644 --- a/test/perf/threads/stockcorr/pstockcorr.jl +++ b/test/perf/threads/stockcorr/pstockcorr.jl @@ -87,7 +87,7 @@ function pstockcorr(n) CorrWiener = Array(Float64, T-1, 2) # Runtime requirement: need per-thread RNG since it stores state - rngs = [MersenneTwister(777) for x in 1:nthreads()] + rngs = [MersenneTwister(777+x) for x in 1:nthreads()] # Optimization: pre-computable factors # NOTE: this should be automatically hoisted out of the loop From 9604d751b61f2b4a1980476652ad67cfa3d413c1 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 1 Sep 2015 17:16:03 -0400 Subject: [PATCH 0020/1938] remove compiler overrides --- Make.inc | 8 ++------ Make.user | 2 -- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Make.inc b/Make.inc index ac9341d41347c..8851368e55058 100644 --- a/Make.inc +++ b/Make.inc @@ -293,7 +293,7 @@ CC = $(CROSS_COMPILE)gcc CXX = $(CROSS_COMPILE)g++ JCFLAGS = -std=gnu99 -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 JCPPFLAGS = -JCXXFLAGS = -pipe $(fPIC) -fno-rtti -std=gnu++11 +JCXXFLAGS = -pipe $(fPIC) -fno-rtti DEBUGFLAGS = -O0 -ggdb3 -DJL_DEBUG_BUILD -fstack-protector-all SHIPFLAGS = -O3 -ggdb3 -falign-functions endif @@ -303,7 +303,7 @@ CC = $(CROSS_COMPILE)clang CXX = $(CROSS_COMPILE)clang++ JCFLAGS = -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 JCPPFLAGS = -JCXXFLAGS = -pipe $(fPIC) -fno-rtti -std=gnu++11 +JCXXFLAGS = -pipe $(fPIC) -fno-rtti DEBUGFLAGS = -O0 -g -DJL_DEBUG_BUILD -fstack-protector-all SHIPFLAGS = -O3 -g ifeq ($(OS), Darwin) @@ -358,10 +358,6 @@ CC_BASE = $(shell echo $(CC) | cut -d' ' -f1) CXX_BASE = $(shell echo $(CXX) | cut -d' ' -f1) endif -ifeq ($(LLVM_VER),svn) -JCXXFLAGS += -std=c++11 -endif - JFFLAGS = -O2 $(fPIC) ifneq ($(USEMSVC),1) CPP = $(CC) -E diff --git a/Make.user b/Make.user index 20b685c13a019..b965caf28b28c 100644 --- a/Make.user +++ b/Make.user @@ -1,6 +1,4 @@ -CC = clang CFLAGS = -ftls-model=global-dynamic -CXX = clang++ LLVM_VER = svn LLVM_GIT_URL_LLVM = https://github.com/JuliaLang/llvm.git -b kf/tlsrebase5 From 2f0d7598d8b14e1d94eab19a755eb01966dab6d7 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 2 Sep 2015 00:20:36 -0400 Subject: [PATCH 0021/1938] for TLS variables, need to call dlsym rather than & to get the address of the metadata rather than the thread-local pointer --- src/codegen.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 436dd5502f96b..aa6848d824964 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4946,14 +4946,14 @@ static void init_julia_llvm_env(Module *m) false, GlobalVariable::ExternalLinkage, NULL, "jl_pgcstack", NULL, GlobalValue::GeneralDynamicTLSModel); - add_named_global(jlpgcstack_var, (void*)&jl_pgcstack); + add_named_global(jlpgcstack_var, jl_dlsym(jl_dl_handle, "jl_pgcstack")); jlexc_var = new GlobalVariable(*m, jl_pvalue_llvmt, false, GlobalVariable::ExternalLinkage, NULL, "jl_exception_in_transit", NULL, GlobalValue::GeneralDynamicTLSModel); - add_named_global(jlexc_var, (void*)&jl_exception_in_transit); + add_named_global(jlexc_var, jl_dlsym(jl_dl_handle, "jl_exception_in_transit")); global_to_llvm("__stack_chk_guard", (void*)&__stack_chk_guard, m); Function *jl__stack_chk_fail = From ac027dcc31ba8ce75b93a8e97de9d4746ed495d6 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 2 Sep 2015 00:20:57 -0400 Subject: [PATCH 0022/1938] reduce cruft from codegen init --- src/codegen.cpp | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index aa6848d824964..9a5fd4599ce06 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5595,6 +5595,11 @@ extern "C" void jl_init_codegen(void) .setTargetOptions(options) .setRelocationModel(Reloc::PIC_) .setCodeModel(CodeModel::Small) +#ifdef DISABLE_OPT + .setOptLevel(CodeGenOpt::None) +#else + .setOptLevel(CodeGenOpt::Aggressive) +#endif #if defined(USE_MCJIT) && !defined(LLVM36) .setUseMCJIT(true) #endif @@ -5614,33 +5619,13 @@ extern "C" void jl_init_codegen(void) MAttrs.append(1, "+vfp2"); // the processors that don't have VFP are old and (hopefully) rare. this affects the platform calling convention. #endif } - TargetMachine *targetMachine = eb.selectTarget( + jl_TargetMachine = eb.selectTarget( TheTriple, "", TheCPU, MAttrs); - assert(targetMachine && "Failed to select target machine -" + assert(jl_TargetMachine && "Failed to select target machine -" " Is the LLVM backend for this CPU enabled?"); - jl_TargetMachine = targetMachine->getTarget().createTargetMachine( - TheTriple.getTriple(), - targetMachine->getTargetCPU(), - targetMachine->getTargetFeatureString(), - targetMachine->Options, -#ifdef CODEGEN_TLS - Reloc::PIC_, - CodeModel::Small, -#else - Reloc::Default, - CodeModel::JITDefault, -#endif -#ifdef DISABLE_OPT - CodeGenOpt::None -#else - CodeGenOpt::Aggressive // -O3 -#endif - ); - delete targetMachine; - assert(jl_TargetMachine); #if defined(LLVM38) engine_module->setDataLayout(jl_TargetMachine->createDataLayout()); #elif defined(LLVM36) && !defined(LLVM37) From 236c5e4bb312cf409b869b4478228fec278280ff Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 2 Sep 2015 00:41:16 -0400 Subject: [PATCH 0023/1938] makefile fixes to build on darwin --- Make.user | 2 +- src/julia.h | 2 +- src/options.h | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Make.user b/Make.user index b965caf28b28c..1fd6a61d95bd9 100644 --- a/Make.user +++ b/Make.user @@ -1,4 +1,4 @@ -CFLAGS = -ftls-model=global-dynamic +JCPPFLAGS = -ftls-model=global-dynamic LLVM_VER = svn LLVM_GIT_URL_LLVM = https://github.com/JuliaLang/llvm.git -b kf/tlsrebase5 diff --git a/src/julia.h b/src/julia.h index 9561509d235e8..5e0dd0c30adaa 100644 --- a/src/julia.h +++ b/src/julia.h @@ -62,7 +62,7 @@ extern "C" { // WARNING: Threading support is incomplete. Changing the 1 to a 0 will break Julia. // Nonetheless, we define JL_THREAD and use it to give advanced notice to maintainers // of what eventual threading support will change. -#if 1 +#if 0 // Definition for compiling non-thread-safe Julia. # define JL_THREAD #elif !defined(_OS_WINDOWS_) diff --git a/src/options.h b/src/options.h index 4d8f01831d1e8..e2644dcac9795 100644 --- a/src/options.h +++ b/src/options.h @@ -18,11 +18,8 @@ // codegen options ------------------------------------------------------------ -// (Experimental) codegen support for thread-local storage -// #define CODEGEN_TLS - // (Experimental) Use MCJIT ELF, even where it's not the native format -// #define FORCE_ELF +#define FORCE_ELF // with KEEP_BODIES, we keep LLVM function bodies around for later debugging // #define KEEP_BODIES From 1a0ddf31208b000708915eb19b8dbca5d6ab74fb Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 2 Sep 2015 15:29:53 -0400 Subject: [PATCH 0024/1938] put GC barrier back --- src/gc.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/gc.c b/src/gc.c index 2060dd9c9bbda..c19d2bfd9b23a 100644 --- a/src/gc.c +++ b/src/gc.c @@ -2011,6 +2011,14 @@ void jl_gc_collect(int full) char *stack_hi = (char*)gc_get_stack_ptr(); gc_debug_print(); JL_SIGATOMIC_BEGIN(); + + ti_threadgroup_barrier(tgworld, ti_tid); + + if (ti_tid != 0) { + ti_threadgroup_barrier(tgworld, ti_tid); + return; + } + jl_in_gc = 1; uint64_t t0 = jl_hrtime(); int recollect = 0; @@ -2232,6 +2240,9 @@ void jl_gc_collect(int full) max_pause = max_pause < pause ? pause : max_pause; #endif jl_in_gc = 0; + + ti_threadgroup_barrier(tgworld, ti_tid); + JL_SIGATOMIC_END(); #ifdef GC_TIME if (estimate_freed != SAVE2) { @@ -2242,7 +2253,6 @@ void jl_gc_collect(int full) if (recollect) { n_pause--; jl_gc_collect(0); - ti_threadgroup_barrier(tgworld, ti_tid); } } From 0152ea1542130287d8a33f6834fe758b90e20e6e Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 2 Sep 2015 15:30:31 -0400 Subject: [PATCH 0025/1938] remove accidental double-init of jlexc_var --- src/codegen.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 9a5fd4599ce06..1cdbdb643be35 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4967,8 +4967,6 @@ static void init_julia_llvm_env(Module *m) jlfalse_var = global_to_llvm("jl_false", (void*)&jl_false, m); jlemptysvec_var = global_to_llvm("jl_emptysvec", (void*)&jl_emptysvec, m); jlemptytuple_var = global_to_llvm("jl_emptytuple", (void*)&jl_emptytuple, m); - jlexc_var = global_to_llvm("jl_exception_in_transit", - (void*)&jl_exception_in_transit, m); jldiverr_var = global_to_llvm("jl_diverror_exception", (void*)&jl_diverror_exception, m); jlundeferr_var = global_to_llvm("jl_undefref_exception", @@ -5594,7 +5592,7 @@ extern "C" void jl_init_codegen(void) #endif .setTargetOptions(options) .setRelocationModel(Reloc::PIC_) - .setCodeModel(CodeModel::Small) + .setCodeModel(CodeModel::JITDefault) #ifdef DISABLE_OPT .setOptLevel(CodeGenOpt::None) #else From a715b4e1d0f590b84cad7a2cc5ad029c3376b2b6 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 2 Sep 2015 15:30:53 -0400 Subject: [PATCH 0026/1938] fix gc-verify code to work with threading --- src/gc-debug.c | 59 +++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/src/gc-debug.c b/src/gc-debug.c index 0214bfc31915c..627567324b674 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -128,41 +128,42 @@ static void clear_mark(int bits) bits_save[i].len = 0; } } - void *current_heap = NULL; - bigval_t *bigs[2]; - bigs[0] = big_objects; - bigs[1] = big_objects_marked; - for (int i = 0; i < 2; i++) { - bigval_t *v = bigs[i]; - while (v != NULL) { - void* gcv = &v->header; - if (!verifying) arraylist_push(&bits_save[gc_bits(gcv)], gcv); - gc_bits(gcv) = bits; - v = v->next; + FOR_EACH_HEAP + bigval_t *bigs[2]; + bigs[0] = big_objects; + bigs[1] = big_objects_marked; + for (int i = 0; i < 2; i++) { + bigval_t *v = bigs[i]; + while (v != NULL) { + void* gcv = &v->header; + if (!verifying) arraylist_push(&bits_save[gc_bits(gcv)], gcv); + gc_bits(gcv) = bits; + v = v->next; + } } - } - for (int h = 0; h < REGION_COUNT; h++) { - region_t* region = regions[h]; - if (!region) break; - for (int pg_i = 0; pg_i < REGION_PG_COUNT/32; pg_i++) { - uint32_t line = region->freemap[pg_i]; - if (!!~line) { - for (int j = 0; j < 32; j++) { - if (!((line >> j) & 1)) { - gcpage_t *pg = page_metadata(®ion->pages[pg_i*32 + j][0] + GC_PAGE_OFFSET); - pool_t *pool = &norm_pools[pg->pool_n]; - pv = (gcval_t*)(pg->data + GC_PAGE_OFFSET); - char *lim = (char*)pv + GC_PAGE_SZ - GC_PAGE_OFFSET - pool->osize; - while ((char*)pv <= lim) { - if (!verifying) arraylist_push(&bits_save[gc_bits(pv)], pv); - gc_bits(pv) = bits; - pv = (gcval_t*)((char*)pv + pool->osize); + for (int h = 0; h < REGION_COUNT; h++) { + region_t* region = regions[h]; + if (!region) break; + for (int pg_i = 0; pg_i < REGION_PG_COUNT/32; pg_i++) { + uint32_t line = region->freemap[pg_i]; + if (!!~line) { + for (int j = 0; j < 32; j++) { + if (!((line >> j) & 1)) { + gcpage_t *pg = page_metadata(®ion->pages[pg_i*32 + j][0] + GC_PAGE_OFFSET); + pool_t *pool = &HEAP(norm_pools)[pg->pool_n]; + pv = (gcval_t*)(pg->data + GC_PAGE_OFFSET); + char *lim = (char*)pv + GC_PAGE_SZ - GC_PAGE_OFFSET - pool->osize; + while ((char*)pv <= lim) { + if (!verifying) arraylist_push(&bits_save[gc_bits(pv)], pv); + gc_bits(pv) = bits; + pv = (gcval_t*)((char*)pv + pool->osize); + } } } } } } - } + END } static void restore(void) From c0983e05a9c24bb9da5bec101961f4c4adad61a8 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 2 Sep 2015 16:29:22 -0400 Subject: [PATCH 0027/1938] more gc fixes and cleanup --- src/gc-debug.c | 36 ++++++++++++++++++------------------ src/gc.c | 13 ++++++------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/gc-debug.c b/src/gc-debug.c index 627567324b674..b78d58332991a 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -141,29 +141,29 @@ static void clear_mark(int bits) v = v->next; } } - for (int h = 0; h < REGION_COUNT; h++) { - region_t* region = regions[h]; - if (!region) break; - for (int pg_i = 0; pg_i < REGION_PG_COUNT/32; pg_i++) { - uint32_t line = region->freemap[pg_i]; - if (!!~line) { - for (int j = 0; j < 32; j++) { - if (!((line >> j) & 1)) { - gcpage_t *pg = page_metadata(®ion->pages[pg_i*32 + j][0] + GC_PAGE_OFFSET); - pool_t *pool = &HEAP(norm_pools)[pg->pool_n]; - pv = (gcval_t*)(pg->data + GC_PAGE_OFFSET); - char *lim = (char*)pv + GC_PAGE_SZ - GC_PAGE_OFFSET - pool->osize; - while ((char*)pv <= lim) { - if (!verifying) arraylist_push(&bits_save[gc_bits(pv)], pv); - gc_bits(pv) = bits; - pv = (gcval_t*)((char*)pv + pool->osize); - } + END + for (int h = 0; h < REGION_COUNT; h++) { + region_t* region = regions[h]; + if (!region) break; + for (int pg_i = 0; pg_i < REGION_PG_COUNT/32; pg_i++) { + uint32_t line = region->freemap[pg_i]; + if (!!~line) { + for (int j = 0; j < 32; j++) { + if (!((line >> j) & 1)) { + gcpage_t *pg = page_metadata(®ion->pages[pg_i*32 + j][0] + GC_PAGE_OFFSET); + pool_t *pool = &jl_all_heaps[pg->thread_n]->norm_pools[pg->pool_n]; + pv = (gcval_t*)(pg->data + GC_PAGE_OFFSET); + char *lim = (char*)pv + GC_PAGE_SZ - GC_PAGE_OFFSET - pool->osize; + while ((char*)pv <= lim) { + if (!verifying) arraylist_push(&bits_save[gc_bits(pv)], pv); + gc_bits(pv) = bits; + pv = (gcval_t*)((char*)pv + pool->osize); } } } } } - END + } } static void restore(void) diff --git a/src/gc.c b/src/gc.c index c19d2bfd9b23a..bee94a02c4e98 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1172,7 +1172,7 @@ static int freed_pages = 0; static int lazy_freed_pages = 0; static int page_done = 0; static gcval_t** sweep_page(pool_t* p, gcpage_t* pg, gcval_t **pfl,int,int); -static void sweep_pool_region(gcval_t **pfl[jl_n_threads][N_POOLS], int region_i, int sweep_mask) +static void sweep_pool_region(gcval_t ***pfl, int region_i, int sweep_mask) { region_t* region = regions[region_i]; @@ -1188,9 +1188,9 @@ static void sweep_pool_region(gcval_t **pfl[jl_n_threads][N_POOLS], int region_i gcpage_t *pg = ®ion->meta[pg_i*32 + j]; int p_n = pg->pool_n; int t_n = pg->thread_n; - pool_t *p = &jl_thread_heap[t_n].norm_pools[p_n]; + pool_t *p = &jl_all_heaps[t_n]->norm_pools[p_n]; int osize = pg->osize; - pfl[t_n][p_n] = sweep_page(p, pg, pfl[t_n][p_n], sweep_mask, osize); + pfl[t_n * N_POOLS + p_n] = sweep_page(p, pg, pfl[t_n * N_POOLS + p_n], sweep_mask, osize); } } } @@ -1370,7 +1370,7 @@ static int gc_sweep_inc(int sweep_mask) page_done = 0; int finished = 1; - gcval_t **pfl[jl_n_threads][N_POOLS]; + gcval_t **pfl[jl_n_threads * N_POOLS]; // update metadata of pages that were pointed to by freelist or newpages from a pool // i.e. pages being the current allocation target @@ -1384,7 +1384,7 @@ static int gc_sweep_inc(int sweep_mask) pg->nfree = p->nfree; } p->freelist = NULL; - pfl[current_heap_index][i] = &p->freelist; + pfl[current_heap_index * N_POOLS + i] = &p->freelist; last = p->newpages; if (last) { @@ -1406,7 +1406,7 @@ static int gc_sweep_inc(int sweep_mask) FOR_EACH_HEAP for (int i = 0; i < N_POOLS; i++) { pool_t* p = &HEAP(norm_pools)[i]; - *pfl[current_heap_index][i] = NULL; + *pfl[current_heap_index * N_POOLS + i] = NULL; if (p->freelist) { p->nfree = page_metadata(p->freelist)->nfree; } @@ -2418,7 +2418,6 @@ struct _jl_thread_heap_t *jl_mk_thread_heap(void) { void jl_gc_init(void) { gc_debug_init(); - jl_mk_thread_heap(); arraylist_new(&finalizer_list, 0); arraylist_new(&finalizer_list_marked, 0); From 4df1fe2b9513b4d636fb2fcc234adae68a71ca5c Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 2 Sep 2015 17:02:20 -0400 Subject: [PATCH 0028/1938] some 0.4 updates to threads code; get test-threads passing again --- base/threading.jl | 7 +++---- src/options.h | 6 +----- src/threading.c | 41 ++++++++++++++--------------------------- test/threads.jl | 2 +- 4 files changed, 19 insertions(+), 37 deletions(-) diff --git a/base/threading.jl b/base/threading.jl index 5af33a846deeb..9d391c76c1e6b 100644 --- a/base/threading.jl +++ b/base/threading.jl @@ -2,9 +2,9 @@ module Threading export threadid, maxthreads, nthreads, @threads -threadid() = int(ccall(:jl_threadid, Int16, ())+1) -maxthreads() = int(unsafe_load(cglobal(:jl_max_threads, Cint))) -nthreads() = int(unsafe_load(cglobal(:jl_n_threads, Cint))) +threadid() = Int(ccall(:jl_threadid, Int16, ())+1) +maxthreads() = Int(unsafe_load(cglobal(:jl_max_threads, Cint))) +nthreads() = Int(unsafe_load(cglobal(:jl_n_threads, Cint))) function _threadsfor(forexpr) fun = gensym("_threadsfor") @@ -88,4 +88,3 @@ macro threads(args...) end end # module - diff --git a/src/options.h b/src/options.h index e2644dcac9795..d3893ab199798 100644 --- a/src/options.h +++ b/src/options.h @@ -99,16 +99,12 @@ // defaults for # threads #define NUM_THREADS_NAME "JULIA_NUM_THREADS" -// FIXME - change back to 8 after debugging -#define DEFAULT_NUM_THREADS 1 +#define DEFAULT_NUM_THREADS 8 // affinitization behavior #define MACHINE_EXCLUSIVE_NAME "JULIA_EXCLUSIVE" #define DEFAULT_MACHINE_EXCLUSIVE 0 -// number of memory pools for lock free pool_alloc -#define N_GC_THREADS 16 - // sanitizer defaults --------------------------------------------------------- // Automatically enable MEMDEBUG and KEEP_BODIES for the sanitizers diff --git a/src/threading.c b/src/threading.c index ea4c279a5dc84..648f17865e3d2 100644 --- a/src/threading.c +++ b/src/threading.c @@ -32,7 +32,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. . invoke Julia function from multiple threads */ - #include #include #include @@ -48,7 +47,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "threadgroup.h" #include "threading.h" - /* TODO: . ugly mixture of uv_thread* and pthread*; fix with patch to libuv? . fix interface to properly support thread groups @@ -85,7 +83,6 @@ uint64_t *user_ticks; uint64_t *join_ticks; #endif - // create a thread and affinitize it if proc_num is specified int ti_threadcreate(pthread_t *pthread_id, int proc_num, void *(*thread_fun)(void *), void *thread_arg) @@ -106,7 +103,6 @@ int ti_threadcreate(pthread_t *pthread_id, int proc_num, return pthread_create(pthread_id, &attr, thread_fun, thread_arg); } - // set thread affinity void ti_threadsetaffinity(uint64_t pthread_id, int proc_num) { @@ -119,7 +115,6 @@ void ti_threadsetaffinity(uint64_t pthread_id, int proc_num) #endif } - struct _jl_thread_heap_t *jl_mk_thread_heap(void); // must be called by each thread at startup @@ -136,7 +131,6 @@ void ti_initthread(int16_t tid) jl_all_task_states[tid].ptask_arg_in_transit = &jl_task_arg_in_transit; } - // all threads call this function to run user code static jl_value_t *ti_run_fun(jl_function_t *f, jl_svec_t *args) { @@ -149,7 +143,6 @@ static jl_value_t *ti_run_fun(jl_function_t *f, jl_svec_t *args) return jl_nothing; } - // thread function: used by all except the main thread void *ti_threadfun(void *arg) { @@ -217,7 +210,6 @@ void *ti_threadfun(void *arg) return NULL; } - #if PROFILE_JL_THREADING void ti_reset_timings(); #endif @@ -304,7 +296,6 @@ void jl_start_threads(void) free(targs); } - // TODO: is this needed? where/when/how to call it? void jl_shutdown_threading(void) { @@ -329,27 +320,14 @@ void jl_shutdown_threading(void) #endif } - // return calling thread's ID -int16_t jl_threadid() -{ - return ti_tid; -} - +int16_t jl_threadid() { return ti_tid; } // return thread's thread group -void *jl_threadgroup() -{ - return (void *)tgworld; -} - +void *jl_threadgroup() { return (void *)tgworld; } // utility -void jl_cpu_pause() -{ - cpu_pause(); -} - +void jl_cpu_pause() { cpu_pause(); } // interface to user code: specialize and compile the user thread function // and run it in all threads @@ -361,7 +339,17 @@ DLLEXPORT jl_value_t *jl_threading_run(jl_function_t *f, jl_svec_t *args) jl_tupletype_t *argtypes = NULL; jl_function_t *fun = NULL; - JL_GC_PUSH2(&argtypes, &fun); + jl_svec_t *argsvec = NULL; + JL_GC_PUSH3(&argtypes, &fun, &argsvec); + if (jl_is_tuple(args)) { + size_t na = jl_nfields(args); + argsvec = jl_alloc_svec(na); + size_t i; + for(i=0; i < na; i++) { + jl_svecset(argsvec, i, jl_get_nth_field((jl_value_t*)args, i)); + } + args = argsvec; + } argtypes = arg_type_tuple(jl_svec_data(args), jl_svec_len(args)); fun = jl_get_specialization(f, argtypes); if (fun == NULL) @@ -409,7 +397,6 @@ DLLEXPORT jl_value_t *jl_threading_run(jl_function_t *f, jl_svec_t *args) return tw->ret; } - #if PROFILE_JL_THREADING void ti_reset_timings() diff --git a/test/threads.jl b/test/threads.jl index b924a0da32fcd..e05145c261f32 100644 --- a/test/threads.jl +++ b/test/threads.jl @@ -3,7 +3,7 @@ using Base.Threading println("Threading tests") -expected = [1:nthreads()] +expected = [1:nthreads();] # test 1 arr = zeros(Int16, nthreads()) From 3fd7acad6011dd1b05414a2bd1ee42b4f9747a32 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 2 Sep 2015 17:10:01 -0400 Subject: [PATCH 0029/1938] fix another translation error in the gc-verify code --- src/gc-debug.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/gc-debug.c b/src/gc-debug.c index b78d58332991a..9357d904d5ab9 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -128,20 +128,25 @@ static void clear_mark(int bits) bits_save[i].len = 0; } } + bigval_t *v; FOR_EACH_HEAP - bigval_t *bigs[2]; - bigs[0] = big_objects; - bigs[1] = big_objects_marked; - for (int i = 0; i < 2; i++) { - bigval_t *v = bigs[i]; - while (v != NULL) { - void* gcv = &v->header; - if (!verifying) arraylist_push(&bits_save[gc_bits(gcv)], gcv); - gc_bits(gcv) = bits; - v = v->next; - } + v = big_objects; + while (v != NULL) { + void* gcv = &v->header; + if (!verifying) arraylist_push(&bits_save[gc_bits(gcv)], gcv); + gc_bits(gcv) = bits; + v = v->next; } END + + v = big_objects_marked; + while (v != NULL) { + void* gcv = &v->header; + if (!verifying) arraylist_push(&bits_save[gc_bits(gcv)], gcv); + gc_bits(gcv) = bits; + v = v->next; + } + for (int h = 0; h < REGION_COUNT; h++) { region_t* region = regions[h]; if (!region) break; From 5a564ff161a4fd24c95b67b17ab743509a63216f Mon Sep 17 00:00:00 2001 From: Christoph Schwarzbach Date: Wed, 2 Sep 2015 17:28:46 -0700 Subject: [PATCH 0030/1938] getindex(A::SparseMatrixCSC, I::AbstractVector, J::AbstractVector): modified heuristic for skinny matrices to reduce memory footprint --- base/sparse/sparsematrix.jl | 8 +++--- test/perf/sparse/getindex_skinny.jl | 41 +++++++++++++++++++++++++++++ test/perf/sparse/perf.jl | 3 +++ 3 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 test/perf/sparse/getindex_skinny.jl diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 80e30d9ae7f52..3ddaa4235e19f 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1375,10 +1375,12 @@ function getindex_I_sorted{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, I::AbstractVector, # Similar to getindex_general but without the transpose trick. (m, n) = size(A) - nI = length(I) - avgM = div(nnz(A),n) + nI = length(I) + nzA = nnz(A) + avgM = div(nzA,n) # heuristics based on experiments - alg = ((nI - avgM) > 2^8) ? 1 : + alg = ((m > nzA) && (m > nI)) ? 0 : + ((nI - avgM) > 2^8) ? 1 : ((avgM - nI) > 2^10) ? 0 : 2 (alg == 0) ? getindex_I_sorted_bsearch_A(A, I, J) : diff --git a/test/perf/sparse/getindex_skinny.jl b/test/perf/sparse/getindex_skinny.jl new file mode 100644 index 0000000000000..27059989f2263 --- /dev/null +++ b/test/perf/sparse/getindex_skinny.jl @@ -0,0 +1,41 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + +# Test getindex for skinny sparse matrix +function sparse_getindex_skinny_perf() + + seed = 1 + srand(seed) + + # matsize = (size(A,1), nnz(A)) + matsize = [ + (2^12,1 ), + (2^12,2^6), + (2^18,1 ), + (2^18,2^9)] + # indsize = (length(I), number of repetitions) + indsize = [ + (1, 2^12), + (2^8, 2^8 ), + (2^16,2^4 )] + + c = 0 # counter + for (m,nz) in matsize + A = sprand(m,1,nz/m) + for (n,p) in indsize + c += 1 + I = rand(1:m,n) + @timeit indexing(A,I,p) "sparse_getindex_skinny$c" "" + end + end + +end + +function indexing(A,I,p) + + J = [1] + for k = 1:p + A[I,J] + end + nothing + +end diff --git a/test/perf/sparse/perf.jl b/test/perf/sparse/perf.jl index 171059385749c..5de1774ad565a 100644 --- a/test/perf/sparse/perf.jl +++ b/test/perf/sparse/perf.jl @@ -6,5 +6,8 @@ include("../perfutil.jl") include("getindex.jl") sparse_getindex_perf() +include("getindex_skinny.jl") +sparse_getindex_skinny_perf() + include("fem.jl") fem_perf() From 2fc8d97af4f6cb42a50aed0608f49fdbcbfd9e41 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 2 Sep 2015 22:26:13 -0400 Subject: [PATCH 0031/1938] syncronize contentious gc state --- src/gc.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/gc.c b/src/gc.c index bee94a02c4e98..83efcfd9405fb 100644 --- a/src/gc.c +++ b/src/gc.c @@ -33,6 +33,8 @@ extern "C" { #endif +JL_DEFINE_MUTEX(gc) + // manipulating mark bits #define GC_CLEAN 0 // freshly allocated @@ -662,6 +664,7 @@ static NOINLINE void *malloc_page(void) int i; region_t* region; int region_i = 0; + JL_LOCK(gc); while(region_i < REGION_COUNT) { region = regions[region_i]; if (region == NULL) { @@ -729,6 +732,7 @@ static NOINLINE void *malloc_page(void) #endif current_pg_count++; max_pg_count = max_pg_count < current_pg_count ? current_pg_count : max_pg_count; + JL_UNLOCK(gc); return ptr; } From 67a93b963968a24595132bcd9238f471c3f2c784 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 3 Sep 2015 11:39:29 -0400 Subject: [PATCH 0032/1938] synchronize type caches rename gc lock to pagealloc, since that's all it actually protects --- src/gc.c | 6 +++--- src/jltypes.c | 18 +++++++++++++----- src/threading.c | 1 + 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/gc.c b/src/gc.c index 83efcfd9405fb..a6b73eded6911 100644 --- a/src/gc.c +++ b/src/gc.c @@ -33,7 +33,7 @@ extern "C" { #endif -JL_DEFINE_MUTEX(gc) +JL_DEFINE_MUTEX(pagealloc) // manipulating mark bits @@ -664,7 +664,7 @@ static NOINLINE void *malloc_page(void) int i; region_t* region; int region_i = 0; - JL_LOCK(gc); + JL_LOCK(pagealloc); while(region_i < REGION_COUNT) { region = regions[region_i]; if (region == NULL) { @@ -732,7 +732,7 @@ static NOINLINE void *malloc_page(void) #endif current_pg_count++; max_pg_count = max_pg_count < current_pg_count ? current_pg_count : max_pg_count; - JL_UNLOCK(gc); + JL_UNLOCK(pagealloc); return ptr; } diff --git a/src/jltypes.c b/src/jltypes.c index 736d721eae7c6..bcc65336bc932 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -1848,6 +1848,8 @@ static int typekey_eq(jl_datatype_t *tt, jl_value_t **key, size_t n) return 1; } +JL_DEFINE_MUTEX_EXT(typecache); + // look up a type in a cache by binary or linear search. // if found, returns the index of the found item. if not found, returns // ~n, where n is the index where the type should be inserted. @@ -1890,11 +1892,14 @@ static ssize_t lookup_type_idx(jl_typename_t *tn, jl_value_t **key, size_t n, in static jl_value_t *lookup_type(jl_typename_t *tn, jl_value_t **key, size_t n) { int ord = is_typekey_ordered(key, n); + JL_LOCK(typecache); ssize_t idx = lookup_type_idx(tn, key, n, ord); - return (idx < 0) ? NULL : jl_svecref(ord ? tn->cache : tn->linearcache, idx); + jl_value_t *t = (idx < 0) ? NULL : jl_svecref(ord ? tn->cache : tn->linearcache, idx); + JL_UNLOCK(typecache); + return t; } -static int t_uid_ctr = 1; +static volatile int t_uid_ctr = 1; int jl_get_t_uid_ctr(void) { return t_uid_ctr; } void jl_set_t_uid_ctr(int i) { t_uid_ctr=i; } @@ -1902,7 +1907,7 @@ void jl_set_t_uid_ctr(int i) { t_uid_ctr=i; } int jl_assign_type_uid(void) { assert(t_uid_ctr != 0); - return t_uid_ctr++; + return JL_ATOMIC_FETCH_AND_ADD(t_uid_ctr, 1); } static int is_cacheable(jl_datatype_t *type) @@ -1971,11 +1976,14 @@ jl_value_t *jl_cache_type_(jl_datatype_t *type) { if (is_cacheable(type)) { int ord = is_typekey_ordered(jl_svec_data(type->parameters), jl_svec_len(type->parameters)); + JL_LOCK(typecache); ssize_t idx = lookup_type_idx(type->name, type->parameters->data, jl_svec_len(type->parameters), ord); if (idx >= 0) - return jl_svecref(ord ? type->name->cache : type->name->linearcache, idx); - cache_insert_type((jl_value_t*)type, ~idx, ord); + type = (jl_datatype_t*)jl_svecref(ord ? type->name->cache : type->name->linearcache, idx); + else + cache_insert_type((jl_value_t*)type, ~idx, ord); + JL_UNLOCK(typecache); } return (jl_value_t*)type; } diff --git a/src/threading.c b/src/threading.c index 648f17865e3d2..d8593cdf1db19 100644 --- a/src/threading.c +++ b/src/threading.c @@ -57,6 +57,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // lock for code generation JL_DEFINE_MUTEX(codegen); +JL_DEFINE_MUTEX(typecache); // thread ID JL_THREAD int16_t ti_tid = 0; From b29089c5a577d5f279a5a36d202f30002de77469 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 3 Sep 2015 12:57:41 -0400 Subject: [PATCH 0033/1938] synchronize add_finalizer. fixes #10718 --- src/gc.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/gc.c b/src/gc.c index a6b73eded6911..f2df0bd5084c1 100644 --- a/src/gc.c +++ b/src/gc.c @@ -34,6 +34,7 @@ extern "C" { #endif JL_DEFINE_MUTEX(pagealloc) +JL_DEFINE_MUTEX(finalizers) // manipulating mark bits @@ -379,6 +380,8 @@ static void run_finalizers(void) void jl_gc_inhibit_finalizers(int state) { + // NOTE: currently only called with the codegen lock held, but might need + // more synchronization in the future if (jl_gc_finalizers_inhibited && !state && !jl_in_gc) { jl_in_gc = 1; run_finalizers(); @@ -408,13 +411,17 @@ void jl_gc_run_all_finalizers(void) DLLEXPORT void jl_gc_add_finalizer(jl_value_t *v, jl_function_t *f) { + JL_LOCK(finalizers); arraylist_push(&finalizer_list, (void*)v); arraylist_push(&finalizer_list, (void*)f); + JL_UNLOCK(finalizers); } void jl_finalize(jl_value_t *o) { + JL_LOCK(finalizers); (void)finalize_object(o); + JL_UNLOCK(finalizers); } static region_t *find_region(void *ptr, int maybe) From 95f22cffcd09d102a64444b34c8612fbfeac93d1 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 3 Sep 2015 16:11:02 -0400 Subject: [PATCH 0034/1938] add JULIA_ENABLE_THREADING guard block around all threading-specific code --- src/codegen.cpp | 11 ++-- src/gc-debug.c | 5 +- src/gc.c | 122 +++++++++++++++++++++++---------------- src/julia.h | 39 +++++-------- src/options.h | 4 -- src/threadgroup.c | 69 ++++++++++++----------- src/threadgroup.h | 11 ++-- src/threading.c | 141 ++++++++++++++++++++++++++-------------------- src/threading.h | 23 ++++---- 9 files changed, 233 insertions(+), 192 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 1cdbdb643be35..fddbf5ef42809 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4941,18 +4941,21 @@ static void init_julia_llvm_env(Module *m) "jl_array_t"); jl_parray_llvmt = PointerType::get(jl_array_llvmt,0); +#ifdef JULIA_ENABLE_THREADING +#define JL_THREAD_MODEL ,GlobalValue::GeneralDynamicTLSModel +#else +#define JL_THREAD_MODEL +#endif jlpgcstack_var = new GlobalVariable(*m, jl_ppvalue_llvmt, false, GlobalVariable::ExternalLinkage, - NULL, "jl_pgcstack", NULL, - GlobalValue::GeneralDynamicTLSModel); + NULL, "jl_pgcstack", NULL JL_THREAD_MODEL); add_named_global(jlpgcstack_var, jl_dlsym(jl_dl_handle, "jl_pgcstack")); jlexc_var = new GlobalVariable(*m, jl_pvalue_llvmt, false, GlobalVariable::ExternalLinkage, - NULL, "jl_exception_in_transit", NULL, - GlobalValue::GeneralDynamicTLSModel); + NULL, "jl_exception_in_transit", NULL JL_THREAD_MODEL); add_named_global(jlexc_var, jl_dlsym(jl_dl_handle, "jl_exception_in_transit")); global_to_llvm("__stack_chk_guard", (void*)&__stack_chk_guard, m); diff --git a/src/gc-debug.c b/src/gc-debug.c index 9357d904d5ab9..e948b7141d170 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -156,7 +156,10 @@ static void clear_mark(int bits) for (int j = 0; j < 32; j++) { if (!((line >> j) & 1)) { gcpage_t *pg = page_metadata(®ion->pages[pg_i*32 + j][0] + GC_PAGE_OFFSET); - pool_t *pool = &jl_all_heaps[pg->thread_n]->norm_pools[pg->pool_n]; + pool_t *pool; + FOR_HEAP(pg->thread_n) + pool = &pools[pg->pool_n]; + END pv = (gcval_t*)(pg->data + GC_PAGE_OFFSET); char *lim = (char*)pv + GC_PAGE_SZ - GC_PAGE_OFFSET - pool->osize; while ((char*)pv <= lim) { diff --git a/src/gc.c b/src/gc.c index f2df0bd5084c1..a9b0efecbec40 100644 --- a/src/gc.c +++ b/src/gc.c @@ -194,55 +194,57 @@ static int regions_lb[REGION_COUNT] = {0}; // an upper bound of the last non-free page static int regions_ub[REGION_COUNT] = {REGION_PG_COUNT/32-1}; -// Variables that become fields of a thread-local struct in the thread-safe -// version. - -#define HEAP_DECL - +// Variables that become fields of a thread-local struct in the thread-safe version. typedef struct _jl_thread_heap_t { + // variable for tracking preserved values. + arraylist_t preserved_values; -// variable for tracking preserved values. -HEAP_DECL arraylist_t preserved_values; + // variable for tracking weak references + arraylist_t weak_refs; -// variable for tracking weak references -HEAP_DECL arraylist_t weak_refs; + // variables for tracking malloc'd arrays + mallocarray_t *mallocarrays; + mallocarray_t *mafreelist; -// variables for tracking malloc'd arrays -HEAP_DECL mallocarray_t *mallocarrays; -HEAP_DECL mallocarray_t *mafreelist; + // variables for tracking big objects + bigval_t *big_objects; -// variables for tracking big objects -HEAP_DECL bigval_t *big_objects; + // variables for tracking "remembered set" + arraylist_t rem_bindings; + arraylist_t _remset[2]; // contains jl_value_t* + // lower bound of the number of pointers inside remembered values + int remset_nptr; + arraylist_t *remset; + arraylist_t *last_remset; -// variables for tracking "remembered set" -HEAP_DECL arraylist_t rem_bindings; -HEAP_DECL arraylist_t _remset[2]; // contains jl_value_t* -// lower bound of the number of pointers inside remembered values -HEAP_DECL int remset_nptr; -HEAP_DECL arraylist_t *remset; -HEAP_DECL arraylist_t *last_remset; - -// variables for allocating objects from pools + // variables for allocating objects from pools #ifdef _P64 #define N_POOLS 41 #else #define N_POOLS 43 #endif -HEAP_DECL pool_t norm_pools[N_POOLS]; - -// End of Variables that become fields of a thread-local struct in the thread-safe version. + pool_t norm_pools[N_POOLS]; } jl_thread_heap_t; static JL_THREAD jl_thread_heap_t *jl_thread_heap; // The following macros are used for accessing these variables. -// In the future multi-threaded version, they establish the desired thread context. +// In the multi-threaded version, they establish the desired thread context. // In the single-threaded version, they are essentially noops, but nonetheless // serve to check that the thread context macros are being used. +#ifdef JULIA_ENABLE_THREADING #define FOR_CURRENT_HEAP {jl_thread_heap_t *current_heap = jl_thread_heap; #define END } #define FOR_EACH_HEAP for( int current_heap_index=jl_n_threads; --current_heap_index>=0; ) { jl_thread_heap_t *current_heap = jl_all_heaps[current_heap_index]; -/*}*/ +#define FOR_HEAP(t_n) {jl_thread_heap_t *current_heap = jl_all_heaps[t_n]; +/*}}*/ +#else +#define FOR_CURRENT_HEAP {jl_thread_heap_t *current_heap = jl_thread_heap; +#define END } +#define FOR_EACH_HEAP FOR_CURRENT_HEAP int current_heap_index = 0; (void)current_heap_index; +#define FOR_HEAP(t_n) FOR_CURRENT_HEAP +#endif + #define HEAP(x) (current_heap->x) #define preserved_values HEAP(preserved_values) #define weak_refs HEAP(weak_refs) @@ -253,7 +255,7 @@ static JL_THREAD jl_thread_heap_t *jl_thread_heap; #define remset_nptr HEAP(remset_nptr) #define last_remset HEAP(last_remset) #define rem_bindings HEAP(rem_bindings) -#define pools (jl_thread_heap->norm_pools) +#define pools HEAP(norm_pools) // List of marked big objects. Not per-thread. Accessed only by master thread. static bigval_t *big_objects_marked = NULL; @@ -1033,7 +1035,9 @@ static inline gcval_t *reset_page(pool_t *p, gcpage_t *pg, gcval_t *fl) { pg->gc_bits = 0; pg->nfree = (GC_PAGE_SZ - GC_PAGE_OFFSET) / p->osize; - pg->pool_n = p - jl_all_heaps[pg->thread_n]->norm_pools; + FOR_HEAP(pg->thread_n) + pg->pool_n = p - pools; + END memset(page_age(pg), 0, LLT_ALIGN(GC_PAGE_SZ / p->osize, 8)); gcval_t *beg = (gcval_t*)(pg->data + GC_PAGE_OFFSET); gcval_t *end = (gcval_t*)((char*)beg + (pg->nfree - 1)*p->osize); @@ -1199,7 +1203,10 @@ static void sweep_pool_region(gcval_t ***pfl, int region_i, int sweep_mask) gcpage_t *pg = ®ion->meta[pg_i*32 + j]; int p_n = pg->pool_n; int t_n = pg->thread_n; - pool_t *p = &jl_all_heaps[t_n]->norm_pools[p_n]; + pool_t *p; + FOR_HEAP(t_n) + p = &pools[p_n]; + END int osize = pg->osize; pfl[t_n * N_POOLS + p_n] = sweep_page(p, pg, pfl[t_n * N_POOLS + p_n], sweep_mask, osize); } @@ -1387,7 +1394,7 @@ static int gc_sweep_inc(int sweep_mask) // i.e. pages being the current allocation target FOR_EACH_HEAP for (int i = 0; i < N_POOLS; i++) { - pool_t* p = &HEAP(norm_pools)[i]; + pool_t* p = &pools[i]; gcval_t* last = p->freelist; if (last) { gcpage_t* pg = page_metadata(last); @@ -1416,7 +1423,7 @@ static int gc_sweep_inc(int sweep_mask) // null out terminal pointers of free lists and cache back pg->nfree in the pool_t FOR_EACH_HEAP for (int i = 0; i < N_POOLS; i++) { - pool_t* p = &HEAP(norm_pools)[i]; + pool_t* p = &pools[i]; *pfl[current_heap_index * N_POOLS + i] = NULL; if (p->freelist) { p->nfree = page_metadata(p->freelist)->nfree; @@ -2023,12 +2030,13 @@ void jl_gc_collect(int full) gc_debug_print(); JL_SIGATOMIC_BEGIN(); +#ifdef JULIA_ENABLE_THREADING ti_threadgroup_barrier(tgworld, ti_tid); - if (ti_tid != 0) { ti_threadgroup_barrier(tgworld, ti_tid); return; } +#endif jl_in_gc = 1; uint64_t t0 = jl_hrtime(); @@ -2252,7 +2260,9 @@ void jl_gc_collect(int full) #endif jl_in_gc = 0; +#ifdef JULIA_ENABLE_THREADING ti_threadgroup_barrier(tgworld, ti_tid); +#endif JL_SIGATOMIC_END(); #ifdef GC_TIME @@ -2286,7 +2296,9 @@ void *allocb(size_t sz) b->pooled = 0; } else { - b = (buff_t*)pool_alloc(&pools[szclass(allocsz)]); + FOR_CURRENT_HEAP + b = (buff_t*)pool_alloc(&pools[szclass(allocsz)]); + END b->header = 0x4EADE800; b->pooled = 1; } @@ -2326,7 +2338,9 @@ DLLEXPORT jl_value_t *jl_gc_allocobj(size_t sz) return jl_valueof(alloc_big(allocsz)); #endif if (allocsz <= GC_MAX_SZCLASS + sizeof(buff_t)) - return jl_valueof(pool_alloc(&pools[szclass(allocsz)])); + FOR_CURRENT_HEAP + return jl_valueof(pool_alloc(&pools[szclass(allocsz)])); + END else return jl_valueof(alloc_big(allocsz)); } @@ -2337,7 +2351,9 @@ DLLEXPORT jl_value_t *jl_gc_alloc_0w(void) #ifdef MEMDEBUG return jl_valueof(alloc_big(sz)); #endif - return jl_valueof(_pool_alloc(&pools[szclass(sz)], sz)); + FOR_CURRENT_HEAP + return jl_valueof(_pool_alloc(&pools[szclass(sz)], sz)); + END } DLLEXPORT jl_value_t *jl_gc_alloc_1w(void) @@ -2346,7 +2362,9 @@ DLLEXPORT jl_value_t *jl_gc_alloc_1w(void) #ifdef MEMDEBUG return jl_valueof(alloc_big(sz)); #endif - return jl_valueof(_pool_alloc(&pools[szclass(sz)], sz)); + FOR_CURRENT_HEAP + return jl_valueof(_pool_alloc(&pools[szclass(sz)], sz)); + END } DLLEXPORT jl_value_t *jl_gc_alloc_2w(void) @@ -2355,7 +2373,9 @@ DLLEXPORT jl_value_t *jl_gc_alloc_2w(void) #ifdef MEMDEBUG return jl_valueof(alloc_big(sz)); #endif - return jl_valueof(_pool_alloc(&pools[szclass(sz)], sz)); + FOR_CURRENT_HEAP + return jl_valueof(_pool_alloc(&pools[szclass(sz)], sz)); + END } DLLEXPORT jl_value_t *jl_gc_alloc_3w(void) @@ -2364,7 +2384,9 @@ DLLEXPORT jl_value_t *jl_gc_alloc_3w(void) #ifdef MEMDEBUG return jl_valueof(alloc_big(sz)); #endif - return jl_valueof(_pool_alloc(&pools[szclass(sz)], sz)); + FOR_CURRENT_HEAP + return jl_valueof(_pool_alloc(&pools[szclass(sz)], sz)); + END } #ifdef GC_FINAL_STATS @@ -2397,12 +2419,13 @@ void jl_print_gc_stats(JL_STREAM *s) } #endif +#ifdef JULIA_ENABLE_THREADING // Per-thread initialization (when threading is fully implemented) struct _jl_thread_heap_t *jl_mk_thread_heap(void) { jl_thread_heap = malloc(sizeof(jl_thread_heap_t)); // FIXME - should be cache-aligned malloc FOR_CURRENT_HEAP const int* szc = sizeclasses; - pool_t *p = HEAP(norm_pools); + pool_t *p = pools; for(int i=0; i < N_POOLS; i++) { assert((szc[i] < 16 && szc[i] % sizeof(void*) == 0) || (szc[i] % 16 == 0)); @@ -2424,6 +2447,7 @@ struct _jl_thread_heap_t *jl_mk_thread_heap(void) { END return jl_thread_heap; } +#endif // System-wide initializations void jl_gc_init(void) @@ -2517,13 +2541,15 @@ static void all_pool_stats(void) int i; size_t nb=0, w, tw=0, no=0,tp=0, nold=0,noldbytes=0, b, np, nol; for(i=0; i < N_POOLS; i++) { - b = pool_stats(&norm_pools[i], &w, &np, &nol); - nb += b; - no += (b/norm_pools[i].osize); - tw += w; - tp += np; - nold += nol; - noldbytes += nol*norm_pools[i].osize; + FOR_EACH_HEAP + b = pool_stats(&pools[i], &w, &np, &nol); + nb += b; + no += (b/pools[i].osize); + tw += w; + tp += np; + nold += nol; + noldbytes += nol*pools[i].osize; + END } jl_printf(JL_STDOUT, "%d objects (%d%% old), %d kB (%d%% old) total allocated, %d total fragments (%d%% overhead), in %d pages\n", diff --git a/src/julia.h b/src/julia.h index 5e0dd0c30adaa..6a25993a3012e 100644 --- a/src/julia.h +++ b/src/julia.h @@ -59,10 +59,13 @@ extern "C" { // threading ------------------------------------------------------------------ -// WARNING: Threading support is incomplete. Changing the 1 to a 0 will break Julia. +// WARNING: Threading support is incomplete and experimental (and only works with llvm-svn) // Nonetheless, we define JL_THREAD and use it to give advanced notice to maintainers // of what eventual threading support will change. -#if 0 + +// #define JULIA_ENABLE_THREADING + +#ifndef JULIA_ENABLE_THREADING // Definition for compiling non-thread-safe Julia. # define JL_THREAD #elif !defined(_OS_WINDOWS_) @@ -100,30 +103,8 @@ DLLEXPORT void jl_threading_profile(); # error "No atomic operations supported." #endif -#if 0 -#define JL_DEFINE_MUTEX(m) \ - uv_mutex_t m ## _mutex; \ - uint64_t m ## _thread_id = -1; - -#define JL_DEFINE_MUTEX_EXT(m) \ - extern uv_mutex_t m ## _mutex; \ - extern uint64_t m ## _thread_id; - -#define JL_LOCK(m) \ - int m ## locked = 0; \ - if (m ## _thread_id != uv_thread_self()) { \ - uv_mutex_lock(&m ## _mutex); \ - m ## locked = 1; \ - m ## _thread_id = uv_thread_self(); \ - } - -#define JL_UNLOCK(m) \ - if (m ## locked) { \ - m ## _thread_id = -1; \ - m ## locked = 0; \ - uv_mutex_unlock(&m ## _mutex); \ - } -#else +#ifdef JULIA_ENABLE_THREADING +#define FORCE_ELF #define JL_DEFINE_MUTEX(m) \ uint64_t volatile m ## _mutex = 0; \ int32_t m ## _lock_count = 0; @@ -153,8 +134,14 @@ DLLEXPORT void jl_threading_profile(); if (m ## _lock_count == 0) \ JL_ATOMIC_COMPARE_AND_SWAP(m ## _mutex, uv_thread_self(), 0); \ } +#else +#define JL_DEFINE_MUTEX(m) +#define JL_DEFINE_MUTEX_EXT(m) +#define JL_LOCK(m) +#define JL_UNLOCK(m) #endif + // core data types ------------------------------------------------------------ #ifndef _COMPILER_MICROSOFT_ diff --git a/src/options.h b/src/options.h index d3893ab199798..421a9954c44f7 100644 --- a/src/options.h +++ b/src/options.h @@ -113,10 +113,6 @@ # define MEMDEBUG # define KEEP_BODIES # endif -// Memory sanitizer also needs thread-local storage -# if __has_feature(memory_sanitizer) -# define CODEGEN_TLS -# endif #endif #endif diff --git a/src/threadgroup.c b/src/threadgroup.c index a83b34eff83ec..e229fe4af5fe6 100644 --- a/src/threadgroup.c +++ b/src/threadgroup.c @@ -32,15 +32,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "options.h" - #include #include #include + +#ifdef __cplusplus +extern "C" { +#endif + +#include "options.h" #include "ia_misc.h" #include "threadgroup.h" - int ti_threadgroup_create(uint8_t num_sockets, uint8_t num_cores, uint8_t num_threads_per_core, ti_threadgroup_t **newtg) @@ -66,16 +69,16 @@ int ti_threadgroup_create(uint8_t num_sockets, uint8_t num_cores, tg->group_sense = 0; tg->forked = 0; - pthread_mutex_init(&tg->alarm_lock, NULL); - pthread_cond_init(&tg->alarm, NULL); + uv_mutex_init(&tg->alarm_lock); + uv_cond_init(&tg->alarm); tg->sleep_threshold = DEFAULT_THREAD_SLEEP_THRESHOLD; cp = getenv(THREAD_SLEEP_THRESHOLD_NAME); if (cp) { - if (!strncasecmp(cp, "infinite", 8)) - tg->sleep_threshold = 0; - else - tg->sleep_threshold = (uint64_t)strtol(cp, NULL, 10); + if (!strncasecmp(cp, "infinite", 8)) + tg->sleep_threshold = 0; + else + tg->sleep_threshold = (uint64_t)strtol(cp, NULL, 10); } *newtg = tg; @@ -151,36 +154,36 @@ int ti_threadgroup_fork(ti_threadgroup_t *tg, int16_t ext_tid, if (tg->tid_map[ext_tid] == 0) { tg->envelope = bcast_val ? *bcast_val : NULL; cpu_sfence(); - tg->forked = 1; + tg->forked = 1; tg->group_sense = tg->thread_sense[0]->sense; - // if it's possible that threads are sleeping, signal them - if (tg->sleep_threshold) { - pthread_mutex_lock(&tg->alarm_lock); - pthread_cond_broadcast(&tg->alarm); - pthread_mutex_unlock(&tg->alarm_lock); - } + // if it's possible that threads are sleeping, signal them + if (tg->sleep_threshold) { + uv_mutex_lock(&tg->alarm_lock); + uv_cond_broadcast(&tg->alarm); + uv_mutex_unlock(&tg->alarm_lock); + } } else { - // spin up to threshold cycles (count sheep), then sleep - uint64_t spin_cycles, spin_start = rdtsc(); + // spin up to threshold cycles (count sheep), then sleep + uint64_t spin_cycles, spin_start = rdtsc(); while (tg->group_sense != tg->thread_sense[tg->tid_map[ext_tid]]->sense) { - if (tg->sleep_threshold) { - spin_cycles = rdtsc() - spin_start; - if (spin_cycles >= tg->sleep_threshold) { - pthread_mutex_lock(&tg->alarm_lock); + if (tg->sleep_threshold) { + spin_cycles = rdtsc() - spin_start; + if (spin_cycles >= tg->sleep_threshold) { + uv_mutex_lock(&tg->alarm_lock); if (tg->group_sense != tg->thread_sense[tg->tid_map[ext_tid]]->sense) { - pthread_cond_wait(&tg->alarm, &tg->alarm_lock); + uv_cond_wait(&tg->alarm, &tg->alarm_lock); } - pthread_mutex_unlock(&tg->alarm_lock); - spin_start = rdtsc(); - continue; - } - } + uv_mutex_unlock(&tg->alarm_lock); + spin_start = rdtsc(); + continue; + } + } cpu_pause(); - } + } cpu_lfence(); if (bcast_val) *bcast_val = tg->envelope; @@ -201,7 +204,7 @@ int ti_threadgroup_join(ti_threadgroup_t *tg, int16_t ext_tid) while (tg->thread_sense[i]->sense == tg->group_sense) cpu_pause(); } - tg->forked = 0; + tg->forked = 0; } return 0; @@ -211,7 +214,7 @@ int ti_threadgroup_join(ti_threadgroup_t *tg, int16_t ext_tid) void ti_threadgroup_barrier(ti_threadgroup_t *tg, int16_t ext_tid) { if (tg->tid_map[ext_tid] == 0 && !tg->forked) - return; + return; ti_threadgroup_join(tg, ext_tid); ti_threadgroup_fork(tg, ext_tid, NULL); @@ -222,8 +225,8 @@ int ti_threadgroup_destroy(ti_threadgroup_t *tg) { int i; - pthread_mutex_destroy(&tg->alarm_lock); - pthread_cond_destroy(&tg->alarm); + uv_mutex_destroy(&tg->alarm_lock); + uv_cond_destroy(&tg->alarm); for (i = 0; i < tg->num_threads; i++) _mm_free(tg->thread_sense[i]); diff --git a/src/threadgroup.h b/src/threadgroup.h index 5c752455f3f70..61d492fd76114 100644 --- a/src/threadgroup.h +++ b/src/threadgroup.h @@ -29,8 +29,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define THREADGROUP_H #include -#include - +#include "uv.h" // for the barrier typedef struct { @@ -45,15 +44,15 @@ typedef struct { uint8_t num_sockets, num_cores, num_threads_per_core; // fork/join/barrier - uint8_t forked; + uint8_t forked; volatile uint8_t group_sense; ti_thread_sense_t **thread_sense; void *envelope; // to let threads sleep - pthread_mutex_t alarm_lock; - pthread_cond_t alarm; - uint64_t sleep_threshold; + uv_mutex_t alarm_lock; + uv_cond_t alarm; + uint64_t sleep_threshold; } ti_threadgroup_t; diff --git a/src/threading.c b/src/threading.c index d8593cdf1db19..052cf65e72c75 100644 --- a/src/threading.c +++ b/src/threading.c @@ -30,8 +30,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. . thread and threadgroup creation . thread function . invoke Julia function from multiple threads + +TODO: + . fix interface to properly support thread groups + . add queue per thread for tasks + . add reduction; reduce values returned from thread function + . make code generation thread-safe and remove the lock */ + #include #include #include @@ -41,31 +48,50 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "julia.h" #include "julia_internal.h" -#include "uv.h" + +#ifdef __cplusplus +extern "C" { +#endif #include "ia_misc.h" #include "threadgroup.h" #include "threading.h" -/* TODO: - . ugly mixture of uv_thread* and pthread*; fix with patch to libuv? - . fix interface to properly support thread groups - . add queue per thread for tasks - . add reduction; reduce values returned from thread function - . make code generation thread-safe and remove the lock -*/ +// thread ID +JL_THREAD int16_t ti_tid = 0; +DLLEXPORT int jl_n_threads; // # threads we're actually using +DLLEXPORT int jl_max_threads; // # threads possible +jl_thread_task_state_t *jl_all_task_states; + +// return calling thread's ID +DLLEXPORT int16_t jl_threadid() { return ti_tid; } + +struct _jl_thread_heap_t *jl_mk_thread_heap(void); +// must be called by each thread at startup +void ti_initthread(int16_t tid) +{ + ti_tid = tid; + jl_pgcstack = NULL; +#ifdef JULIA_ENABLE_THREADING + jl_all_pgcstacks[tid] = &jl_pgcstack; + jl_all_heaps[tid] = jl_mk_thread_heap(); +#endif + + jl_all_task_states[tid].pcurrent_task = &jl_current_task; + jl_all_task_states[tid].proot_task = &jl_root_task; + jl_all_task_states[tid].pexception_in_transit = &jl_exception_in_transit; + jl_all_task_states[tid].ptask_arg_in_transit = &jl_task_arg_in_transit; +} + +#ifdef JULIA_ENABLE_THREADING // lock for code generation JL_DEFINE_MUTEX(codegen); JL_DEFINE_MUTEX(typecache); -// thread ID -JL_THREAD int16_t ti_tid = 0; - // thread heap struct _jl_thread_heap_t **jl_all_heaps; jl_gcframe_t ***jl_all_pgcstacks; -jl_thread_task_state_t *jl_all_task_states; // only one thread group for now ti_threadgroup_t *tgworld; @@ -73,9 +99,6 @@ ti_threadgroup_t *tgworld; // for broadcasting work to threads ti_threadwork_t threadwork; -DLLEXPORT int jl_max_threads; // # threads possible -DLLEXPORT int jl_n_threads; // # threads we're actually using - #if PROFILE_JL_THREADING double cpu_ghz; uint64_t prep_ticks; @@ -85,53 +108,38 @@ uint64_t *join_ticks; #endif // create a thread and affinitize it if proc_num is specified -int ti_threadcreate(pthread_t *pthread_id, int proc_num, - void *(*thread_fun)(void *), void *thread_arg) +int ti_threadcreate(uv_thread_t *thread_id, int proc_num, + void (*thread_fun)(void *), void *thread_arg) { +#ifdef _OS_LINUX_ pthread_attr_t attr; pthread_attr_init(&attr); -#ifdef _OS_LINUX_ cpu_set_t cset; if (proc_num >= 0) { - CPU_ZERO(&cset); - CPU_SET(proc_num, &cset); - pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cset); + CPU_ZERO(&cset); + CPU_SET(proc_num, &cset); + pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cset); } -#endif - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - return pthread_create(pthread_id, &attr, thread_fun, thread_arg); + return pthread_create(thread_id, &attr, thread_fun, thread_arg); +#else + return uv_thread_create(thread_id, thread_fun, thread_arg); +#endif } // set thread affinity -void ti_threadsetaffinity(uint64_t pthread_id, int proc_num) +void ti_threadsetaffinity(uint64_t thread_id, int proc_num) { #ifdef _OS_LINUX_ cpu_set_t cset; CPU_ZERO(&cset); CPU_SET(proc_num, &cset); - pthread_setaffinity_np(pthread_id, sizeof(cpu_set_t), &cset); + pthread_setaffinity_np(thread_id, sizeof(cpu_set_t), &cset); #endif } -struct _jl_thread_heap_t *jl_mk_thread_heap(void); - -// must be called by each thread at startup -void ti_initthread(int16_t tid) -{ - ti_tid = tid; - jl_pgcstack = NULL; - jl_all_pgcstacks[tid] = &jl_pgcstack; - jl_all_heaps[tid] = jl_mk_thread_heap(); - - jl_all_task_states[tid].pcurrent_task = &jl_current_task; - jl_all_task_states[tid].proot_task = &jl_root_task; - jl_all_task_states[tid].pexception_in_transit = &jl_exception_in_transit; - jl_all_task_states[tid].ptask_arg_in_transit = &jl_task_arg_in_transit; -} - // all threads call this function to run user code static jl_value_t *ti_run_fun(jl_function_t *f, jl_svec_t *args) { @@ -145,7 +153,7 @@ static jl_value_t *ti_run_fun(jl_function_t *f, jl_svec_t *args) } // thread function: used by all except the main thread -void *ti_threadfun(void *arg) +void ti_threadfun(void *arg) { ti_threadarg_t *ta = (ti_threadarg_t *)arg; ti_threadgroup_t *tg; @@ -193,22 +201,20 @@ void *ti_threadfun(void *arg) } #if PROFILE_JL_THREADING - uint64_t tuser = rdtsc(); + uint64_t tuser = rdtsc(); user_ticks[ti_tid] += tuser - tfork; #endif ti_threadgroup_join(tg, ti_tid); #if PROFILE_JL_THREADING - uint64_t tjoin = rdtsc(); + uint64_t tjoin = rdtsc(); join_ticks[ti_tid] += tjoin - tuser; #endif // TODO: // nowait should skip the join, but confirm that fork is reentrant } - - return NULL; } #if PROFILE_JL_THREADING @@ -225,7 +231,7 @@ void jl_init_threading(void) jl_n_threads = DEFAULT_NUM_THREADS; cp = getenv(NUM_THREADS_NAME); if (cp) { - jl_n_threads = (uint64_t)strtol(cp, NULL, 10); + jl_n_threads = (uint64_t)strtol(cp, NULL, 10); } if (jl_n_threads > jl_max_threads) jl_n_threads = jl_max_threads; @@ -256,20 +262,20 @@ void jl_start_threads(void) { char *cp; int i, exclusive; - pthread_t ptid; + uv_thread_t ptid; ti_threadarg_t **targs; // do we have exclusive use of the machine? default is no exclusive = DEFAULT_MACHINE_EXCLUSIVE; cp = getenv(MACHINE_EXCLUSIVE_NAME); if (cp) - exclusive = strtol(cp, NULL, 10); + exclusive = strtol(cp, NULL, 10); // exclusive use: affinitize threads, master thread on proc 0, rest // according to a 'compact' policy // non-exclusive: no affinity settings; let the kernel move threads about if (exclusive) - ti_threadsetaffinity(uv_thread_self(), 0); + ti_threadsetaffinity(uv_thread_self(), 0); // create threads targs = malloc((jl_n_threads - 1) * sizeof (ti_threadarg_t *)); @@ -321,9 +327,6 @@ void jl_shutdown_threading(void) #endif } -// return calling thread's ID -int16_t jl_threadid() { return ti_tid; } - // return thread's thread group void *jl_threadgroup() { return (void *)tgworld; } @@ -405,7 +408,7 @@ void ti_reset_timings() int i; prep_ticks = 0; for (i = 0; i < jl_n_threads; i++) - fork_ticks[i] = user_ticks[i] = join_ticks[i] = 0; + fork_ticks[i] = user_ticks[i] = join_ticks[i] = 0; } void ti_timings(uint64_t *times, uint64_t *min, uint64_t *max, uint64_t *avg) @@ -430,24 +433,42 @@ void jl_threading_profile() if (!fork_ticks) return; printf("\nti profile:\n"); - printf("prep: %g (%lu)\n", TICKS_TO_SECS(prep_ticks), prep_ticks); + printf("prep: %g (%llu)\n", TICKS_TO_SECS(prep_ticks), (unsigned long long)prep_ticks); uint64_t min, max, avg; ti_timings(fork_ticks, &min, &max, &avg); printf("fork: %g (%g - %g)\n", TICKS_TO_SECS(min), TICKS_TO_SECS(max), - TICKS_TO_SECS(avg)); + TICKS_TO_SECS(avg)); ti_timings(user_ticks, &min, &max, &avg); printf("user: %g (%g - %g)\n", TICKS_TO_SECS(min), TICKS_TO_SECS(max), - TICKS_TO_SECS(avg)); + TICKS_TO_SECS(avg)); ti_timings(join_ticks, &min, &max, &avg); printf("join: %g (%g - %g)\n", TICKS_TO_SECS(min), TICKS_TO_SECS(max), - TICKS_TO_SECS(avg)); + TICKS_TO_SECS(avg)); } -#else +#else //!PROFILE_JL_THREADING void jl_threading_profile() { } +#endif //!PROFILE_JL_THREADING + +#else // !JULIA_ENABLE_THREADING + +void jl_init_threading(void) { + static jl_thread_task_state_t _jl_all_task_states; + jl_all_task_states = &_jl_all_task_states; + jl_max_threads = 1; + jl_n_threads = 1; + ti_initthread(0); +} + +void jl_start_threads(void) { } + +#endif // !JULIA_ENABLE_THREADING + +#ifdef __cplusplus +} #endif diff --git a/src/threading.h b/src/threading.h index 0f670b69af250..3851b0946f648 100644 --- a/src/threading.h +++ b/src/threading.h @@ -29,22 +29,25 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define THREADING_H #include -#include "julia.h" -#include +#ifdef __cplusplus +extern "C" { +#endif + #include "threadgroup.h" +#include "julia.h" #define PROFILE_JL_THREADING 1 - // thread ID extern JL_THREAD int16_t ti_tid; +extern jl_thread_task_state_t *jl_all_task_states; +extern DLLEXPORT int jl_n_threads; // # threads we're actually using +#ifdef JULIA_ENABLE_THREADING // GC extern struct _jl_thread_heap_t **jl_all_heaps; extern jl_gcframe_t ***jl_all_pgcstacks; -extern jl_thread_task_state_t *jl_all_task_states; - -extern DLLEXPORT int jl_n_threads; // # threads we're actually using +#endif // thread state enum { @@ -80,12 +83,12 @@ typedef struct { // basic functions for thread creation -int ti_threadcreate(pthread_t *pthread_id, int proc_num, - void *(*thread_fun)(void *), void *thread_arg); -void ti_threadsetaffinity(uint64_t pthread_id, int proc_num); +int ti_threadcreate(uv_thread_t *thread_id, int proc_num, + void (*thread_fun)(void *), void *thread_arg); +void ti_threadsetaffinity(uint64_t thread_id, int proc_num); // thread function -void *ti_threadfun(void *arg); +void ti_threadfun(void *arg); // helpers for thread function void ti_initthread(int16_t tid); From 8b694009cb34517d775c7f9fe2ef6667cb634ffd Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 3 Sep 2015 17:50:23 -0400 Subject: [PATCH 0035/1938] slightly reduce overhead in jl_threading_run --- base/threading.jl | 2 +- src/threading.c | 21 ++++++++------------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/base/threading.jl b/base/threading.jl index 9d391c76c1e6b..117e108056b52 100644 --- a/base/threading.jl +++ b/base/threading.jl @@ -59,7 +59,7 @@ end function _threadscall(callexpr) fun = callexpr.args[1] esc(quote - ccall(:jl_threading_run, Void, (Any, Any), $fun, $(Expr(:tuple, callexpr.args[2:end]...))) + ccall(:jl_threading_run, Void, (Any, Any), $fun, $(Expr(:call, Core.svec, callexpr.args[2:end]...))) end) end diff --git a/src/threading.c b/src/threading.c index 052cf65e72c75..8304087e738ce 100644 --- a/src/threading.c +++ b/src/threading.c @@ -343,22 +343,17 @@ DLLEXPORT jl_value_t *jl_threading_run(jl_function_t *f, jl_svec_t *args) jl_tupletype_t *argtypes = NULL; jl_function_t *fun = NULL; - jl_svec_t *argsvec = NULL; - JL_GC_PUSH3(&argtypes, &fun, &argsvec); - if (jl_is_tuple(args)) { - size_t na = jl_nfields(args); - argsvec = jl_alloc_svec(na); - size_t i; - for(i=0; i < na; i++) { - jl_svecset(argsvec, i, jl_get_nth_field((jl_value_t*)args, i)); - } - args = argsvec; - } - argtypes = arg_type_tuple(jl_svec_data(args), jl_svec_len(args)); + if ((jl_value_t*)args == jl_emptytuple) + args = jl_emptysvec; + assert(jl_is_svec(args)); + JL_GC_PUSH2(&argtypes, &fun); + if (jl_svec_len(args) == 0) + argtypes = (jl_tupletype_t*)jl_typeof(jl_emptytuple); + else + argtypes = arg_type_tuple(jl_svec_data(args), jl_svec_len(args)); fun = jl_get_specialization(f, argtypes); if (fun == NULL) fun = f; - jl_compile(fun); jl_generate_fptr(fun); threadwork.command = TI_THREADWORK_RUN; From cc71a4aa6b25023a32f99bde964b6d9b2d65e444 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 3 Sep 2015 16:49:26 -0400 Subject: [PATCH 0036/1938] fixup for jn/threading build --- src/gc.c | 9 +++++---- src/threading.c | 2 ++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/gc.c b/src/gc.c index a9b0efecbec40..3dd8e8dea2424 100644 --- a/src/gc.c +++ b/src/gc.c @@ -226,19 +226,20 @@ typedef struct _jl_thread_heap_t { pool_t norm_pools[N_POOLS]; } jl_thread_heap_t; -static JL_THREAD jl_thread_heap_t *jl_thread_heap; - // The following macros are used for accessing these variables. // In the multi-threaded version, they establish the desired thread context. // In the single-threaded version, they are essentially noops, but nonetheless // serve to check that the thread context macros are being used. #ifdef JULIA_ENABLE_THREADING +static JL_THREAD jl_thread_heap_t *jl_thread_heap; #define FOR_CURRENT_HEAP {jl_thread_heap_t *current_heap = jl_thread_heap; #define END } #define FOR_EACH_HEAP for( int current_heap_index=jl_n_threads; --current_heap_index>=0; ) { jl_thread_heap_t *current_heap = jl_all_heaps[current_heap_index]; #define FOR_HEAP(t_n) {jl_thread_heap_t *current_heap = jl_all_heaps[t_n]; /*}}*/ #else +static jl_thread_heap_t _jl_thread_heap; +static jl_thread_heap_t *const jl_thread_heap = &_jl_thread_heap; #define FOR_CURRENT_HEAP {jl_thread_heap_t *current_heap = jl_thread_heap; #define END } #define FOR_EACH_HEAP FOR_CURRENT_HEAP int current_heap_index = 0; (void)current_heap_index; @@ -2419,10 +2420,11 @@ void jl_print_gc_stats(JL_STREAM *s) } #endif -#ifdef JULIA_ENABLE_THREADING // Per-thread initialization (when threading is fully implemented) struct _jl_thread_heap_t *jl_mk_thread_heap(void) { +#ifdef JULIA_ENABLE_THREADING jl_thread_heap = malloc(sizeof(jl_thread_heap_t)); // FIXME - should be cache-aligned malloc +#endif FOR_CURRENT_HEAP const int* szc = sizeclasses; pool_t *p = pools; @@ -2447,7 +2449,6 @@ struct _jl_thread_heap_t *jl_mk_thread_heap(void) { END return jl_thread_heap; } -#endif // System-wide initializations void jl_gc_init(void) diff --git a/src/threading.c b/src/threading.c index 8304087e738ce..310dfec4a584e 100644 --- a/src/threading.c +++ b/src/threading.c @@ -75,6 +75,8 @@ void ti_initthread(int16_t tid) #ifdef JULIA_ENABLE_THREADING jl_all_pgcstacks[tid] = &jl_pgcstack; jl_all_heaps[tid] = jl_mk_thread_heap(); +#else + jl_mk_thread_heap(); #endif jl_all_task_states[tid].pcurrent_task = &jl_current_task; From 1a47fb1f8a5d363c03e4c5af8fdddc095c452d75 Mon Sep 17 00:00:00 2001 From: Shashi Gowda Date: Fri, 4 Sep 2015 03:20:43 +0530 Subject: [PATCH 0037/1938] add hard new lines to markdown parser --- base/markdown/Common/block.jl | 6 +++++- base/markdown/render/terminal/render.jl | 4 ++++ test/markdown.jl | 6 +++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/base/markdown/Common/block.jl b/base/markdown/Common/block.jl index 3053ddbac3d32..b5d715991e23c 100644 --- a/base/markdown/Common/block.jl +++ b/base/markdown/Common/block.jl @@ -15,11 +15,14 @@ function paragraph(stream::IO, md::MD) p = Paragraph() push!(md, p) skipwhitespace(stream) + prev_char = '\n' while !eof(stream) char = read(stream, Char) if char == '\n' || char == '\r' char == '\r' && peek(stream) == '\n' && read(stream, Char) - if blankline(stream) || parse(stream, md, breaking = true) + if prev_char == '\\' + write(buffer, '\n') + elseif blankline(stream) || parse(stream, md, breaking = true) break else write(buffer, ' ') @@ -27,6 +30,7 @@ function paragraph(stream::IO, md::MD) else write(buffer, char) end + prev_char = char end p.content = parseinline(seek(buffer, 0), md) return true diff --git a/base/markdown/render/terminal/render.jl b/base/markdown/render/terminal/render.jl index fd2c3b5d0e711..de76020494e05 100644 --- a/base/markdown/render/terminal/render.jl +++ b/base/markdown/render/terminal/render.jl @@ -104,6 +104,10 @@ function terminline(io::IO, md::Italic) with_output_format(:underline, terminline, io, md.text) end +function terminline(io::IO, md::LineBreak) + println(io) +end + function terminline(io::IO, md::Image) print(io, "(Image: $(md.alt))") end diff --git a/test/markdown.jl b/test/markdown.jl index 774083d9499ba..b02755dbc86b5 100644 --- a/test/markdown.jl +++ b/test/markdown.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license using Base.Markdown -import Base.Markdown: MD, Paragraph, Header, Italic, Bold, plain, term, html, Table, Code, LaTeX +import Base.Markdown: MD, Paragraph, Header, Italic, Bold, LineBreak, plain, term, html, Table, Code, LaTeX import Base: writemime # Basics @@ -10,6 +10,10 @@ import Base: writemime @test md"foo" == MD(Paragraph("foo")) @test md"foo *bar* baz" == MD(Paragraph(["foo ", Italic("bar"), " baz"])) +@test md"""foo +bar""" == MD(Paragraph(["foo bar"])) +@test md"""foo\ +bar""" == MD(Paragraph(["foo", LineBreak(), "bar"])) @test md"#no title" == MD(Paragraph(["#no title"])) @test md"# title" == MD(Header{1}("title")) From 707e81feea42f1809597ab49000762301ae09a87 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 3 Sep 2015 18:25:51 -0400 Subject: [PATCH 0038/1938] fix a `return` that needs to be `continue` with the function body wrapped in FOR_EACH_HEAP --- src/gc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gc.c b/src/gc.c index 3dd8e8dea2424..ad750bf458eb6 100644 --- a/src/gc.c +++ b/src/gc.c @@ -825,7 +825,7 @@ DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value) { jl_weakref_t *wr = (jl_weakref_t*)jl_gc_alloc_1w(); jl_set_typeof(wr, jl_weakref_type); - wr->value = value; + wr->value = value; // NOTE: wb not needed here FOR_CURRENT_HEAP arraylist_push(&weak_refs, wr); END @@ -841,7 +841,7 @@ static void sweep_weak_refs(void) void *tmp; #define SWAP_wr(a,b) (tmp=a,a=b,b=tmp,1) if (l == 0) - return; + continue; do { wr = (jl_weakref_t*)lst[n]; if (gc_marked(jl_astaggedvalue(wr))) { From 87f9a9c8cdabcb4334d57c2cc8ff85fe0231dcff Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 3 Sep 2015 18:55:50 -0400 Subject: [PATCH 0039/1938] fix build of gc.c when threads are disabled --- src/gc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gc.c b/src/gc.c index ad750bf458eb6..be6a70233635b 100644 --- a/src/gc.c +++ b/src/gc.c @@ -242,7 +242,7 @@ static jl_thread_heap_t _jl_thread_heap; static jl_thread_heap_t *const jl_thread_heap = &_jl_thread_heap; #define FOR_CURRENT_HEAP {jl_thread_heap_t *current_heap = jl_thread_heap; #define END } -#define FOR_EACH_HEAP FOR_CURRENT_HEAP int current_heap_index = 0; (void)current_heap_index; +#define FOR_EACH_HEAP for( int current_heap_index=1; --current_heap_index>=0; ) { jl_thread_heap_t *current_heap = jl_thread_heap; (void)current_heap_index; #define FOR_HEAP(t_n) FOR_CURRENT_HEAP #endif From cf70bc9441c3b138fc8ff271c1d74fc86edad43c Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 3 Sep 2015 20:16:34 -0400 Subject: [PATCH 0040/1938] mark all thread stacks in GC --- src/gc.c | 24 +++++++++++++----------- src/julia.h | 4 ++++ src/task.c | 2 ++ src/threading.c | 8 +++++--- src/threading.h | 2 +- 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/gc.c b/src/gc.c index be6a70233635b..c5b4b4d7da162 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1604,10 +1604,10 @@ static void gc_mark_task_stack(jl_task_t *ta, int d) } #ifdef COPY_STACKS ptrint_t offset; - if (ta == jl_current_task) { + if (ta == *jl_all_task_states[ta->tid].pcurrent_task) { offset = 0; // FIXME - do we need to mark stacks on other threads? - gc_mark_stack((jl_value_t*)ta, jl_pgcstack, offset, d); + gc_mark_stack((jl_value_t*)ta, *jl_all_pgcstacks[ta->tid], offset, d); } else { offset = (char *)ta->stkbuf - ((char *)jl_stackbase - ta->ssize); @@ -1862,19 +1862,22 @@ static void pre_mark(void) gc_push_root(jl_current_module, 0); if (jl_old_base_module) gc_push_root(jl_old_base_module, 0); gc_push_root(jl_internal_main_module, 0); - gc_push_root(jl_root_task, 0); - gc_push_root(jl_current_task, 0); + + size_t i; + for(i=0; i < jl_n_threads; i++) { + jl_thread_task_state_t *ts = &jl_all_task_states[i]; + gc_push_root(*ts->pcurrent_task, 0); + gc_push_root(*ts->proot_task, 0); + gc_push_root(*ts->pexception_in_transit, 0); + gc_push_root(*ts->ptask_arg_in_transit, 0); + } // invisible builtin values if (jl_an_empty_cell) gc_push_root(jl_an_empty_cell, 0); - gc_push_root(jl_exception_in_transit, 0); - gc_push_root(jl_task_arg_in_transit, 0); gc_push_root(typeToTypeId, 0); if (jl_module_init_order != NULL) gc_push_root(jl_module_init_order, 0); - size_t i; - // stuff randomly preserved FOR_EACH_HEAP for(i=0; i < preserved_values.len; i++) { @@ -2017,8 +2020,6 @@ static int saved_mark_sp = 0; static int sweep_mask = GC_MARKED; #define MIN_SCAN_BYTES 1024*1024 -static void gc_mark_task_stack(jl_task_t*,int); - void prepare_sweep(void) { } @@ -2421,7 +2422,8 @@ void jl_print_gc_stats(JL_STREAM *s) #endif // Per-thread initialization (when threading is fully implemented) -struct _jl_thread_heap_t *jl_mk_thread_heap(void) { +struct _jl_thread_heap_t *jl_mk_thread_heap(void) +{ #ifdef JULIA_ENABLE_THREADING jl_thread_heap = malloc(sizeof(jl_thread_heap_t)); // FIXME - should be cache-aligned malloc #endif diff --git a/src/julia.h b/src/julia.h index 6a25993a3012e..a905f01305d20 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1404,6 +1404,10 @@ typedef struct _jl_task_t { jl_gcframe_t *gcstack; // current module, or NULL if this task has not set one jl_module_t *current_module; + + // id of owning thread + // does not need to be defined until the task runs + int16_t tid; } jl_task_t; typedef struct { diff --git a/src/task.c b/src/task.c index fdc59242bcf51..58d0000504dd7 100644 --- a/src/task.c +++ b/src/task.c @@ -857,6 +857,7 @@ DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize) t->eh = NULL; t->gcstack = NULL; t->stkbuf = NULL; + t->tid = 0; #ifdef COPY_STACKS t->bufsz = 0; @@ -960,6 +961,7 @@ void jl_init_root_task(void *stack, size_t ssize) jl_current_task->backtrace = jl_nothing; jl_current_task->eh = NULL; jl_current_task->gcstack = NULL; + jl_current_task->tid = ti_tid; jl_root_task = jl_current_task; diff --git a/src/threading.c b/src/threading.c index 310dfec4a584e..d44bde638d6ad 100644 --- a/src/threading.c +++ b/src/threading.c @@ -62,6 +62,7 @@ JL_THREAD int16_t ti_tid = 0; DLLEXPORT int jl_n_threads; // # threads we're actually using DLLEXPORT int jl_max_threads; // # threads possible jl_thread_task_state_t *jl_all_task_states; +jl_gcframe_t ***jl_all_pgcstacks; // return calling thread's ID DLLEXPORT int16_t jl_threadid() { return ti_tid; } @@ -72,8 +73,8 @@ void ti_initthread(int16_t tid) { ti_tid = tid; jl_pgcstack = NULL; -#ifdef JULIA_ENABLE_THREADING jl_all_pgcstacks[tid] = &jl_pgcstack; +#ifdef JULIA_ENABLE_THREADING jl_all_heaps[tid] = jl_mk_thread_heap(); #else jl_mk_thread_heap(); @@ -93,7 +94,6 @@ JL_DEFINE_MUTEX(typecache); // thread heap struct _jl_thread_heap_t **jl_all_heaps; -jl_gcframe_t ***jl_all_pgcstacks; // only one thread group for now ti_threadgroup_t *tgworld; @@ -454,11 +454,13 @@ void jl_threading_profile() #else // !JULIA_ENABLE_THREADING -void jl_init_threading(void) { +void jl_init_threading(void) +{ static jl_thread_task_state_t _jl_all_task_states; jl_all_task_states = &_jl_all_task_states; jl_max_threads = 1; jl_n_threads = 1; + jl_all_pgcstacks = malloc(jl_n_threads * sizeof(void*)); ti_initthread(0); } diff --git a/src/threading.h b/src/threading.h index 3851b0946f648..f6e226b0be5bf 100644 --- a/src/threading.h +++ b/src/threading.h @@ -46,8 +46,8 @@ extern DLLEXPORT int jl_n_threads; // # threads we're actually using #ifdef JULIA_ENABLE_THREADING // GC extern struct _jl_thread_heap_t **jl_all_heaps; -extern jl_gcframe_t ***jl_all_pgcstacks; #endif +extern jl_gcframe_t ***jl_all_pgcstacks; // thread state enum { From 3a86a98788cf1247782d058233d0278c3c01227d Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 3 Sep 2015 20:22:51 -0400 Subject: [PATCH 0041/1938] hoist loads of constant closed variables improves the code generated for closures. slightly helps #10752 --- src/codegen.cpp | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index fddbf5ef42809..663e7a594ed31 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4373,8 +4373,26 @@ static Function *emit_function(jl_lambda_info_t *lam) ctx.gensym_SAvalues.assign(n_gensyms, (Value*)NULL); // fetch env out of function object if we need it - if (hasCapt) { + if (hasCapt) ctx.envArg = emit_nthptr(fArg, offsetof(jl_function_t,env)/sizeof(jl_value_t*), tbaa_func); + // hoist loads of constant closed variables + for(i=0; i < captvinfoslen; i++) { + jl_sym_t *s = (jl_sym_t*)jl_cellref(jl_cellref(captvinfos,i),0); + assert(jl_is_symbol(s)); + jl_varinfo_t &vi = ctx.vars[s]; + if (!vi.isAssigned) { + vi.SAvalue = emit_var(s, vi.declType, &ctx, true); + if (store_unboxed_p(vi.declType)) + vi.SAvalue = emit_unbox(julia_struct_to_llvm(vi.declType), + vi.SAvalue, vi.declType); + vi.hasGCRoot = false; + if (jl_is_array_type(vi.declType)) { + maybe_alloc_arrayvar(s, &ctx); + jl_arrayvar_t *av = arrayvar_for((jl_value_t*)s, &ctx); + if (av != NULL) + assign_arrayvar(*av, vi.SAvalue); + } + } } // step 8. set up GC frame From dbacfa2027f1c82c986d03d30ec55b0b827bf591 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Fri, 11 Sep 2015 03:42:57 -0700 Subject: [PATCH 0042/1938] Always run doc/genstdlib.jl after bootstrap Should make it more noticeable when doc changes need to be committed to the generated RST --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5d8ba3a7f3b3f..0381502a6af19 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,10 @@ julia-sysimg-release : julia-inference julia-ui-release julia-sysimg-debug : julia-inference julia-ui-debug @$(MAKE) $(QUIET_MAKE) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) JULIA_BUILD_MODE=debug -julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest +julia-genstdlib : julia-sysimg-$(JULIA_BUILD_MODE) + @$(JULIA_EXECUTABLE) doc/genstdlib.jl + +julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest julia-genstdlib debug release : % : julia-% From 6e3c589b521a09b5c97c6a62b2499a9196dd41d5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 3 Aug 2015 13:14:58 -0400 Subject: [PATCH 0043/1938] try to avoid a possible bug in gnumake some AppVeyor builds (like https://ci.appveyor.com/project/StefanKarpinski/julia/build/1.0.7584/job/2yoa82naaj9xfhyl) seem to be getting a corrupt expansion. this is more accurate at being a recursive wildcard anyways --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5d8ba3a7f3b3f..c538c24313e43 100644 --- a/Makefile +++ b/Makefile @@ -170,7 +170,7 @@ CORE_SRCS := base/boot.jl base/coreimg.jl \ base/reduce.jl \ base/reflection.jl \ base/tuple.jl -BASE_SRCS := $(wildcard base/*.jl base/*/*.jl base/*/*/*.jl) +BASE_SRCS := $(shell find base -name \*.jl) $(build_private_libdir)/inference0.ji: $(CORE_SRCS) | $(build_private_libdir) @$(call PRINT_JULIA, cd base && \ From df4a918cdd07576b687dae3f66aa15bb5b4b51eb Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 4 Aug 2015 02:43:08 -0400 Subject: [PATCH 0044/1938] support out-of-tree builds in each subdirectories inspired by KBuild, supporting specification of both (or either) the input (SRCDIR) and intermediary (BUILDDIR) directories: mkdir -p build/src && cd build/src && \ make -f ../src/Makefile all && \ make -C ../src/Makefile BUILDDIR=`pwd` all --- Makefile | 4 +- base/Makefile | 65 +- deps/.gitignore | 1 + deps/Makefile | 1099 +++++++++++++++++---------------- deps/jlchecksum | 10 +- deps/osx-10.10.llvm-3.3.patch | 4 +- src/Makefile | 75 +-- src/julia.h | 8 +- src/julia_internal.h | 4 +- src/mk_julia_flisp_boot.scm | 7 +- src/options.h | 7 +- ui/Makefile | 14 +- 12 files changed, 671 insertions(+), 627 deletions(-) diff --git a/Makefile b/Makefile index c538c24313e43..1904d62cc009e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -JULIAHOME = $(abspath .) +JULIAHOME := $(abspath .) include $(JULIAHOME)/Make.inc # TODO: Code bundled with Julia should be installed into a versioned directory, @@ -325,7 +325,7 @@ ifeq ($(OS),WINNT) endif $(INSTALL_F) $(build_includedir)/uv* $(DESTDIR)$(includedir)/julia endif - $(INSTALL_F) src/julia.h src/julia_version.h src/options.h src/support/*.h $(DESTDIR)$(includedir)/julia + $(INSTALL_F) src/julia.h src/julia_version.h src/support/*.h $(DESTDIR)$(includedir)/julia # Copy system image -$(INSTALL_F) $(build_private_libdir)/sys.ji $(DESTDIR)$(private_libdir) $(INSTALL_M) $(build_private_libdir)/sys.$(SHLIB_EXT) $(DESTDIR)$(private_libdir) diff --git a/base/Makefile b/base/Makefile index 5888408e15c31..57846b05ef23a 100644 --- a/base/Makefile +++ b/base/Makefile @@ -1,6 +1,8 @@ -JULIAHOME = .. -include ../deps/Versions.make -include ../Make.inc +SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +JULIAHOME := $(abspath $(SRCDIR)/..) +BUILDDIR ?= . +include $(JULIAHOME)/deps/Versions.make +include $(JULIAHOME)/Make.inc TAGGED_RELEASE_BANNER = "" @@ -10,7 +12,7 @@ else CPP_STDOUT = $(CPP) -E endif -all: pcre_h.jl errno_h.jl build_h.jl.phony fenv_constants.jl file_constants.jl uv_constants.jl version_git.jl.phony +all: $(addprefix $(BUILDDIR)/,pcre_h.jl errno_h.jl build_h.jl.phony fenv_constants.jl file_constants.jl uv_constants.jl version_git.jl.phony) PCRE_CONST = 0x[0-9a-fA-F]+|[0-9]+ ifeq ($(USE_SYSTEM_PCRE), 1) @@ -19,22 +21,22 @@ else PCRE_INCL_PATH = $(build_includedir)/pcre2.h endif -pcre_h.jl: - @$(call PRINT_PERL, $(CPP) -D PCRE2_CODE_UNIT_WIDTH=8 -dM $(PCRE_INCL_PATH) | perl -nle '/^\s*#define\s+PCRE2_(\w*)\s*\(?($(PCRE_CONST))\)?u?\s*$$/ and print "const $$1 = UInt32($$2)"' | sort > $@) +$(BUILDDIR)/pcre_h.jl: $(PCRE_INCL_PATH) + @$(call PRINT_PERL, $(CPP) -D PCRE2_CODE_UNIT_WIDTH=8 -dM $< | perl -nle '/^\s*#define\s+PCRE2_(\w*)\s*\(?($(PCRE_CONST))\)?u?\s*$$/ and print "const $$1 = UInt32($$2)"' | sort > $@) -errno_h.jl: - @$(call PRINT_PERL, echo '#include "errno.h"' | $(CPP) -dM - | perl -nle 'print "const $$1 = Int32($$2)" if /^#define\s+(E\w+)\s+(\d+)\s*$$/' | sort > $@) +$(BUILDDIR)/errno_h.jl: + @$(call PRINT_PERL, echo '#include ' | $(CPP) -dM - | perl -nle 'print "const $$1 = Int32($$2)" if /^#define\s+(E\w+)\s+(\d+)\s*$$/' | sort > $@) -fenv_constants.jl: ../src/fenv_constants.h - @$(PRINT_PERL) $(CPP_STDOUT) -DJULIA ../src/fenv_constants.h | tail -n 8 | perl -ple 's/\sFE_UN\w+/ 0x10/g; s/\sFE_O\w+/ 0x08/g; s/\sFE_DI\w+/ 0x04/g; s/\sFE_INV\w+/ 0x01/g; s/\sFE_TON\w+/ 0x00/g; s/\sFE_UP\w+/ 0x800/g; s/\sFE_DO\w+/ 0x400/g; s/\sFE_TOW\w+/ 0xc00/g' > $@ +$(BUILDDIR)/fenv_constants.jl: $(SRCDIR)/../src/fenv_constants.h + @$(PRINT_PERL) $(CPP_STDOUT) -DJULIA $< | tail -n 8 | perl -ple 's/\sFE_UN\w+/ 0x10/g; s/\sFE_O\w+/ 0x08/g; s/\sFE_DI\w+/ 0x04/g; s/\sFE_INV\w+/ 0x01/g; s/\sFE_TON\w+/ 0x00/g; s/\sFE_UP\w+/ 0x800/g; s/\sFE_DO\w+/ 0x400/g; s/\sFE_TOW\w+/ 0xc00/g' > $@ -file_constants.jl: ../src/file_constants.h - @$(call PRINT_PERL, $(CPP_STDOUT) -DJULIA ../src/file_constants.h | perl -nle 'print "$$1 0o$$2" if /^(\s*const\s+[A-z_]+\s+=)\s+(0[0-9]*)\s*$$/; print "$$1" if /^\s*(const\s+[A-z_]+\s+=\s+([1-9]|0x)[0-9A-z]*)\s*$$/' > $@) +$(BUILDDIR)/file_constants.jl: $(SRCDIR)/../src/file_constants.h + @$(call PRINT_PERL, $(CPP_STDOUT) -DJULIA $< | perl -nle 'print "$$1 0o$$2" if /^(\s*const\s+[A-z_]+\s+=)\s+(0[0-9]*)\s*$$/; print "$$1" if /^\s*(const\s+[A-z_]+\s+=\s+([1-9]|0x)[0-9A-z]*)\s*$$/' > $@) -uv_constants.jl: ../src/uv_constants.h $(build_includedir)/uv-errno.h - @$(call PRINT_PERL, $(CPP_STDOUT) "-I$(LIBUV_INC)" -DJULIA ../src/uv_constants.h | tail -n 16 > $@) +$(BUILDDIR)/uv_constants.jl: $(SRCDIR)/../src/uv_constants.h $(build_includedir)/uv-errno.h + @$(call PRINT_PERL, $(CPP_STDOUT) "-I$(LIBUV_INC)" -DJULIA $< | tail -n 16 > $@) -build_h.jl.phony: +$(BUILDDIR)/build_h.jl.phony: @echo "# This file is automatically generated in base/Makefile" > $@ @echo "const ARCH = :$(ARCH)" >> $@ ifeq ($(XC_HOST),) @@ -73,9 +75,9 @@ endif rm -f $@; \ fi -version_git.jl.phony: +$(BUILDDIR)/version_git.jl.phony: $(SRCDIR)/version_git.sh ifneq ($(NO_GIT), 1) - @sh version_git.sh > $@ + @sh $< > $@ @# This to avoid touching git_version.jl when it is not modified, @# so that the system image does not need to be rebuilt. @if ! cmp -s $@ version_git.jl; then \ @@ -85,29 +87,26 @@ ifneq ($(NO_GIT), 1) rm -f $@; \ fi else -ifeq ($(shell [ -f version_git.jl ] && echo "true"), true) +ifeq ($(shell [ -f $(BUILDDIR)/version_git.jl ] && echo "true"), true) @# Give warning if boilerplate git is used - @if grep -q "Default output if git is not available" version_git.jl; then \ + @if grep -q "Default output if git is not available" $(BUILDDIR)/version_git.jl; then \ echo "WARNING: Using boilerplate git version info" >&2; \ fi else $(warning "WARNING: Generating boilerplate git version info") - @sh version_git.sh NO_GIT > version_git.jl + @sh $(SRCDIR)/version_git.sh NO_GIT > $(BUILDDIR)/version_git.jl endif endif -.PHONY: build_h.jl.phony version_git.jl.phony - - +.PHONY: $(BUILDDIR)/build_h.jl.phony $(BUILDDIR)/version_git.jl.phony clean all clean: - rm -f *# *~ - rm -f pcre_h.jl - rm -f errno_h.jl - rm -f build_h.jl - rm -f build_h.jl.phony - rm -f fenv_constants.jl - rm -f uv_constants.jl - rm -f file_constants.jl - rm -f version_git.jl - rm -f version_git.jl.phony + rm -f $(BUILDDIR)/pcre_h.jl + rm -f $(BUILDDIR)/errno_h.jl + rm -f $(BUILDDIR)/build_h.jl + rm -f $(BUILDDIR)/build_h.jl.phony + rm -f $(BUILDDIR)/fenv_constants.jl + rm -f $(BUILDDIR)/uv_constants.jl + rm -f $(BUILDDIR)/file_constants.jl + rm -f $(BUILDDIR)/version_git.jl + rm -f $(BUILDDIR)/version_git.jl.phony diff --git a/deps/.gitignore b/deps/.gitignore index 11fdbc52bfac4..c4a02377c0308 100644 --- a/deps/.gitignore +++ b/deps/.gitignore @@ -31,6 +31,7 @@ /Rmath-julia* # git-externals: +/srccache /libgit2 /libgit2-* /libuv diff --git a/deps/Makefile b/deps/Makefile index 2a49983db59ff..ddfaf79259708 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -1,6 +1,8 @@ ## high-level setup ## -JULIAHOME = $(abspath ..) -include Versions.make +SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +JULIAHOME := $(abspath $(SRCDIR)/..) +BUILDDIR ?= . +include $(SRCDIR)/Versions.make include $(JULIAHOME)/Make.inc # Special comments: @@ -10,11 +12,12 @@ include $(JULIAHOME)/Make.inc # additionally all targets should be listed in the getall target for easier off-line compilation # if you are adding a new target, it can help to copy an similar, existing target # -# autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf libuv -# custom Makefile rules: openlibm Rmath-julia dsfmt suitesparse-wrapper suitesparse lapack openblas utf8proc objconv +# autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf libuv git +# custom Makefile rules: openlibm Rmath-julia dsfmt suitesparse-wrapper suitesparse lapack openblas utf8proc objconv osxunwind +# entirely custom: virtualenv # CMake libs: libgit2 # -# downloaded from git: llvm-svn, libuv, libopenlibm, utf8proc, openspecfun +# downloaded from git: llvm-svn, libuv, libopenlibm, utf8proc, openspecfun, libgit2 # # there are rules in this file with the . replaced by a % # this is some magic Makefile trick that tells make @@ -208,10 +211,11 @@ DIRS = $(sort $(build_bindir) $(build_libdir) $(build_includedir) $(build_syscon $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) $(build_prefix): | $(DIRS) +$(eval $(call dir_target,$(SRCDIR)/srccache)) ## A rule for calling `make install` ## # rule: dependencies -# $(call make-install build-directory,add-args) +# $(call make-install,rel-build-directory,add-args) # touch -c $@ # this rule ensures that make install is more nearly atomic # so it's harder to get half-installed (or half-reinstalled) dependencies @@ -224,16 +228,17 @@ define staged-install endef define make-install - $(call staged-install,$1,+$(MAKE) -C $1 install $(MAKE_COMMON) $2 DESTDIR=$(call MAKE_DESTDIR,$1)) + $(call staged-install,$1,+$(MAKE) -C $(BUILDDIR)/$1 install $(MAKE_COMMON) $2 DESTDIR=$(call MAKE_DESTDIR,$1)) endef ## A rule for making a git-external dependency ## # call syntax: -# $(eval $(call git-external,dirname,VARNAME,file_from_download,file_from_compile) +# $(eval $(call git-external,dirname,VARNAME,file_from_download,file_from_compile,SRCDIR) # dirname is the folder name to create # VARNAME is the uppercased variable name prefix # file_from_download is the name of a file inside the git repo to indicate when it's cloned (relative to dirname) # file_from_compile is the rule to run to compile the dependency (relative to dirname) +# SRCDIR is either $(SRCDIR)/srccache or $(BUILDDIR), depending on whether the target supports out-of-tree builds # # also, in a file named dirname.version, define variables VARNAME_BRANCH and VARNAME_SHA1 # @@ -242,42 +247,62 @@ endef # this will pass along the VARNAME_SHA1 target in $1 for its use # # this defines rules for: -# VARNAME_SRC_DIR = source directory for the output +# VARNAME_SRC_DIR = source directory for the output, relative to $(SRCDIR)/srccache or $(BUILDDIR) # dirname: # dirname/file_from_download: # distclean-dirname: # define git-external -include $1.version +include $(SRCDIR)/$1.version + ifneq ($(NO_GIT),1) $2_SRC_DIR = $1 -$1: - git clone -q --depth=50 --branch $$($2_BRANCH) $$($2_GIT_URL) $1 - touch -c $1/$3 -$1/$3: $1.version | $1 - cd $1 && git fetch -q $$($2_GIT_URL) $$($2_BRANCH):remotes/origin/$$($2_BRANCH) - cd $1 && git checkout -q --detach $$($2_SHA1) - @[ '$$($2_SHA1)' = "$$$$(cd $1 && git show -s --format='%H' HEAD)" ] || echo $$(WARNCOLOR)'==> warning: SHA1 hash did not match $1.version file'$$(ENDCOLOR) +$$(SRCDIR)/srccache/$1.git: | $$(SRCDIR)/srccache + git clone -q --mirror --depth=10 --branch $$($2_BRANCH) $$($2_GIT_URL) $$@ +$5/$1: | $$(SRCDIR)/srccache/$1.git + # try to update the cache, if that fails, attempt to continue anyways (the ref might already be local) + -cd $$(SRCDIR)/srccache/$1.git && git fetch -q $$($2_GIT_URL) $$($2_BRANCH):remotes/origin/$$($2_BRANCH) + git clone -q --branch $$($2_BRANCH) $$(SRCDIR)/srccache/$1.git $$@ + cd $5/$1 && git remote set-url origin $$($2_GIT_URL) + touch -c $5/$1/$3 +ifneq ($5,$$(BUILDDIR)) +$$(BUILDDIR)/$1: + mkdir -p $$@ +$5/$1/$3: | $$(BUILDDIR)/$1 +endif +$5/$1/$3: $$(SRCDIR)/$1.version | $5/$1 + # try to update the cache, if that fails, attempt to continue anyways (the ref might already be local) + -cd $$(SRCDIR)/srccache/$1.git && git fetch -q $$($2_GIT_URL) $$($2_BRANCH):remotes/origin/$$($2_BRANCH) + cd $5/$1 && git fetch -q $$(SRCDIR)/srccache/$1.git $$($2_BRANCH):remotes/origin/$$($2_BRANCH) + cd $5/$1 && git checkout -q --detach $$($2_SHA1) + @[ '$$($2_SHA1)' = "$$$$(cd $5/$1 && git show -s --format='%H' HEAD)" ] || echo $$(WARNCOLOR)'==> warning: SHA1 hash did not match $1.version file'$$(ENDCOLOR) touch -c $$@ -# also allow backwards compatibility with an existing git-submodule: -ifeq (exists, $$(shell [ -d $1/.git ] && echo exists )) -$1/$4: $1/.git/HEAD +ifeq ($5,.) +ifeq (exists, $$(shell [ -d $5/$1/.git ] && echo exists )) +$5/$1/$4: $5/$1/.git/HEAD endif ifeq (exists, $$(shell [ -d $$(JULIAHOME)/.git/modules/deps/$1 ] && echo exists )) -$1/$4: $$(JULIAHOME)/.git/modules/deps/$1/HEAD +# also allow backwards compatibility with an existing git-submodule: +$5/$1/$4: $$(JULIAHOME)/.git/modules/deps/$1/HEAD +endif +else #unconditionally add dependency on expected location +$5/$1/$4: $5/$1/.git/HEAD endif + else # NO_GIT + $2_SRC_DIR = $1-$$($2_SHA1) -$$($2_SRC_DIR).tar.gz: +$$(SRCDIR)/$$($2_SRC_DIR).tar.gz: $$(JLDOWNLOAD) $$@ $$(call $2_TAR_URL,$$($2_SHA1)) -$$($2_SRC_DIR)/$3: $$($2_SRC_DIR).tar.gz - $$(JLCHECKSUM) $$($2_SRC_DIR).tar.gz - mkdir -p $$($2_SRC_DIR) && \ - $(TAR) -C $$($2_SRC_DIR) --strip-components 1 -xf $$< +$5/$$($2_SRC_DIR)/$3: $$(SRCDIR)/$$($2_SRC_DIR).tar.gz + $$(JLCHECKSUM) $< + mkdir -p $$(dir $$@) && \ + $(TAR) -C $$(dir $$@) --strip-components 1 -xf $$< touch -c $$@ endif + distclean-$1: - -rm -rf $$($2_SRC_DIR) $$($2_SRC_DIR).tar.gz + -rm -rf $5/$$($2_SRC_DIR) $$(SRCDIR)/srccache/$$($2_SRC_DIR).tar.gz endef @@ -312,36 +337,38 @@ LLVM_BUILDTYPE := $(LLVM_BUILDTYPE)+ASAN endif endif +LLVM_SRC_DIR=$(SRCDIR)/srccache/llvm-$(LLVM_VER) +LLVM_BUILD_DIR=$(BUILDDIR)/llvm-$(LLVM_VER) LLVM_LIB_FILE = libLLVMCodeGen.a ifeq ($(LLVM_USE_CMAKE),1) -LLVM_OBJ_SOURCE = llvm-$(LLVM_VER)/build_$(LLVM_BUILDTYPE)/lib/$(LLVM_LIB_FILE) +LLVM_OBJ_SOURCE = $(LLVM_BUILD_DIR)/build_$(LLVM_BUILDTYPE)/lib/$(LLVM_LIB_FILE) else -LLVM_OBJ_SOURCE = llvm-$(LLVM_VER)/build_$(LLVM_BUILDTYPE)/$(LLVM_FLAVOR)/lib/$(LLVM_LIB_FILE) +LLVM_OBJ_SOURCE = $(LLVM_BUILD_DIR)/build_$(LLVM_BUILDTYPE)/$(LLVM_FLAVOR)/lib/$(LLVM_LIB_FILE) endif LLVM_OBJ_TARGET = $(build_libdir)/$(LLVM_LIB_FILE) ifneq ($(LLVM_VER),svn) ifeq ($(LLVM_VER), 3.3) -LLVM_TAR=llvm-$(LLVM_VER).src.tar.gz +LLVM_TAR=$(SRCDIR)/srccache/llvm-$(LLVM_VER).src.tar.gz else -LLVM_TAR=llvm-$(LLVM_VER).src.tar.xz +LLVM_TAR=$(SRCDIR)/srccache/llvm-$(LLVM_VER).src.tar.xz endif ifeq ($(BUILD_LLDB),1) ifeq ($(LLVM_VER), 3.3) -LLVM_LLDB_TAR=lldb-$(LLVM_VER).src.tar.gz +LLVM_LLDB_TAR=$(SRCDIR)/srccache/lldb-$(LLVM_VER).src.tar.gz else -LLVM_LLDB_TAR=lldb-$(LLVM_VER).src.tar.xz +LLVM_LLDB_TAR=$(SRCDIR)/srccache/lldb-$(LLVM_VER).src.tar.xz endif # LLVM_VER == 3.3 endif # BUILD_LLDB ifeq ($(BUILD_LLVM_CLANG),1) ifeq ($(LLVM_VER), 3.3) -LLVM_CLANG_TAR=cfe-$(LLVM_VER).src.tar.gz -LLVM_COMPILER_RT_TAR=compiler-rt-$(LLVM_VER).src.tar.gz +LLVM_CLANG_TAR=$(SRCDIR)/srccache/cfe-$(LLVM_VER).src.tar.gz +LLVM_COMPILER_RT_TAR=$(SRCDIR)/srccache/compiler-rt-$(LLVM_VER).src.tar.gz else -LLVM_CLANG_TAR=cfe-$(LLVM_VER).src.tar.xz -LLVM_COMPILER_RT_TAR=compiler-rt-$(LLVM_VER).src.tar.xz +LLVM_CLANG_TAR=$(SRCDIR)/srccache/cfe-$(LLVM_VER).src.tar.xz +LLVM_COMPILER_RT_TAR=$(SRCDIR)/srccache/compiler-rt-$(LLVM_VER).src.tar.xz endif # LLVM_VER else LLVM_CLANG_TAR= @@ -350,7 +377,7 @@ LLVM_LIBCXX_TAR= endif # BUILD_LLVM_CLANG ifeq ($(BUILD_CUSTOM_LIBCXX),1) -LLVM_LIBCXX_TAR=libcxx-$(LLVM_VER).src.tar.gz +LLVM_LIBCXX_TAR=$(SRCDIR)/srccache/libcxx-$(LLVM_VER).src.tar.gz endif endif # LLVM_VER != svn @@ -454,44 +481,55 @@ LLVM_FLAGS += LDFLAGS="$(LLVM_LDFLAGS)" LLVM_MFLAGS += LDFLAGS="$(LLVM_LDFLAGS)" endif +ifeq ($(BUILD_LLVM_CLANG),1) +LLVM_MFLAGS += OPTIONAL_PARALLEL_DIRS=clang +else +# block default building of Clang +LLVM_MFLAGS += OPTIONAL_PARALLEL_DIRS= +endif +ifeq ($(BUILD_LLDB),1) +LLVM_MFLAGS += OPTIONAL_DIRS=lldb +else +# block default building of lldb +LLVM_MFLAGS += OPTIONAL_DIRS= +endif + ifneq ($(LLVM_CLANG_TAR),) $(LLVM_CLANG_TAR): - $(JLDOWNLOAD) $@ http://llvm.org/releases/$(LLVM_VER)/$@ + $(JLDOWNLOAD) $@ http://llvm.org/releases/$(LLVM_VER)/$(notdir $@) endif ifneq ($(LLVM_COMPILER_RT_TAR),) $(LLVM_COMPILER_RT_TAR): - $(JLDOWNLOAD) $@ http://llvm.org/releases/$(LLVM_VER)/$@ + $(JLDOWNLOAD) $@ http://llvm.org/releases/$(LLVM_VER)/$(notdir $@) endif ifneq ($(LLVM_LIBCXX_TAR),) $(LLVM_LIBCXX_TAR): - $(JLDOWNLOAD) $@ http://llvm.org/releases/$(LLVM_VER)/$@ + $(JLDOWNLOAD) $@ http://llvm.org/releases/$(LLVM_VER)/$(notdir $@) endif ifneq ($(LLVM_VER),svn) $(LLVM_TAR): - $(JLDOWNLOAD) $@ http://llvm.org/releases/$(LLVM_VER)/$@ + $(JLDOWNLOAD) $@ http://llvm.org/releases/$(LLVM_VER)/$(notdir $@) endif ifneq ($(LLVM_LLDB_TAR),) $(LLVM_LLDB_TAR): - $(JLDOWNLOAD) $@ http://llvm.org/releases/$(LLVM_VER)/$@ + $(JLDOWNLOAD) $@ http://llvm.org/releases/$(LLVM_VER)/$(notdir $@) endif ifeq ($(BUILD_LLDB),1) -llvm-$(LLVM_VER)/tools/lldb: -llvm-$(LLVM_VER)/configure: llvm-$(LLVM_VER)/tools/lldb +$(LLVM_SRC_DIR)/tools/lldb: +$(LLVM_SRC_DIR)/configure: $(LLVM_SRC_DIR)/tools/lldb endif -llvm-$(LLVM_VER)/python2_path: - mkdir -p llvm-$(LLVM_VER)/python2_path +# LLDB still relies on plenty of python 2.x infrastructure, without checking +llvm_python_workaround=$(SRCDIR)/srccache/python2_path +$(llvm_python_workaround): + mkdir -p $@ -python -c 'import sys; sys.exit(not sys.version_info > (3, 0))' && \ /usr/bin/python2 -c 'import sys; sys.exit(not sys.version_info < (3, 0))' && \ - ln -sf /usr/bin/python2 "llvm-$(LLVM_VER)/python2_path/python" && \ - ln -sf /usr/bin/python2-config "llvm-$(LLVM_VER)/python2_path/python-config" -LLVM_FLAGS += --with-python="$(shell ./find_python2)" -# LLDB still relies on plenty of python 2.x infrastructure, without checking -ifeq ($(BUILD_LLDB),1) -llvm_python_workaround=llvm-$(LLVM_VER)/python2_path -endif + ln -sf /usr/bin/python2 "$@/python" && \ + ln -sf /usr/bin/python2-config "$@/python-config" +LLVM_FLAGS += --with-python="$(shell $(SRCDIR)/find_python2)" ifeq ($(BUILD_CUSTOM_LIBCXX),1) @@ -499,49 +537,49 @@ ifeq ($(USEICC),1) LIBCXX_EXTRA_FLAGS = -Bstatic -lirc -Bdynamic endif -llvm-$(LLVM_VER)/projects/libcxx: $(LLVM_LIBCXX_TAR) | llvm-svn/configure - ([ ! -d llvm-$(LLVM_VER)/projects/libcxx ] && \ - git clone $(LLVM_GIT_URL_LIBCXX) llvm-$(LLVM_VER)/projects/libcxx ) || \ - (cd llvm-$(LLVM_VER)/projects/libcxx && \ +$(LLVM_SRC_DIR)/projects/libcxx: $(LLVM_LIBCXX_TAR) | $(LLVM_SRC_DIR)/configure + ([ ! -d $@ ] && \ + git clone $(LLVM_GIT_URL_LIBCXX) $@ ) || \ + (cd $@ && \ git pull --ff-only) -llvm-$(LLVM_VER)/projects/libcxxabi: $(LLVM_LIBCXXABI_TAR) | llvm-svn/configure - ([ ! -d llvm-$(LLVM_VER)/projects/libcxxabi ] && \ - git clone $(LLVM_GIT_URL_LIBCXXABI) llvm-$(LLVM_VER)/projects/libcxxabi ) || \ - (cd llvm-$(LLVM_VER)/projects/libcxxabi && \ +$(LLVM_SRC_DIR)/projects/libcxx/.git/HEAD: | $(LLVM_SRC_DIR)/projects/libcxx/.git/HEAD +$(LLVM_SRC_DIR)/projects/libcxxabi: $(LLVM_LIBCXXABI_TAR) | $(LLVM_SRC_DIR)/configure + ([ ! -d $@ ] && \ + git clone $(LLVM_GIT_URL_LIBCXXABI) $@ ) || \ + (cd $@ && \ git pull --ff-only) -libcxx-build: - mkdir -p libcxx-build -libcxx-build/Makefile: llvm-$(LLVM_VER)/projects/libcxx | llvm-$(LLVM_VER)/projects/libcxxabi libcxx-build - cd libcxx-build && \ - $(CMAKE) -G "Unix Makefiles" $(CMAKE_COMMON) $(LLVM_CMAKE) -DLIBCXX_CXX_ABI=libcxxabi -DLIBCXX_CXX_ABI_INCLUDE_PATHS="../llvm-$(LLVM_VER)/projects/libcxxabi/include" ../llvm-$(LLVM_VER)/projects/libcxx -DCMAKE_SHARED_LINKER_FLAGS="-L$(build_libdir) $(LIBCXX_EXTRA_FLAGS)" -DCMAKE_CXX_FLAGS="$(CXXFLAGS)" -libcxxabi-build: - mkdir -p libcxxabi-build -libcxxabi-build/Makefile: llvm-$(LLVM_VER)/projects/libcxxabi | llvm-$(LLVM_VER)/projects/libcxx libcxxabi-build - cd libcxxabi-build && \ - $(CMAKE) -G "Unix Makefiles" $(CMAKE_COMMON) $(LLVM_CMAKE) -DLLVM_ABI_BREAKING_CHECKS="WITH_ASSERTS" -DLLVM_PATH="../llvm-$(LLVM_VER)" ../llvm-$(LLVM_VER)/projects/libcxxabi -DLIBCXXABI_CXX_ABI_LIBRARIES="$(LIBCXX_EXTRA_FLAGS)" -DCMAKE_CXX_FLAGS="$(CXXFLAGS) -std=c++11" -llvm-$(LLVM_VER)/projects/libcxxabi/lib/libc++abi.so.1.0: llvm-$(LLVM_VER)/projects/libcxxabi | llvm-$(LLVM_VER)/projects/libcxx -libcxxabi-build/lib/libc++abi.so.1.0: | libcxxabi-build/Makefile - cd libcxxabi-build && $(MAKE) -$(build_libdir)/libc++abi.so.1.0: libcxxabi-build/lib/libc++abi.so.1.0 - cd libcxxabi-build && $(MAKE) install - touch $@ +$(LLVM_SRC_DIR)/projects/libcxxabi/.git/HEAD: | $(LLVM_SRC_DIR)/projects/libcxxabi +$(LLVM_BUILD_DIR)/libcxx-build/Makefile: | $(LLVM_SRC_DIR)/projects/libcxx $(LLVM_SRC_DIR)/projects/libcxxabi + mkdir -p $(dir $@) + cd $(dir $@) && \ + $(CMAKE) -G "Unix Makefiles" $(CMAKE_COMMON) $(LLVM_CMAKE) -DLIBCXX_CXX_ABI=libcxxabi -DLIBCXX_CXX_ABI_INCLUDE_PATHS="$(LLVM_SRC_DIR)/projects/libcxxabi/include" $(LLVM_SRC_DIR)/projects/libcxx -DCMAKE_SHARED_LINKER_FLAGS="-L$(build_libdir) $(LIBCXX_EXTRA_FLAGS)" -DCMAKE_CXX_FLAGS="$(CXXFLAGS)" +$(LLVM_BUILD_DIR)/libcxxabi-build/Makefile: | $(LLVM_SRC_DIR)/projects/libcxxabi llvm-$(LLVM_VER)/projects/libcxx + mkdir -p $(dir $@) + cd $(dir $@) && \ + $(CMAKE) -G "Unix Makefiles" $(CMAKE_COMMON) $(LLVM_CMAKE) -DLLVM_ABI_BREAKING_CHECKS="WITH_ASSERTS" -DLLVM_PATH="$(LLVM_SRC_DIR)" $(LLVM_SRC_DIR)/projects/libcxxabi -DLIBCXXABI_CXX_ABI_LIBRARIES="$(LIBCXX_EXTRA_FLAGS)" -DCMAKE_CXX_FLAGS="$(CXXFLAGS) -std=c++11" +$(LLVM_BUILD_DIR)/libcxxabi-build/lib/libc++abi.so.1.0: $(LLVM_BUILD_DIR)/libcxxabi-build/Makefile $(LLVM_SRC_DIR)/projects/libcxxabi/.git/HEAD + $(MAKE) -C $(LLVM_BUILD_DIR)/libcxxabi-build + touch -c $@ +$(build_libdir)/libc++abi.so.1.0: $(LLVM_BUILD_DIR)/libcxxabi-build/lib/libc++abi.so.1.0 + $(MAKE) -C $(LLVM_BUILD_DIR)/libcxxabi-build install + touch -c $@ +$(LLVM_BUILD_DIR)/libcxx-build/lib/libc++.so.1.0: $(build_libdir)/libc++abi.so.1.0 $(LLVM_BUILD_DIR)/libcxx-build/Makefile $(LLVM_SRC_DIR)/projects/libcxx/.git/HEAD + $(MAKE) -C $(LLVM_BUILD_DIR)/libcxx-build +$(build_libdir)/libc++.so.1.0: $(LLVM_BUILD_DIR)/libcxx-build/lib/libc++.so.1.0 + $(MAKE) -C $(LLVM_BUILD_DIR)/libcxx-build install + touch -c $@ +get-libcxx: $(LLVM_SRC_DIR)/projects/libcxx +get-libcxxabi: $(LLVM_SRC_DIR)/projects/libcxxabi install-libcxxabi: $(build_libdir)/libc++abi.so.1.0 -libcxx-build/lib/libc++.so.1.0: | $(build_libdir)/libc++abi.so.1.0 libcxx-build/Makefile - cd libcxx-build && $(MAKE) -$(build_libdir)/libc++.so.1.0: libcxx-build/lib/libc++.so.1.0 - cd libcxx-build && $(MAKE) install - touch $@ install-libcxx: $(build_libdir)/libc++.so.1.0 -get-libcxx: llvm-$(LLVM_VER)/projects/libcxx -get-libcxxabi: llvm-$(LLVM_VER)/projects/libcxxabi endif ifeq ($(BUILD_CUSTOM_LIBCXX),1) LIBCXX_DEPENDENCY = $(build_libdir)/libc++abi.so.1.0 $(build_libdir)/libc++.so.1.0 -LIBCXX_GET_DEPENDENCY = get-libcxx get-libcxxabi +get-llvm: get-libcxx get-libcxxabi endif -llvm-$(LLVM_VER)/configure: $(LLVM_TAR) $(LLVM_CLANG_TAR) $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR) +$(LLVM_SRC_DIR)/configure: $(LLVM_TAR) $(LLVM_CLANG_TAR) $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR) ifneq ($(LLVM_CLANG_TAR),) $(JLCHECKSUM) $(LLVM_CLANG_TAR) endif @@ -558,116 +596,110 @@ ifneq ($(LLVM_LLDB_TAR),) $(JLCHECKSUM) $(LLVM_LLDB_TAR) endif ifneq ($(LLVM_VER),svn) - mkdir -p llvm-$(LLVM_VER) && \ - $(TAR) -C llvm-$(LLVM_VER) --strip-components 1 -xf $(LLVM_TAR) + mkdir -p $(LLVM_SRC_DIR) + $(TAR) -C $(LLVM_SRC_DIR) --strip-components 1 -xf $(LLVM_TAR) else - ([ ! -d llvm-$(LLVM_VER) ] && \ - git clone $(LLVM_GIT_URL_LLVM) llvm-$(LLVM_VER) ) || \ - (cd llvm-$(LLVM_VER) && \ + ([ ! -d $(LLVM_SRC_DIR) ] && \ + git clone $(LLVM_GIT_URL_LLVM) $(LLVM_SRC_DIR) ) || \ + (cd $(LLVM_SRC_DIR) && \ git pull --ff-only) ifneq ($(LLVM_GIT_VER),) - (cd llvm-$(LLVM_VER) && \ + (cd $(LLVM_SRC_DIR) && \ git checkout $(LLVM_GIT_VER)) endif # LLVM_GIT_VER endif # LLVM_VER ifneq ($(LLVM_VER),svn) ifneq ($(LLVM_CLANG_TAR),) - mkdir -p llvm-$(LLVM_VER)/tools/clang && \ - $(TAR) -C llvm-$(LLVM_VER)/tools/clang --strip-components 1 -xf $(LLVM_CLANG_TAR) + mkdir -p $(LLVM_SRC_DIR)/tools/clang + $(TAR) -C $(LLVM_SRC_DIR)/tools/clang --strip-components 1 -xf $(LLVM_CLANG_TAR) endif # LLVM_CLANG_TAR ifneq ($(LLVM_COMPILER_RT_TAR),) - mkdir -p llvm-$(LLVM_VER)/projects/compiler-rt && \ - $(TAR) -C llvm-$(LLVM_VER)/projects/compiler-rt --strip-components 1 -xf $(LLVM_COMPILER_RT_TAR) + mkdir -p $(LLVM_SRC_DIR)/projects/compiler-rt + $(TAR) -C $(LLVM_SRC_DIR)/projects/compiler-rt --strip-components 1 -xf $(LLVM_COMPILER_RT_TAR) endif # LLVM_COMPILER_RT_TAR ifneq ($(LLVM_LLDB_TAR),) - mkdir -p llvm-$(LLVM_VER)/tools/lldb && \ - $(TAR) -C llvm-$(LLVM_VER)/tools/lldb --strip-components 1 -xf $(LLVM_LLDB_TAR) + mkdir -p $(LLVM_SRC_DIR)/tools/lldb + $(TAR) -C $(LLVM_SRC_DIR)/tools/lldb --strip-components 1 -xf $(LLVM_LLDB_TAR) endif # LLVM_LLDB_TAR else ifeq ($(BUILD_LLVM_CLANG),1) - ([ ! -d llvm-$(LLVM_VER)/tools/clang ] && \ - git clone $(LLVM_GIT_URL_CLANG) llvm-$(LLVM_VER)/tools/clang ) || \ - (cd llvm-$(LLVM_VER)/tools/clang && \ + ([ ! -d $(LLVM_SRC_DIR)/tools/clang ] && \ + git clone $(LLVM_GIT_URL_CLANG) $(LLVM_SRC_DIR)/tools/clang ) || \ + (cd $(LLVM_SRC_DIR)/tools/clang && \ git pull --ff-only) endif # BUILD_LLVM_CLANG ifeq ($(BUILD_LLDB),1) - ([ ! -d llvm-$(LLVM_VER)/tools/lldb ] && \ - git clone $(LLVM_GIT_URL_LLDB) llvm-$(LLVM_VER)/tools/lldb ) || \ - (cd llvm-$(LLVM_VER)/tools/lldb && \ + ([ ! -d $(LLVM_SRC_DIR)/tools/lldb ] && \ + git clone $(LLVM_GIT_URL_LLDB) $(LLVM_SRC_DIR)/tools/lldb ) || \ + (cd $(LLVM_SRC_DIR)/tools/lldb && \ git pull --ff-only) endif # BUILD_LLDB endif # LLVM_VER ifeq ($(LLVM_VER),svn) ifeq ($(BUILD_LLVM_CLANG),1) - ([ ! -d llvm-$(LLVM_VER)/tools/clang ] && \ - git clone $(LLVM_GIT_URL_CLANG) llvm-$(LLVM_VER)/tools/clang ) || \ - (cd llvm-$(LLVM_VER)/tools/clang && \ + ([ ! -d $(LLVM_SRC_DIR)/tools/clang ] && \ + git clone $(LLVM_GIT_URL_CLANG) $(LLVM_SRC_DIR)/tools/clang ) || \ + (cd $(LLVM_SRC_DIR)/tools/clang && \ git pull --ff-only) - ([ ! -d llvm-$(LLVM_VER)/projects/compiler-rt ] && \ - git clone $(LLVM_GIT_URL_COMPILER_RT) llvm-$(LLVM_VER)/projects/compiler-rt ) || \ - (cd llvm-$(LLVM_VER)/projects/compiler-rt && \ + ([ ! -d $(LLVM_SRC_DIR)/projects/compiler-rt ] && \ + git clone $(LLVM_GIT_URL_COMPILER_RT) $(LLVM_SRC_DIR)/projects/compiler-rt ) || \ + (cd $(LLVM_SRC_DIR)/projects/compiler-rt && \ git pull --ff-only) ifneq ($(LLVM_CLANG_VER),) - (cd llvm-$(LLVM_VER) && \ + (cd $(LLVM_SRC_DIR) && \ git checkout $(LLVM_GIT_VER)) endif # LLVM_CLANG_VER endif # BUILD_LLVM_CLANG endif # LLVM_VER + touch -c $@ # Apply version-specific LLVM patches +LLVM_PATCH_LIST= +define LLVM_PATCH +$$(LLVM_SRC_DIR)/$1.patch-applied: $$(SRCDIR)/$1.patch + cd $$(LLVM_SRC_DIR) && patch -p1 < $$< + echo 1 > $$@ +LLVM_PATCH_LIST += $$(LLVM_SRC_DIR)/$1.patch-applied +endef ifeq ($(LLVM_VER),3.3) - patch -p0 < llvm-3.3.patch - patch -p0 < instcombine-llvm-3.3.patch - patch -p0 < int128-vector.llvm-3.3.patch - (cd llvm-$(LLVM_VER) && patch < ../osx-10.10.llvm-3.3.patch) - -ifeq ($(OS),WINNT) -ifeq ($(ARCH),x86_64) - patch -p0 < win64-int128.llvm-3.3.patch -endif # x86_64 -endif # WINNT - +$(eval $(call LLVM_PATCH,llvm-3.3)) +$(eval $(call LLVM_PATCH,instcombine-llvm-3.3)) +$(eval $(call LLVM_PATCH,int128-vector.llvm-3.3)) +$(eval $(call LLVM_PATCH,osx-10.10.llvm-3.3)) else ifeq ($(LLVM_VER),3.6.0) - cd llvm-3.6.0 && patch -p1 < ../zerosign-llvm-3.6.0.patch -ifeq ($(OS),WINNT) -ifeq ($(ARCH),x86_64) - cd llvm-3.6.0 && patch -p1 < ../win64-allocas-llvm-3.6.0.patch -endif # x86_64 -endif # WINNT - +$(eval $(call LLVM_PATCH,zerosign-llvm-3.6.0)) +$(eval $(call LLVM_PATCH,win64-allocas-llvm-3.6.0)) endif # LLVM_VER - touch -c $@ -# end target llvm-$(LLVM_VER)/configure -LLVM_BUILDDIR = llvm-$(LLVM_VER)/build_$(LLVM_BUILDTYPE) +LLVM_BUILDDIR_withtype = $(LLVM_BUILD_DIR)/build_$(LLVM_BUILDTYPE) ifneq ($(LLVM_USE_CMAKE),) -$(LLVM_BUILDDIR)/CMakeCache.txt: llvm-$(LLVM_VER)/configure | $(llvm_python_workaround) $(LIBCXX_DEPENDENCY) - cd llvm-$(LLVM_VER) && \ - mkdir -p build_$(LLVM_BUILDTYPE) && cd build_$(LLVM_BUILDTYPE) && \ - export PATH=$(abspath llvm-$(LLVM_VER)/python2_path):$$PATH && \ - $(CMAKE) .. $(CMAKE_GENERATOR_COMMAND) $(CMAKE_COMMON) $(LLVM_CMAKE) +$(LLVM_BUILDDIR_withtype)/CMakeCache.txt: $(LLVM_SRC_DIR)/configure | $(llvm_python_workaround) $(LIBCXX_DEPENDENCY) + mkdir -p $(dir $@) + cd $(dir $@) && \ + export PATH=$(llvm_python_workaround):$$PATH && \ + $(CMAKE) $(LLVM_SRC_DIR) $(CMAKE_GENERATOR_COMMAND) $(CMAKE_COMMON) $(LLVM_CMAKE) touch -c $@ -$(LLVM_OBJ_SOURCE): $(LLVM_BUILDDIR)/CMakeCache.txt | $(llvm_python_workaround) - cd llvm-$(LLVM_VER)/build_$(LLVM_BUILDTYPE) && \ - export PATH=$(abspath llvm-$(LLVM_VER)/python2_path):$$PATH && \ - $(CMAKE) --build . +$(LLVM_OBJ_SOURCE): $(LLVM_BUILDDIR_withtype)/CMakeCache.txt $(LLVM_PATCH_LIST) | $(llvm_python_workaround) + cd $(LLVM_BUILDDIR_withtype) && \ + export PATH=$(llvm_python_workaround):$$PATH && \ + $(CMAKE) --build . touch -c $@ else -$(LLVM_BUILDDIR)/config.status: llvm-$(LLVM_VER)/configure | $(llvm_python_workaround) $(LIBCXX_DEPENDENCY) - cd llvm-$(LLVM_VER) && \ - mkdir -p build_$(LLVM_BUILDTYPE) && cd build_$(LLVM_BUILDTYPE) && \ - export PATH=$(abspath llvm-$(LLVM_VER)/python2_path):$$PATH && \ - ../configure $(CONFIGURE_COMMON) $(LLVM_CC) $(LLVM_FLAGS) +$(LLVM_BUILDDIR_withtype)/config.status: $(LLVM_SRC_DIR)/configure | $(llvm_python_workaround) $(LIBCXX_DEPENDENCY) + mkdir -p $(dir $@) + cd $(dir $@) && \ + export PATH=$(llvm_python_workaround):$$PATH && \ + $< $(CONFIGURE_COMMON) $(LLVM_CC) $(LLVM_FLAGS) touch -c $@ -$(LLVM_OBJ_SOURCE): $(LLVM_BUILDDIR)/config.status | $(llvm_python_workaround) - cd llvm-$(LLVM_VER)/build_$(LLVM_BUILDTYPE) && \ - export PATH=$(abspath llvm-$(LLVM_VER)/python2_path):$$PATH && \ - $(MAKE) $(LLVM_MFLAGS) $(MAKE_COMMON) +$(LLVM_OBJ_SOURCE): $(LLVM_BUILDDIR_withtype)/config.status $(LLVM_PATCH_LIST) | $(llvm_python_workaround) + cd $(LLVM_BUILDDIR_withtype) && \ + export PATH=$(llvm_python_workaround):$$PATH && \ + $(MAKE) $(LLVM_MFLAGS) $(MAKE_COMMON) touch -c $@ endif # LLVM_USE_CMAKE @@ -677,55 +709,59 @@ CHECK_COMMAND = $(CMAKE) --build . check else CHECK_COMMAND = $(MAKE) $(LLVM_MFLAGS) check endif -$(LLVM_BUILDDIR)/checked: $(LLVM_OBJ_SOURCE) | $(llvm_python_workaround) +$(LLVM_BUILDDIR_withtype)/checked: $(LLVM_OBJ_SOURCE) | $(llvm_python_workaround) ifeq ($(OS),$(BUILD_OS)) - cd llvm-$(LLVM_VER)/build_$(LLVM_BUILDTYPE) && \ - export PATH=$(abspath llvm-$(LLVM_VER)/python2_path):$$PATH && \ - $(CHECK_COMMAND) + cd $(LLVM_BUILDDIR_withtype) && \ + export PATH=$(llvm_python_workaround):$$PATH && \ + $(CHECK_COMMAND) endif echo 1 > $@ $(LLVM_OBJ_TARGET): $(LLVM_OBJ_SOURCE) | $(llvm_python_workaround) ifeq ($(LLVM_USE_CMAKE),1) - $(call staged-install,$(LLVM_BUILDDIR),cd $(LLVM_BUILDDIR) && $(CMAKE) -DCMAKE_INSTALL_PREFIX="$(call MAKE_DESTDIR,$(LLVM_BUILDDIR))$(build_prefix)" -P cmake_install.cmake) + $(call staged-install,build_$(LLVM_BUILDTYPE),cd $(LLVM_BUILDDIR_withtype) && $(CMAKE) -DCMAKE_INSTALL_PREFIX="$(call MAKE_DESTDIR,$(LLVM_BUILDDIR_withtype))$(build_prefix)" -P cmake_install.cmake) else - $(call make-install,$(LLVM_BUILDDIR),$(LLVM_MFLAGS) PATH="$(abspath llvm-$(LLVM_VER)/python2_path):$$PATH") + $(call make-install,llvm-$(LLVM_VER)/build_$(LLVM_BUILDTYPE),$(LLVM_MFLAGS) PATH="$(llvm_python_workaround):$$PATH") endif # LLVM_USE_CMAKE touch -c $@ reinstall-llvm: -rm $(LLVM_OBJ_TARGET) - $(MAKE) -s install-llvm + $(MAKE) -f $(SRCDIR)/Makefile -s install-llvm clean-llvm: - -$(MAKE) -C llvm-$(LLVM_VER)/build_$(LLVM_BUILDTYPE) clean + -$(MAKE) -C $(LLVM_BUILDDIR_withtype) clean -rm -f $(build_bindir)/llvm-config distclean-llvm: - -rm -rf llvm-$(LLVM_VER).tar.gz llvm-$(LLVM_VER).src.tar.gz clang-$(LLVM_VER).src.tar.gz clang-$(LLVM_VER).tar.gz libcxx-$(LLVM_VER).src.tar.gz compiler-rt-$(LLVM_VER).src.tar.gz llvm-$(LLVM_VER) + -rm -rf $(LLVM_TAR) $(LLVM_CLANG_TAR) \ + $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR) \ + $(LLVM_SRC_DIR) ifneq ($(LLVM_VER),svn) get-llvm: $(LLVM_TAR) $(LLVM_CLANG_TAR) $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR) else -get-llvm: llvm-$(LLVM_VER)/configure $(LIBCXX_GET_DEPENDENCY) +get-llvm: $(LLVM_SRC_DIR)/configure endif -configure-llvm: llvm-$(LLVM_VER)/build_$(LLVM_BUILDTYPE)/config.status +configure-llvm: $(LLVM_BUILD_DIR)/build_$(LLVM_BUILDTYPE)/config.status compile-llvm: $(LLVM_OBJ_SOURCE) -check-llvm: llvm-$(LLVM_VER)/build_$(LLVM_BUILDTYPE)/checked +check-llvm: $(LLVM_BUILD_DIR)/build_$(LLVM_BUILDTYPE)/checked install-llvm: $(LLVM_OBJ_TARGET) #todo: LLVM make check target is broken on julia.mit.edu (and really slow elsewhere) +ifeq ($(LLVM_VER),svn) update-llvm: - (cd llvm-$(LLVM_VER); git pull --ff-only) - ([ -d "llvm-$(LLVM_VER)/tools/clang" ] || exit 0; cd llvm-$(LLVM_VER)/tools/clang; git pull --ff-only) - ([ -d "llvm-$(LLVM_VER)/projects/compiler-rt" ] || exit 0; cd llvm-$(LLVM_VER)/projects/compiler-rt; git pull --ff-only) - ([ -d "llvm-$(LLVM_VER)/tools/lldb" ] || exit 0; cd llvm-$(LLVM_VER)/tools/lldb; git pull --ff-only) + (cd $(LLVM_SRC_DIR); git pull --ff-only) + ([ -d "$(LLVM_SRC_DIR)/tools/clang" ] || exit 0; cd $(LLVM_SRC_DIR)/tools/clang; git pull --ff-only) + ([ -d "$(LLVM_SRC_DIR)/projects/compiler-rt" ] || exit 0; cd $(LLVM_SRC_DIR)/projects/compiler-rt; git pull --ff-only) + ([ -d "$(LLVM_SRC_DIR)/tools/lldb" ] || exit 0; cd $(LLVM_SRC_DIR)/tools/lldb; git pull --ff-only) +endif ## LIBUV ## LIBUV_GIT_URL=git://github.com/JuliaLang/libuv.git LIBUV_TAR_URL=https://api.github.com/repos/JuliaLang/libuv/tarball/$1 -$(eval $(call git-external,libuv,LIBUV,configure,.libs/libuv.la)) +$(eval $(call git-external,libuv,LIBUV,configure,.libs/libuv.la,$(SRCDIR)/srccache)) -UV_SRC_TARGET = $(LIBUV_SRC_DIR)/.libs/libuv.la +UV_SRC_TARGET = $(BUILDDIR)/$(LIBUV_SRC_DIR)/.libs/libuv.la UV_OBJ_TARGET = $(build_libdir)/libuv.la UV_CFLAGS = @@ -749,19 +785,19 @@ else UV_FLAGS = --disable-shared $(UV_MFLAGS) endif -$(LIBUV_SRC_DIR)/config.status: $(LIBUV_SRC_DIR)/configure - touch -c $(LIBUV_SRC_DIR)/aclocal.m4 # touch a few files to prevent autogen from getting called - touch -c $(LIBUV_SRC_DIR)/Makefile.in - touch -c $(LIBUV_SRC_DIR)/configure - cd $(LIBUV_SRC_DIR) && \ - ./configure --with-pic $(CONFIGURE_COMMON) $(UV_FLAGS) +$(BUILDDIR)/$(LIBUV_SRC_DIR)/config.status: $(SRCDIR)/srccache/$(LIBUV_SRC_DIR)/configure + touch -c $(SRCDIR)/srccache/$(LIBUV_SRC_DIR)/aclocal.m4 # touch a few files to prevent autogen from getting called + touch -c $(SRCDIR)/srccache/$(LIBUV_SRC_DIR)/Makefile.in + touch -c $(SRCDIR)/srccache/$(LIBUV_SRC_DIR)/configure + cd $(dir $@) && \ + $< --with-pic $(CONFIGURE_COMMON) $(UV_FLAGS) touch -c $@ -$(UV_SRC_TARGET): $(LIBUV_SRC_DIR)/config.status - $(MAKE) -C $(LIBUV_SRC_DIR) $(UV_MFLAGS) +$(UV_SRC_TARGET): $(BUILDDIR)/$(LIBUV_SRC_DIR)/config.status + $(MAKE) -C $(dir $<) $(UV_MFLAGS) touch -c $@ -$(LIBUV_SRC_DIR)/checked: $(UV_SRC_TARGET) +$(BUILDDIR)/$(LIBUV_SRC_DIR)/checked: $(UV_SRC_TARGET) ifeq ($(OS),$(BUILD_OS)) - -$(MAKE) -C $(LIBUV_SRC_DIR) check + -$(MAKE) -C $(dir $@) check endif echo 1 > $@ $(UV_OBJ_TARGET): $(UV_SRC_TARGET) @@ -769,19 +805,19 @@ $(UV_OBJ_TARGET): $(UV_SRC_TARGET) $(INSTALL_NAME_CMD)libuv.$(SHLIB_EXT) $(build_shlibdir)/libuv.$(SHLIB_EXT) clean-libuv: - -$(MAKE) -C $(LIBUV_SRC_DIR) clean + -$(MAKE) -C $(BUILDDIR)/$(LIBUV_SRC_DIR) clean -rm -rf $(build_libdir)/libuv.a $(build_libdir)/libuv.la $(build_includedir)/libuv.h $(build_includedir)/libuv-private -get-libuv: $(LIBUV_SRC_DIR)/configure -configure-libuv: $(LIBUV_SRC_DIR)/config.status +get-libuv: $(SRCDIR)/srccache/$(LIBUV_SRC_DIR)/configure +configure-libuv: $(BUILDDIR)/$(LIBUV_SRC_DIR)/config.status compile-libuv: $(UV_SRC_TARGET) -check-libuv: $(LIBUV_SRC_DIR)/checked +check-libuv: $(BUILDDIR)/$(LIBUV_SRC_DIR)/checked install-libuv: $(UV_OBJ_TARGET) ## PCRE ## -PCRE_SRC_TARGET = pcre2-$(PCRE_VER)/.libs/libpcre2-8.$(SHLIB_EXT) +PCRE_SRC_TARGET = $(BUILDDIR)/pcre2-$(PCRE_VER)/.libs/libpcre2-8.$(SHLIB_EXT) PCRE_OBJ_TARGET = $(build_shlibdir)/libpcre2-8.$(SHLIB_EXT) # Force optimization for PCRE flags (Issue #11668) @@ -790,27 +826,28 @@ ifneq ($(OS),WINNT) PCRE_LDFLAGS = "-Wl,-rpath,'$(build_libdir)'" endif -pcre2-$(PCRE_VER).tar.bz2: +$(SRCDIR)/srccache/pcre2-$(PCRE_VER).tar.bz2: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre2-$(PCRE_VER).tar.bz2 -pcre2-$(PCRE_VER)/configure: pcre2-$(PCRE_VER).tar.bz2 +$(SRCDIR)/srccache/pcre2-$(PCRE_VER)/configure: $(SRCDIR)/srccache/pcre2-$(PCRE_VER).tar.bz2 $(JLCHECKSUM) $< - $(TAR) jxf $< + cd $(dir $<) && $(TAR) jxf $(notdir $<) touch -c $@ -pcre2-$(PCRE_VER)/config.status: pcre2-$(PCRE_VER)/configure - cd pcre2-$(PCRE_VER) && \ - ./configure $(CONFIGURE_COMMON) --enable-jit --includedir=$(build_includedir) CFLAGS="$(CFLAGS) $(PCRE_CFLAGS)" LDFLAGS="$(LDFLAGS) $(PCRE_LDFLAGS)" +$(BUILDDIR)/pcre2-$(PCRE_VER)/config.status: $(SRCDIR)/srccache/pcre2-$(PCRE_VER)/configure + mkdir -p $(dir $@) + cd $(dir $@) && \ + $< $(CONFIGURE_COMMON) --enable-jit --includedir=$(build_includedir) CFLAGS="$(CFLAGS) $(PCRE_CFLAGS)" LDFLAGS="$(LDFLAGS) $(PCRE_LDFLAGS)" touch -c $@ -$(PCRE_SRC_TARGET): pcre2-$(PCRE_VER)/config.status - $(MAKE) -C pcre2-$(PCRE_VER) $(LIBTOOL_CCLD) +$(PCRE_SRC_TARGET): $(BUILDDIR)/pcre2-$(PCRE_VER)/config.status + $(MAKE) -C $(dir $<) $(LIBTOOL_CCLD) touch -c $@ -pcre2-$(PCRE_VER)/checked: $(PCRE_SRC_TARGET) +$(BUILDDIR)/pcre2-$(PCRE_VER)/checked: $(PCRE_SRC_TARGET) ifeq ($(OS),$(BUILD_OS)) ifneq ($(OS),WINNT) - $(MAKE) -C pcre2-$(PCRE_VER) check -j1 + $(MAKE) -C $(dir $@) check -j1 endif endif echo 1 > $@ -$(PCRE_OBJ_TARGET): $(PCRE_SRC_TARGET) pcre2-$(PCRE_VER)/checked +$(PCRE_OBJ_TARGET): $(PCRE_SRC_TARGET) $(BUILDDIR)/pcre2-$(PCRE_VER)/checked $(call make-install,pcre2-$(PCRE_VER),$(LIBTOOL_CCLD)) $(INSTALL_NAME_CMD)libpcre2-8.$(SHLIB_EXT) $@ touch -c $@ @@ -819,29 +856,29 @@ clean-pcre: -$(MAKE) -C pcre2-$(PCRE_VER) clean -rm -f $(build_shlibdir)/libpcre* distclean-pcre: clean-pcre - -rm -rf pcre2-$(PCRE_VER).tar.bz2 pcre2-$(PCRE_VER) + -rm -rf $(SRCDIR)/srccache/pcre2-$(PCRE_VER).tar.bz2 $(SRCDIR)/srccache/pcre2-$(PCRE_VER) -get-pcre: pcre2-$(PCRE_VER).tar.bz2 -configure-pcre: pcre2-$(PCRE_VER)/config.status +get-pcre: $(SRCDIR)/srccache/pcre2-$(PCRE_VER).tar.bz2 +configure-pcre: $(BUILDDIR)/pcre2-$(PCRE_VER)/config.status compile-pcre: $(PCRE_SRC_TARGET) -check-pcre: pcre2-$(PCRE_VER)/checked +check-pcre: $(BUILDDIR)/pcre2-$(PCRE_VER)/checked install-pcre: $(PCRE_OBJ_TARGET) ## openlibm ## OPENLIBM_GIT_URL = git://github.com/JuliaLang/openlibm.git OPENLIBM_TAR_URL = https://api.github.com/repos/JuliaLang/openlibm/tarball/$1 -$(eval $(call git-external,openlibm,OPENLIBM,Makefile,libopenlibm.$(SHLIB_EXT))) +$(eval $(call git-external,openlibm,OPENLIBM,Makefile,libopenlibm.$(SHLIB_EXT),$(BUILDDIR))) OPENLIBM_OBJ_TARGET = $(build_shlibdir)/libopenlibm.$(SHLIB_EXT) $(build_libdir)/libopenlibm.a -OPENLIBM_OBJ_SOURCE = $(OPENLIBM_SRC_DIR)/libopenlibm.$(SHLIB_EXT) +OPENLIBM_OBJ_SOURCE = $(BUILDDIR)/$(OPENLIBM_SRC_DIR)/libopenlibm.$(SHLIB_EXT) OPENLIBM_FLAGS = ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) ifeq (CYGWIN,$(findstring CYGWIN,$(BUILD_OS))) OPENLIBM_FLAGS += OPENLIBM_HOME="$(call cygpath_w,$(abspath $(OPENLIBM_SRC_DIR)))" endif -$(OPENLIBM_OBJ_SOURCE): $(OPENLIBM_SRC_DIR)/Makefile - $(MAKE) -C $(OPENLIBM_SRC_DIR) $(OPENLIBM_FLAGS) $(MAKE_COMMON) +$(OPENLIBM_OBJ_SOURCE): $(BUILDDIR)/$(OPENLIBM_SRC_DIR)/Makefile + $(MAKE) -C $(dir $<) $(OPENLIBM_FLAGS) $(MAKE_COMMON) touch -c $@ $(build_shlibdir)/libopenlibm%$(SHLIB_EXT) $(build_libdir)/libopenlibm%a: $(OPENLIBM_OBJ_SOURCE) $(call make-install,$(OPENLIBM_SRC_DIR),$(OPENLIBM_FLAGS)) @@ -849,20 +886,21 @@ $(build_shlibdir)/libopenlibm%$(SHLIB_EXT) $(build_libdir)/libopenlibm%a: $(OPEN touch -c $(OPENLIBM_OBJ_TARGET) clean-openlibm: - -$(MAKE) -C $(OPENLIBM_SRC_DIR) distclean $(OPENLIBM_FLAGS) + -$(MAKE) -C $(BUILDDIR)/$(OPENLIBM_SRC_DIR) distclean $(OPENLIBM_FLAGS) -rm $(OPENLIBM_OBJ_TARGET) -rm $(build_libdir)/libopenlibm.a -get-openlibm: $(OPENLIBM_SRC_DIR)/Makefile -configure-openlibm: get-openlibm +get-openlibm: $(SRCDIR)/srccache/$(OPENLIBM_SRC_DIR).git +configure-openlibm: $(BUILDDIR)/$(OPENLIBM_SRC_DIR)/Makefile compile-openlibm: $(OPENLIBM_OBJ_SOURCE) check-openlibm: compile-openlibm install-openlibm: $(OPENLIBM_OBJ_TARGET) + ## openspecfun ## OPENSPECFUN_GIT_URL = git://github.com/JuliaLang/openspecfun.git OPENSPECFUN_TAR_URL = https://api.github.com/repos/JuliaLang/openspecfun/tarball/$1 -$(eval $(call git-external,openspecfun,OPENSPECFUN,Makefile,libopenspecfun.$(SHLIB_EXT))) +$(eval $(call git-external,openspecfun,OPENSPECFUN,Makefile,libopenspecfun.$(SHLIB_EXT),$(BUILDDIR))) # issue 8799 OPENSPECFUN_CFLAGS = -O3 -std=c99 @@ -871,7 +909,7 @@ ifeq ($(USEICC),1) endif OPENSPECFUN_OBJ_TARGET = $(build_shlibdir)/libopenspecfun.$(SHLIB_EXT) -OPENSPECFUN_OBJ_SOURCE = $(OPENSPECFUN_SRC_DIR)/libopenspecfun.$(SHLIB_EXT) +OPENSPECFUN_OBJ_SOURCE = $(BUILDDIR)/$(OPENSPECFUN_SRC_DIR)/libopenspecfun.$(SHLIB_EXT) OPENSPECFUN_FLAGS = ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) FFLAGS="$(JFFLAGS)" CFLAGS="$(CFLAGS) $(OPENSPECFUN_CFLAGS)" ifeq ($(USE_SYSTEM_LIBM),0) @@ -879,8 +917,8 @@ ifeq ($(USE_SYSTEM_LIBM),0) $(OPENSPECFUN_OBJ_SOURCE): $(OPENLIBM_OBJ_TARGET) endif -$(OPENSPECFUN_OBJ_SOURCE): $(OPENSPECFUN_SRC_DIR)/Makefile - $(MAKE) -C $(OPENSPECFUN_SRC_DIR) $(OPENSPECFUN_FLAGS) $(MAKE_COMMON) +$(OPENSPECFUN_OBJ_SOURCE): $(BUILDDIR)/$(OPENSPECFUN_SRC_DIR)/Makefile + $(MAKE) -C $(dir $<) $(OPENSPECFUN_FLAGS) $(MAKE_COMMON) touch -c $@ $(OPENSPECFUN_OBJ_TARGET): $(OPENSPECFUN_OBJ_SOURCE) $(call make-install,$(OPENSPECFUN_SRC_DIR),$(OPENSPECFUN_FLAGS)) @@ -888,20 +926,21 @@ $(OPENSPECFUN_OBJ_TARGET): $(OPENSPECFUN_OBJ_SOURCE) touch -c $@ clean-openspecfun: - -$(MAKE) -C $(OPENSPECFUN_SRC_DIR) distclean $(OPENSPECFUN_FLAGS) + -$(MAKE) -C $(BUILDDIR)/$(OPENSPECFUN_SRC_DIR) distclean $(OPENSPECFUN_FLAGS) -rm $(OPENSPECFUN_OBJ_TARGET) -rm $(build_libdir)/libopenspecfun.a -get-openspecfun: $(OPENSPECFUN_SRC_DIR)/Makefile -configure-openspecfun: get-openspecfun +get-openspecfun: $(SRCDIR)/srccache/$(OPENSPECFUN_SRC_DIR).git +configure-openspecfun: $(BUILDDIR)/$(OPENSPECFUN_SRC_DIR)/Makefile compile-openspecfun: $(OPENSPECFUN_OBJ_SOURCE) check-openspecfun: compile-openspecfun install-openspecfun: $(OPENSPECFUN_OBJ_TARGET) + ## DSFMT ## DSFMT_OBJ_TARGET = $(build_shlibdir)/libdSFMT.$(SHLIB_EXT) $(build_includedir)/dSFMT.h -DSFMT_OBJ_SOURCE = dsfmt-$(DSFMT_VER)/libdSFMT.$(SHLIB_EXT) +DSFMT_OBJ_SOURCE = $(BUILDDIR)/dsfmt-$(DSFMT_VER)/libdSFMT.$(SHLIB_EXT) DSFMT_CFLAGS = $(CFLAGS) -DNDEBUG -DDSFMT_MEXP=19937 $(fPIC) -DDSFMT_DO_NOT_USE_OLD_NAMES ifneq ($(USEMSVC), 1) @@ -914,30 +953,31 @@ ifeq ($(ARCH), x86_64) DSFMT_CFLAGS += -msse2 -DHAVE_SSE2 endif -dsfmt-$(DSFMT_VER).tar.gz: +$(SRCDIR)/srccache/dsfmt-$(DSFMT_VER).tar.gz: $(JLDOWNLOAD) $@ http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/dSFMT-src-$(DSFMT_VER).tar.gz touch -c $@ -dsfmt-$(DSFMT_VER)/config.status: dsfmt-$(DSFMT_VER).tar.gz +$(BUILDDIR)/dsfmt-$(DSFMT_VER)/config.status: $(SRCDIR)/srccache/dsfmt-$(DSFMT_VER).tar.gz $(JLCHECKSUM) $< - mkdir -p dsfmt-$(DSFMT_VER) && \ - $(TAR) -C dsfmt-$(DSFMT_VER) --strip-components 1 -xf dsfmt-$(DSFMT_VER).tar.gz && \ - cd dsfmt-$(DSFMT_VER) && patch < ../dSFMT.h.patch && patch < ../dSFMT.c.patch + mkdir -p $(dir $@) && \ + $(TAR) -C $(dir $@) --strip-components 1 -xf $< && \ + cd $(dir $@) && patch < $(SRCDIR)/dSFMT.h.patch + cd $(dir $@) && patch < $(SRCDIR)/dSFMT.c.patch echo 1 > $@ -$(DSFMT_OBJ_SOURCE): dsfmt-$(DSFMT_VER)/config.status - cd dsfmt-$(DSFMT_VER) && \ +$(DSFMT_OBJ_SOURCE): $(BUILDDIR)/dsfmt-$(DSFMT_VER)/config.status + cd $(dir $<) && \ $(CC) $(CPPFLAGS) $(DSFMT_CFLAGS) $(LDFLAGS) dSFMT.c -o libdSFMT.$(SHLIB_EXT) $(build_shlibdir)/libdSFMT%$(SHLIB_EXT) $(build_includedir)/dSFMT%h: $(DSFMT_OBJ_SOURCE) | $(build_includedir) $(build_shlibdir) - cp dsfmt-$(DSFMT_VER)/dSFMT.h $(build_includedir) + cp $(dir $<)/dSFMT.h $(build_includedir) cp $< $(build_shlibdir)/libdSFMT.$(SHLIB_EXT) && \ $(INSTALL_NAME_CMD)libdSFMT.$(SHLIB_EXT) $(build_shlibdir)/libdSFMT.$(SHLIB_EXT) clean-dsfmt: - -rm -f dsfmt-$(DSFMT_VER)/libdSFMT.$(SHLIB_EXT) + -rm -f $(BUILDDIR)/dsfmt-$(DSFMT_VER)/libdSFMT.$(SHLIB_EXT) distclean-dsfmt: - -rm -rf dsfmt*.tar.gz dsfmt-$(DSFMT_VER) + -rm -rf $(SRCDIR)/srccache/dsfmt*.tar.gz $(SRCDIR)/srccache/dsfmt-$(DSFMT_VER) -get-dsfmt: dsfmt-$(DSFMT_VER).tar.gz -configure-dsfmt: dsfmt-$(DSFMT_VER)/config.status +get-dsfmt: $(SRCDIR)/srccache/dsfmt-$(DSFMT_VER).tar.gz +configure-dsfmt: $(BUILDDIR)/dsfmt-$(DSFMT_VER)/config.status compile-dsfmt: $(DSFMT_OBJ_SOURCE) check-dsfmt: compile-dsfmt install-dsfmt: $(DSFMT_OBJ_TARGET) @@ -946,7 +986,7 @@ install-dsfmt: $(DSFMT_OBJ_TARGET) ## Rmath-julia ## RMATH_JULIA_OBJ_TARGET = $(build_shlibdir)/libRmath-julia.$(SHLIB_EXT) -RMATH_JULIA_OBJ_SOURCE = Rmath-julia-$(RMATH_JULIA_VER)/src/libRmath-julia.$(SHLIB_EXT) +RMATH_JULIA_OBJ_SOURCE = $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER)/src/libRmath-julia.$(SHLIB_EXT) ifeq ($(USE_SYSTEM_DSFMT),0) $(RMATH_JULIA_OBJ_SOURCE): $(DSFMT_OBJ_TARGET) @@ -957,25 +997,27 @@ RMATH_JULIA_FLAGS += CC="$(CC)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) \ USE_DSFMT=1 DSFMT_libdir="$(build_shlibdir)" \ DSFMT_includedir="$(build_includedir)" -Rmath-julia-$(RMATH_JULIA_VER).tar.gz: +$(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER).tar.gz: $(JLDOWNLOAD) $@ https://api.github.com/repos/JuliaLang/Rmath-julia/tarball/v$(RMATH_JULIA_VER) -$(RMATH_JULIA_OBJ_SOURCE): Rmath-julia-$(RMATH_JULIA_VER).tar.gz - $(JLCHECKSUM) Rmath-julia-$(RMATH_JULIA_VER).tar.gz && \ - mkdir -p Rmath-julia-$(RMATH_JULIA_VER) && \ - $(TAR) -C Rmath-julia-$(RMATH_JULIA_VER) --strip-components 1 -xf $< && \ - $(MAKE) -C Rmath-julia-$(RMATH_JULIA_VER)/src $(RMATH_JULIA_FLAGS) $(MAKE_COMMON) && \ +$(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER)/Makefile: $(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER).tar.gz + $(JLCHECKSUM) $< + mkdir -p $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER) + $(TAR) -C $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER) --strip-components 1 -xf $< + touch -c $@ +$(RMATH_JULIA_OBJ_SOURCE): $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER)/Makefile + $(MAKE) -C $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER)/src $(RMATH_JULIA_FLAGS) $(MAKE_COMMON) touch -c $@ $(RMATH_JULIA_OBJ_TARGET): $(RMATH_JULIA_OBJ_SOURCE) | $(build_shlibdir) cp $< $@ $(INSTALL_NAME_CMD)libRmath-julia.$(SHLIB_EXT) $@ clean-Rmath-julia: - -$(MAKE) -C Rmath-julia-$(RMATH_JULIA_VER) clean + -$(MAKE) -C $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER) clean distclean-Rmath-julia: clean-Rmath-julia - -rm -rf Rmath-julia-$(RMATH_JULIA_VER).tar.gz Rmath-julia-$(RMATH_JULIA_VER) + -rm -rf $(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER).tar.gz $(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER) -get-Rmath-julia: Rmath-julia-$(RMATH_JULIA_VER).tar.gz -configure-Rmath-julia: get-Rmath-julia +get-Rmath-julia: $(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER).tar.gz +configure-Rmath-julia: $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER)/Makefile compile-Rmath-julia: $(RMATH_JULIA_OBJ_SOURCE) check-Rmath-julia: compile-Rmath-julia install-Rmath-julia: $(RMATH_JULIA_OBJ_TARGET) @@ -983,27 +1025,28 @@ install-Rmath-julia: $(RMATH_JULIA_OBJ_TARGET) ## objconv ## -OBJCONV_SOURCE = objconv/objconv +OBJCONV_SOURCE = $(BUILDDIR)/objconv/objconv OBJCONV_TARGET = $(build_bindir)/objconv -objconv.zip: +$(SRCDIR)/srccache/objconv.zip: $(JLDOWNLOAD) $@ http://www.agner.org/optimize/objconv.zip -objconv/config.status: objconv.zip - unzip -d objconv $< - cd objconv && unzip source.zip +$(BUILDDIR)/objconv/config.status: $(SRCDIR)/srccache/objconv.zip + -rm -r $(dir $@) + unzip -d $(dir $@) $< + cd $(dir $@) && unzip source.zip echo 1 > $@ -$(OBJCONV_SOURCE): objconv/config.status - cd objconv && $(CXX) -o objconv -O2 *.cpp +$(OBJCONV_SOURCE): $(BUILDDIR)/objconv/config.status + cd $(dir $<) && $(CXX) -o objconv -O2 *.cpp $(OBJCONV_TARGET): $(OBJCONV_SOURCE) | $(build_bindir) cp -f $< $@ clean-objconv: -rm -f $(OBJCONV_TARGET) distclean-objconv: - -rm -rf objconv.zip objconv + -rm -rf $(SRCDIR)/srccache/objconv.zip $(BUILDDIR)/objconv -get-objconv: objconv.zip -configure-objconv: objconv/config.status +get-objconv: $(SRCDIR)/srccache/objconv.zip +configure-objconv: $(BUILDDIR)/objconv/config.status compile-objconv: $(OBJCONV_SOURCE) check-objconv: compile-objconv install-objconv: $(OBJCONV_TARGET) @@ -1013,9 +1056,9 @@ install-objconv: $(OBJCONV_TARGET) # LAPACK is built into OpenBLAS by default OPENBLAS_GIT_URL = git://github.com/xianyi/OpenBLAS.git OPENBLAS_TAR_URL = https://api.github.com/repos/xianyi/OpenBLAS/tarball/$1 -$(eval $(call git-external,openblas,OPENBLAS,Makefile,libopenblas.$(SHLIB_EXT))) +$(eval $(call git-external,openblas,OPENBLAS,Makefile,libopenblas.$(SHLIB_EXT),$(BUILDDIR))) -OPENBLAS_OBJ_SOURCE = $(OPENBLAS_SRC_DIR)/libopenblas.$(SHLIB_EXT) +OPENBLAS_OBJ_SOURCE = $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/libopenblas.$(SHLIB_EXT) OPENBLAS_OBJ_TARGET = $(build_shlibdir)/libopenblas.$(SHLIB_EXT) OPENBLAS_BUILD_OPTS = CC="$(CC)" FC="$(FC)" RANLIB="$(RANLIB)" FFLAGS="$(FFLAGS) $(JFFLAGS)" TARGET=$(OPENBLAS_TARGET_ARCH) BINARY=$(BINARY) @@ -1054,7 +1097,7 @@ endif ifeq ($(USE_BLAS64), 1) OPENBLAS_BUILD_OPTS += INTERFACE64=1 SYMBOLSUFFIX="$(OPENBLAS_SYMBOLSUFFIX)" ifeq ($(OS), Darwin) -OPENBLAS_BUILD_OPTS += OBJCONV=$(JULIAHOME)/deps/objconv/objconv +OPENBLAS_BUILD_OPTS += OBJCONV=$(abspath $(BUILDDIR)/objconv/objconv) $(OPENBLAS_OBJ_SOURCE): $(OBJCONV_SOURCE) endif endif @@ -1082,26 +1125,26 @@ else ifeq ($(OPENBLAS_NO_AVX2), 1) OPENBLAS_BUILD_OPTS += NO_AVX2=1 endif -$(OPENBLAS_SRC_DIR)/config.status: $(OPENBLAS_SRC_DIR)/Makefile - perl -i -ple 's/^\s*(EXTRALIB\s*\+=\s*-lSystemStubs)\s*$$/# $$1/g' $(OPENBLAS_SRC_DIR)/Makefile.system +$(BUILDDIR)/$(OPENBLAS_SRC_DIR)/config.status: $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/Makefile + perl -i -ple 's/^\s*(EXTRALIB\s*\+=\s*-lSystemStubs)\s*$$/# $$1/g' $<.system touch $@ -$(OPENBLAS_OBJ_SOURCE): $(OPENBLAS_SRC_DIR)/config.status - echo $(MAKE) -C $(OPENBLAS_SRC_DIR) $(OPENBLAS_BUILD_OPTS) # echo first, so we only print the error message below in a failure case - @$(MAKE) -C $(OPENBLAS_SRC_DIR) $(OPENBLAS_BUILD_OPTS) || (echo $(WARNCOLOR)"*** Clean the OpenBLAS build with 'make -C deps clean-openblas'. Rebuild with 'make OPENBLAS_USE_THREAD=0 if OpenBLAS had trouble linking libpthread.so, and with 'make OPENBLAS_TARGET_ARCH=NEHALEM' if there were errors building SandyBridge support. Both these options can also be used simultaneously. ***"$(ENDCOLOR) && false) +$(OPENBLAS_OBJ_SOURCE): $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/config.status + echo $(MAKE) -C $(dir $<) $(OPENBLAS_BUILD_OPTS) # echo first, so we only print the error message below in a failure case + @$(MAKE) -C $(dir $<) $(OPENBLAS_BUILD_OPTS) || (echo $(WARNCOLOR)"*** Clean the OpenBLAS build with 'make -C deps clean-openblas'. Rebuild with 'make OPENBLAS_USE_THREAD=0 if OpenBLAS had trouble linking libpthread.so, and with 'make OPENBLAS_TARGET_ARCH=NEHALEM' if there were errors building SandyBridge support. Both these options can also be used simultaneously. ***"$(ENDCOLOR) && false) touch -c $@ $(OPENBLAS_OBJ_TARGET): $(OPENBLAS_OBJ_SOURCE) | $(build_shlibdir) - cp -f $(OPENBLAS_SRC_DIR)/libopenblas.$(SHLIB_EXT) $(build_shlibdir) + cp -f $< $@ ifeq ($(OS), Linux) - cd $(build_shlibdir) && \ + cd $(dir $@) && \ ln -sf libopenblas.$(SHLIB_EXT) libopenblas.$(SHLIB_EXT).0 endif - $(INSTALL_NAME_CMD)libopenblas.$(SHLIB_EXT) $(build_shlibdir)/libopenblas.$(SHLIB_EXT) + $(INSTALL_NAME_CMD)libopenblas.$(SHLIB_EXT) $@ clean-openblas: - -$(MAKE) -C $(OPENBLAS_SRC_DIR) clean + -$(MAKE) -C $(BUILDDIR)/$(OPENBLAS_SRC_DIR) clean -get-openblas: $(OPENBLAS_SRC_DIR)/Makefile -configure-openblas: $(OPENBLAS_SRC_DIR)/config.status +get-openblas: $(SRCDIR)/srccache/$(OPENBLAS_SRC_DIR).git +configure-openblas: $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/config.status compile-openblas: $(OPENBLAS_OBJ_SOURCE) check-openblas: compile-openblas install-openblas: $(OPENBLAS_OBJ_TARGET) @@ -1113,7 +1156,7 @@ install-openblas: $(OPENBLAS_OBJ_TARGET) # should always be compiled with (a real) gcc, it's # configure script will search for the best match # (gcc 4.7, gcc, clang,ICC/microsoft/others) -ATLAS_OBJ_SOURCE = atlas/build/lib/libsatlas.$(SHLIB_EXT) +ATLAS_OBJ_SOURCE = $(BUILDDIR)/atlas/build/lib/libsatlas.$(SHLIB_EXT) ATLAS_OBJ_TARGET = $(build_shlibdir)/libsatlas.$(SHLIB_EXT) ATLAS_FLAGS = --shared --prefix=$(build_prefix) --cc=gcc -t 0 \ --with-netlib-lapack-tarfile=$(JULIAHOME)/deps/lapack-$(LAPACK_VER).tgz @@ -1127,18 +1170,18 @@ endif #ATLAS_FLAGS += -V -1 -A 11 # any x87 (PentiumPro or Athlon & later) #ATLAS_FLAGS += -A 25 # requires Corei132 (Corei232 doesn't have definition yet) -atlas/configure: - git clone git://github.com/vtjnash/atlas-3.10.0.git atlas +$(SRCDIR)/srccache/atlas/configure: + git clone git://github.com/vtjnash/atlas-3.10.0.git $(SRCDIR)/srccache/atlas ifeq "$(MAKECMDGOALS)" "compile-atlas" # only allow building atlas as the sole target (without -jN) # since it internally handles parallelism, for tuning timing accuracy -atlas/build/Make.top: atlas/configure lapack-$(LAPACK_VER).tgz - cd atlas && \ - (test -d build || mkdir build) && cd build && \ - ../configure $(ATLAS_FLAGS) +$(BUILDDIR)/atlas/Make.top: $(SRCDIR)/srccache/atlas/configure $(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz + mkdir -p $(dir $@) + cd $(dir $@) && \ + $< $(ATLAS_FLAGS) touch -c $@ -$(ATLAS_OBJ_SOURCE): atlas/build/Make.top - $(MAKE) -C atlas/build -j1 +$(ATLAS_OBJ_SOURCE): $(BUILDDIR)/atlas/Make.top + $(MAKE) -C $(dir $<) -j1 touch -c $@ else $(ATLAS_OBJ_SOURCE): @@ -1154,8 +1197,8 @@ clean-atlas: distclean-atlas: rm -rf atlas -get-atlas: atlas/configure -configure-atlas: atlas/build/Make.top +get-atlas: $(SRCDIR)/srccache/atlas/configure $(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz +configure-atlas: $(BUILDDIR)/atlas/Make.top compile-atlas: $(ATLAS_OBJ_SOURCE) check-atlas: compile-atlas install-atlas: $(ATLAS_OBJ_TARGET) @@ -1180,10 +1223,10 @@ GFORTBLAS_FFLAGS += -cpp -ffree-line-length-0 -ffixed-line-length-0 \ endif endif -libgfortblas.$(SHLIB_EXT): gfortblas.c gfortblas.alias +$(BUILDDIR)/libgfortblas.$(SHLIB_EXT): $(SRCDIR)/gfortblas.c $(SRCDIR)/gfortblas.alias $(CC) -Wall -O3 $(CPPFLAGS) $(CFLAGS) $(fPIC) -shared $< -o $@ -pipe \ - -Wl,-reexport_framework,Accelerate -Wl,-alias_list,gfortblas.alias -$(build_shlibdir)/libgfortblas.$(SHLIB_EXT): libgfortblas.$(SHLIB_EXT) + -Wl,-reexport_framework,Accelerate -Wl,-alias_list,$(SRCDIR)/gfortblas.alias +$(build_shlibdir)/libgfortblas.$(SHLIB_EXT): $(BUILDDIR)/libgfortblas.$(SHLIB_EXT) cp -f $< $@ $(INSTALL_NAME_CMD)libgfortblas.$(SHLIB_EXT) $@ endif @@ -1192,7 +1235,7 @@ endif ifeq ($(USE_SYSTEM_LAPACK), 0) LAPACK_OBJ_TARGET = $(build_shlibdir)/liblapack.$(SHLIB_EXT) -LAPACK_OBJ_SOURCE = lapack-$(LAPACK_VER)/liblapack.$(SHLIB_EXT) +LAPACK_OBJ_SOURCE = $(BUILDDIR)/lapack-$(LAPACK_VER)/liblapack.$(SHLIB_EXT) else LAPACK_OBJ_TARGET = LAPACK_OBJ_SOURCE = @@ -1203,29 +1246,26 @@ ifneq ($(OS),WINNT) LAPACK_MFLAGS += BLASLIB="-Wl,-rpath,'$(build_libdir)' $(LIBBLAS)" endif -lapack-$(LAPACK_VER).tgz: - $(JLDOWNLOAD) $@ http://www.netlib.org/lapack/$@ -lapack-$(LAPACK_VER)/Makefile: lapack-$(LAPACK_VER).tgz +$(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz: + $(JLDOWNLOAD) $@ http://www.netlib.org/lapack/$(notdir $@) +$(BUILDDIR)/lapack-$(LAPACK_VER)/make.inc: $(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz $(JLCHECKSUM) $< - $(TAR) zxf $< + cd $(BUILDDIR) && $(TAR) zxf $< + cp $(dir $@)INSTALL/make.inc.gfortran $(dir $@)make.inc touch -c $@ ifeq ($(USE_SYSTEM_BLAS), 0) -lapack-$(LAPACK_VER)/liblapack.a: | $(OPENBLAS_OBJ_TARGET) +$(BUILDDIR)/lapack-$(LAPACK_VER)/liblapack.a: | $(OPENBLAS_OBJ_TARGET) else ifeq ($(OS),Darwin) -lapack-$(LAPACK_VER)/liblapack.a: | $(build_shlibdir)/libgfortblas.$(SHLIB_EXT) +$(BUILDDIR)/lapack-$(LAPACK_VER)/liblapack.a: | $(build_shlibdir)/libgfortblas.$(SHLIB_EXT) endif -lapack-$(LAPACK_VER)/liblapack.a: lapack-$(LAPACK_VER)/Makefile - cd lapack-$(LAPACK_VER) && \ - cp INSTALL/make.inc.gfortran ./make.inc && \ - $(MAKE) lapacklib $(LAPACK_MFLAGS) +$(BUILDDIR)/lapack-$(LAPACK_VER)/liblapack.a: $(BUILDDIR)/lapack-$(LAPACK_VER)/make.inc + $(MAKE) -C $(dir $@) lapacklib $(LAPACK_MFLAGS) touch -c $@ -lapack-$(LAPACK_VER)/checked: lapack-$(LAPACK_VER)/liblapack.a - cd lapack-$(LAPACK_VER) && \ - $(MAKE) lapack_testing $(LAPACK_MFLAGS) -k +$(BUILDDIR)/lapack-$(LAPACK_VER)/checked: $(BUILDDIR)/lapack-$(LAPACK_VER)/liblapack.a + $(MAKE) -C $(dir $@) lapack_testing $(LAPACK_MFLAGS) -k touch $@ -$(LAPACK_OBJ_SOURCE): lapack-$(LAPACK_VER)/liblapack.a - cd lapack-$(LAPACK_VER) && \ - $(FC) -shared $(FFLAGS) $(JFFLAGS) SRC/*.o INSTALL/dlamch.o INSTALL/dsecnd_INT_ETIME.o INSTALL/ilaver.o INSTALL/slamch.o $(LIBBLAS) -o liblapack.$(SHLIB_EXT) +$(LAPACK_OBJ_SOURCE): $(BUILDDIR)/lapack-$(LAPACK_VER)/liblapack.a + $(FC) -shared $(FFLAGS) $(JFFLAGS) $(dir $<)/SRC/*.o $(dir $<)/INSTALL/dlamch.o $(dir $<)/INSTALL/dsecnd_INT_ETIME.o $(dir $<)/INSTALL/ilaver.o $(dir $<)/INSTALL/slamch.o $(LIBBLAS) -o $@ $(LAPACK_OBJ_TARGET): $(LAPACK_OBJ_SOURCE) cp $< $@ $(INSTALL_NAME_CMD)liblapack.$(SHLIB_EXT) $@ @@ -1236,10 +1276,10 @@ clean-lapack: distclean-lapack: -rm -rf lapack-$(LAPACK_VER).tgz lapack-$(LAPACK_VER) -get-lapack: lapack-$(LAPACK_VER).tgz -configure-lapack: get-lapack +get-lapack: $(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz +configure-lapack: $(BUILDDIR)/lapack-$(LAPACK_VER)/make.inc compile-lapack: $(LAPACK_OBJ_SOURCE) -check-lapack: lapack-$(LAPACK_VER)/checked +check-lapack: $(BUILDDIR)/lapack-$(LAPACK_VER)/checked install-lapack: $(LAPACK_OBJ_TARGET) @@ -1271,9 +1311,9 @@ endif endif ifeq ($(OS),WINNT) -ARPACK_OBJ_SOURCE = arpack-ng-$(ARPACK_VER)/.libs/libarpack-2.$(SHLIB_EXT) +ARPACK_OBJ_SOURCE = $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/.libs/libarpack-2.$(SHLIB_EXT) else -ARPACK_OBJ_SOURCE = arpack-ng-$(ARPACK_VER)/.libs/libarpack.$(SHLIB_EXT) +ARPACK_OBJ_SOURCE = $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/.libs/libarpack.$(SHLIB_EXT) endif ARPACK_OBJ_TARGET = $(build_shlibdir)/libarpack.$(SHLIB_EXT) @@ -1285,38 +1325,40 @@ ARPACK_FLAGS += LDFLAGS="$(LDFLAGS) -Wl,-rpath,'$(build_libdir)'" endif # ARPACK-NG upstream keeps changing their download filenames -arpack-ng-$(ARPACK_VER).tar.gz: +$(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER).tar.gz: $(JLDOWNLOAD) $@ https://github.com/opencollab/arpack-ng/archive/$(ARPACK_VER).tar.gz touch -c $@ -arpack-ng-$(ARPACK_VER)/configure: arpack-ng-$(ARPACK_VER).tar.gz +$(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)-testA.mtx: + $(JLDOWNLOAD) $@ https://raw.githubusercontent.com/opencollab/arpack-ng/$(ARPACK_VER)/TESTS/testA.mtx + touch -c $@ +$(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)/configure: $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER).tar.gz $(JLCHECKSUM) $< - $(TAR) zxf $< + cd $(dir $<) && $(TAR) zxf $< touch -c $@ ifeq ($(USE_ATLAS), 1) -arpack-ng-$(ARPACK_VER)/config.status: | $(ATLAS_OBJ_TARGET) +$(BUILDDIR)/arpack-ng-$(ARPACK_VER)/config.status: | $(ATLAS_OBJ_TARGET) endif ifeq ($(USE_SYSTEM_BLAS), 0) -arpack-ng-$(ARPACK_VER)/config.status: | $(OPENBLAS_OBJ_TARGET) +$(BUILDDIR)/arpack-ng-$(ARPACK_VER)/config.status: | $(OPENBLAS_OBJ_TARGET) else ifeq ($(USE_SYSTEM_LAPACK), 0) -arpack-ng-$(ARPACK_VER)/config.status: | $(LAPACK_OBJ_TARGET) +$(BUILDDIR)/arpack-ng-$(ARPACK_VER)/config.status: | $(LAPACK_OBJ_TARGET) endif -arpack-ng-$(ARPACK_VER)/config.status: arpack-ng-$(ARPACK_VER)/configure - cd arpack-ng-$(ARPACK_VER) && \ - ./configure $(CONFIGURE_COMMON) $(ARPACK_FLAGS) +$(BUILDDIR)/arpack-ng-$(ARPACK_VER)/config.status: $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)/configure + mkdir -p $(dir $@) + cd $(dir $@) && \ + $< $(CONFIGURE_COMMON) $(ARPACK_FLAGS) touch -c $@ -$(ARPACK_OBJ_SOURCE): arpack-ng-$(ARPACK_VER)/config.status - cd arpack-ng-$(ARPACK_VER) && \ - $(MAKE) $(ARPACK_MFLAGS) +$(ARPACK_OBJ_SOURCE): $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/config.status + $(MAKE) -C $(dir $<) $(ARPACK_MFLAGS) touch -c $@ -arpack-ng-$(ARPACK_VER)/checked: $(ARPACK_OBJ_SOURCE) - cd arpack-ng-$(ARPACK_VER) -# $(MAKE) check $(ARPACK_MFLAGS) && \ -# cd TESTS && $(call spawn,./dnsimp$(EXE)) +$(BUILDDIR)/arpack-ng-$(ARPACK_VER)/checked: $(ARPACK_OBJ_SOURCE) $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)-testA.mtx + cp $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)-testA.mtx $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/TESTS + $(MAKE) -C $(dir $@) check $(ARPACK_MFLAGS) echo 1 > $@ -$(ARPACK_OBJ_TARGET): $(ARPACK_OBJ_SOURCE) arpack-ng-$(ARPACK_VER)/checked | $(build_shlibdir) +$(ARPACK_OBJ_TARGET): $(ARPACK_OBJ_SOURCE) $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/checked | $(build_shlibdir) $(call make-install,arpack-ng-$(ARPACK_VER),$(ARPACK_MFLAGS)) ifeq ($(OS), WINNT) mv $(build_shlibdir)/libarpack-2.dll $@ @@ -1338,20 +1380,20 @@ clean-arpack: distclean-arpack: -rm -rf arpack-ng-$(ARPACK_VER).tar.gz arpack-ng-$(ARPACK_VER) -get-arpack: arpack-ng-$(ARPACK_VER).tar.gz -configure-arpack: arpack-ng-$(ARPACK_VER)/config.status +get-arpack: $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER).tar.gz $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)-testA.mtx +configure-arpack: $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/config.status compile-arpack: $(ARPACK_OBJ_SOURCE) -check-arpack: arpack-ng-$(ARPACK_VER)/checked +check-arpack: $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/checked install-arpack: $(ARPACK_OBJ_TARGET) ## FFTW ## ifeq ($(OS),WINNT) -FFTW_SINGLE_SRC_TARGET = fftw-$(FFTW_VER)-single/.libs/libfftw3f-3.$(SHLIB_EXT) -FFTW_DOUBLE_SRC_TARGET = fftw-$(FFTW_VER)-double/.libs/libfftw3-3.$(SHLIB_EXT) +FFTW_SINGLE_SRC_TARGET = $(BUILDDIR)/fftw-$(FFTW_VER)-single/.libs/libfftw3f-3.$(SHLIB_EXT) +FFTW_DOUBLE_SRC_TARGET = $(BUILDDIR)/fftw-$(FFTW_VER)-double/.libs/libfftw3-3.$(SHLIB_EXT) else -FFTW_SINGLE_SRC_TARGET = fftw-$(FFTW_VER)-single/.libs/libfftw3f.$(SHLIB_EXT) -FFTW_DOUBLE_SRC_TARGET = fftw-$(FFTW_VER)-double/.libs/libfftw3.$(SHLIB_EXT) +FFTW_SINGLE_SRC_TARGET = $(BUILDDIR)/fftw-$(FFTW_VER)-single/.libs/libfftw3f.$(SHLIB_EXT) +FFTW_DOUBLE_SRC_TARGET = $(BUILDDIR)/fftw-$(FFTW_VER)-double/.libs/libfftw3.$(SHLIB_EXT) endif FFTW_SINGLE_OBJ_TARGET = $(build_shlibdir)/libfftw3f.$(SHLIB_EXT) FFTW_DOUBLE_OBJ_TARGET = $(build_shlibdir)/libfftw3.$(SHLIB_EXT) @@ -1370,29 +1412,34 @@ ifneq ($(ARCH),x86_64) FFTW_CONFIG += --with-incoming-stack-boundary=2 endif endif +FFTW_ENABLE_single = --enable-single +FFTW_ENABLE_double = +FFTW_CONFIG += $(FFTW_ENABLE_$*) -fftw-$(FFTW_VER).tar.gz: - $(JLDOWNLOAD) $@ http://www.fftw.org/$@ - -fftw-$(FFTW_VER)-single/configure: fftw-$(FFTW_VER).tar.gz +$(SRCDIR)/srccache/fftw-$(FFTW_VER).tar.gz: + $(JLDOWNLOAD) $@ http://www.fftw.org/$(notdir $@) +$(SRCDIR)/srccache/fftw-$(FFTW_VER)/configure: $(SRCDIR)/srccache/fftw-$(FFTW_VER).tar.gz $(JLCHECKSUM) $< - mkdir -p fftw-$(FFTW_VER)-single && \ - $(TAR) -C fftw-$(FFTW_VER)-single --strip-components 1 -xf $< - touch -c $@ -fftw-$(FFTW_VER)-single/config.status: fftw-$(FFTW_VER)-single/configure - cd fftw-$(FFTW_VER)-single && \ - (./configure $(CONFIGURE_COMMON) $(FFTW_CONFIG) --enable-avx --enable-single || \ - ./configure $(CONFIGURE_COMMON) $(FFTW_CONFIG) --enable-single) && \ - $(MAKE) clean - touch -c $@ -$(FFTW_SINGLE_SRC_TARGET): fftw-$(FFTW_VER)-single/config.status - $(MAKE) -C fftw-$(FFTW_VER)-single -fftw-$(FFTW_VER)-single/checked: $(FFTW_SINGLE_SRC_TARGET) + mkdir -p $(dir $@) && \ + $(TAR) -C $(dir $@) --strip-components 1 -xf $< + touch -c $@ +$(BUILDDIR)/fftw-$(FFTW_VER)-%/config.status: $(SRCDIR)/srccache/fftw-$(FFTW_VER)/configure + mkdir -p $(dir $@) + # try to configure with avx support. if that fails, try again without it + cd $(dir $@) && \ + ($< $(CONFIGURE_COMMON) $(FFTW_CONFIG) --enable-avx || \ + $< $(CONFIGURE_COMMON) $(FFTW_CONFIG)) + $(MAKE) -C $(dir $@) clean + touch -c $@ + +$(FFTW_SINGLE_SRC_TARGET): $(BUILDDIR)/fftw-$(FFTW_VER)-single/config.status + $(MAKE) -C $(dir $<) +$(BUILDDIR)/fftw-$(FFTW_VER)-single/checked: $(FFTW_SINGLE_SRC_TARGET) ifeq ($(OS),$(BUILD_OS)) - $(MAKE) -C fftw-$(FFTW_VER)-single check + $(MAKE) -C $(dir $@) check endif echo 1 > $@ -$(FFTW_SINGLE_OBJ_TARGET): $(FFTW_SINGLE_SRC_TARGET) fftw-$(FFTW_VER)-single/checked +$(FFTW_SINGLE_OBJ_TARGET): $(FFTW_SINGLE_SRC_TARGET) $(BUILDDIR)/fftw-$(FFTW_VER)-single/checked $(call make-install,fftw-$(FFTW_VER)-single,) ifeq ($(OS), Darwin) $(INSTALL_NAME_CMD)libfftw3f.$(SHLIB_EXT) $(build_shlibdir)/libfftw3f.$(SHLIB_EXT) @@ -1407,26 +1454,14 @@ else ifeq ($(OS), Linux) endif touch -c $@ -fftw-$(FFTW_VER)-double/configure: fftw-$(FFTW_VER).tar.gz - $(JLCHECKSUM) $< - mkdir -p fftw-$(FFTW_VER)-double && \ - $(TAR) -C fftw-$(FFTW_VER)-double --strip-components 1 -xf $< - touch -c $@ -fftw-$(FFTW_VER)-double/config.status: fftw-$(FFTW_VER)-double/configure - cd fftw-$(FFTW_VER)-double && \ - (./configure $(CONFIGURE_COMMON) $(FFTW_CONFIG) --enable-avx || \ - ./configure $(CONFIGURE_COMMON) $(FFTW_CONFIG)) && \ - $(MAKE) clean - touch -c $@ -$(FFTW_DOUBLE_SRC_TARGET): fftw-$(FFTW_VER)-double/config.status - $(MAKE) -C fftw-$(FFTW_VER)-double - touch -c $@ -fftw-$(FFTW_VER)-double/checked: $(FFTW_DOUBLE_SRC_TARGET) +$(FFTW_DOUBLE_SRC_TARGET): $(BUILDDIR)/fftw-$(FFTW_VER)-double/config.status + $(MAKE) -C $(dir $<) +$(BUILDDIR)/fftw-$(FFTW_VER)-double/checked: $(FFTW_DOUBLE_SRC_TARGET) ifeq ($(OS),$(BUILD_OS)) - $(MAKE) -C fftw-$(FFTW_VER)-double check + $(MAKE) -C $(dir $@) check endif echo 1 > $@ -$(FFTW_DOUBLE_OBJ_TARGET): $(FFTW_DOUBLE_SRC_TARGET) fftw-$(FFTW_VER)-double/checked +$(FFTW_DOUBLE_OBJ_TARGET): $(FFTW_DOUBLE_SRC_TARGET) $(BUILDDIR)/fftw-$(FFTW_VER)-double/checked $(call make-install,fftw-$(FFTW_VER)-double,) ifeq ($(OS), Darwin) $(INSTALL_NAME_CMD)libfftw3.$(SHLIB_EXT) $(build_shlibdir)/libfftw3.$(SHLIB_EXT) @@ -1448,77 +1483,76 @@ endif clean-fftw: clean-fftw-single clean-fftw-double clean-fftw-single: - -$(MAKE) -C fftw-$(FFTW_VER)-single clean + -$(MAKE) -C $(BUILDDIR)/fftw-$(FFTW_VER)-single clean -rm -f $(FFTW_SINGLE_OBJ_TARGET) clean-fftw-double: - -$(MAKE) -C fftw-$(FFTW_VER)-double clean + -$(MAKE) -C $(BUILDDIR)/fftw-$(FFTW_VER)-double clean -rm -f $(FFTW_DOUBLE_OBJ_TARGET) distclean-fftw: distclean-fftw-single distclean-fftw-double + -rm -rf $(SRCDIR)/srccache/fftw-$(FFTW_VER).tar.gz fftw-$(FFTW_VER)-single distclean-fftw-single: - -rm -rf fftw-$(FFTW_VER).tar.gz fftw-$(FFTW_VER)-single distclean-fftw-double: - -rm -rf fftw-$(FFTW_VER).tar.gz fftw-$(FFTW_VER)-double get-fftw: get-fftw-single get-fftw-double configure-fftw: configure-fftw-single configure-fftw-double compile-fftw: compile-fftw-single compile-fftw-double check-fftw: check-fftw-single check-fftw-double -install-fftw: check-fftw-single check-fftw-double +install-fftw: check-fftw-single check-fftw-double # do these serially, to prevent write conflicts @$(MAKE) -s install-fftw-single @$(MAKE) -s install-fftw-double -get-fftw-single: fftw-$(FFTW_VER).tar.gz -configure-fftw-single: fftw-$(FFTW_VER)-single/config.status -compile-fftw-single: $(FFTW_SINGLE_OBJ_TARGET) -check-fftw-single: fftw-$(FFTW_VER)-single/checked +get-fftw-single: $(SRCDIR)/srccache/fftw-$(FFTW_VER).tar.gz +configure-fftw-single: $(BUILDDIR)/fftw-$(FFTW_VER)-single/config.status +compile-fftw-single: $(FFTW_SINGLE_SRC_TARGET) +check-fftw-single: $(BUILDDIR)/fftw-$(FFTW_VER)-single/checked install-fftw-single: $(FFTW_SINGLE_OBJ_TARGET) -get-fftw-double: fftw-$(FFTW_VER).tar.gz -configure-fftw-double: fftw-$(FFTW_VER)-double/config.status -compile-fftw-double: $(FFTW_DOUBLE_OBJ_TARGET) -check-fftw-double: fftw-$(FFTW_VER)-double/checked +get-fftw-double: $(SRCDIR)/srccache/fftw-$(FFTW_VER).tar.gz +configure-fftw-double: $(BUILDDIR)/fftw-$(FFTW_VER)-double/config.status +compile-fftw-double: $(FFTW_DOUBLE_SRC_TARGET) +check-fftw-double: $(BUILDDIR)/fftw-$(FFTW_VER)-double/checked install-fftw-double: $(FFTW_DOUBLE_OBJ_TARGET) ## UTF8PROC ## UTF8PROC_GIT_URL = git://github.com/JuliaLang/utf8proc.git UTF8PROC_TAR_URL = https://api.github.com/repos/JuliaLang/utf8proc/tarball/$1 -$(eval $(call git-external,utf8proc,UTF8PROC,Makefile,libutf8proc.a)) +$(eval $(call git-external,utf8proc,UTF8PROC,Makefile,libutf8proc.a,$(BUILDDIR))) -UTF8PROC_SRC_TARGET = $(UTF8PROC_SRC_DIR)/libutf8proc.a +UTF8PROC_SRC_TARGET = $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/libutf8proc.a UTF8PROC_OBJ_LIB = $(build_libdir)/libutf8proc.a UTF8PROC_OBJ_HEADER = $(build_includedir)/utf8proc.h UTF8PROC_OBJ_TARGET = $(UTF8PROC_OBJ_LIB) $(UTF8PROC_OBJ_HEADER) UTF8PROC_CFLAGS = -O2 -$(UTF8PROC_SRC_TARGET): $(UTF8PROC_SRC_DIR)/Makefile - $(MAKE) -C $(UTF8PROC_SRC_DIR) CC="$(CC) $(DEPS_CFLAGS)" CFLAGS="$(CFLAGS) $(UTF8PROC_CFLAGS)" PICFLAG="$(fPIC)" AR="$(AR)" libutf8proc.a +$(UTF8PROC_SRC_TARGET): $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/Makefile + $(MAKE) -C $(dir $<) CC="$(CC) $(DEPS_CFLAGS)" CFLAGS="$(CFLAGS) $(UTF8PROC_CFLAGS)" PICFLAG="$(fPIC)" AR="$(AR)" libutf8proc.a touch -c $@ -$(UTF8PROC_SRC_DIR)/checked: $(UTF8PROC_SRC_TARGET) +$(BUILDDIR)/$(UTF8PROC_SRC_DIR)/checked: $(UTF8PROC_SRC_TARGET) ifeq ($(OS),$(BUILD_OS)) - -$(MAKE) -C $(UTF8PROC_SRC_DIR) check + $(MAKE) -C $(dir $@) check endif echo 1 > $@ -$(UTF8PROC_OBJ_LIB): $(UTF8PROC_SRC_TARGET) +$(UTF8PROC_OBJ_LIB): $(UTF8PROC_SRC_TARGET) $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/checked cp -f $< $@ $(UTF8PROC_OBJ_HEADER): $(UTF8PROC_SRC_TARGET) - cp -f $(UTF8PROC_SRC_DIR)/utf8proc.h $@ + cp -f $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/utf8proc.h $@ clean-utf8proc: - -$(MAKE) -C $(UTF8PROC_SRC_DIR) clean + -$(MAKE) -C $(BUILDDIR)/$(UTF8PROC_SRC_DIR) clean -rm -rf $(build_libdir)/libutf8proc.a $(build_includedir)/utf8proc.h -get-utf8proc: $(UTF8PROC_SRC_DIR)/Makefile -configure-utf8proc: get-utf8proc +get-utf8proc: $(SRCDIR)/srccache/$(UTF8PROC_SRC_DIR).git +configure-utf8proc: $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/Makefile compile-utf8proc: $(UTF8PROC_SRC_TARGET) -check-utf8proc: $(UTF8PROC_SRC_DIR)/checked +check-utf8proc: $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/checked install-utf8proc: $(UTF8PROC_OBJ_TARGET) + ## SUITESPARSE ## -SUITESPARSE_OBJ_SOURCE = SuiteSparse-$(SUITESPARSE_VER)/UMFPACK/Lib/libumfpack.a +SUITESPARSE_OBJ_SOURCE = $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/UMFPACK/Lib/libumfpack.a SUITESPARSE_OBJ_TARGET = $(build_shlibdir)/libspqr.$(SHLIB_EXT) - ifeq ($(USE_BLAS64), 1) UMFPACK_CONFIG = -DLONGBLAS='long long' CHOLMOD_CONFIG = -DLONGBLAS='long long' @@ -1545,12 +1579,12 @@ SUITESPARSE_MFLAGS = CC="$(CC)" CXX="$(CXX)" F77="$(FC)" AR="$(AR)" RANLIB="$(RA INSTALL_LIB="$(build_libdir)" INSTALL_INCLUDE="$(build_includedir)" LIB="$(SUITE_SPARSE_LIB)" \ UMFPACK_CONFIG="$(UMFPACK_CONFIG)" CHOLMOD_CONFIG="$(CHOLMOD_CONFIG)" SPQR_CONFIG="$(SPQR_CONFIG)" -SuiteSparse-$(SUITESPARSE_VER).tar.gz: - $(JLDOWNLOAD) $@ http://faculty.cse.tamu.edu/davis/SuiteSparse/$@ -SuiteSparse-$(SUITESPARSE_VER)/Makefile: SuiteSparse-$(SUITESPARSE_VER).tar.gz +$(SRCDIR)/srccache/SuiteSparse-$(SUITESPARSE_VER).tar.gz: + $(JLDOWNLOAD) $@ http://faculty.cse.tamu.edu/davis/SuiteSparse/$(notdir $@) +$(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/Makefile: $(SRCDIR)/srccache/SuiteSparse-$(SUITESPARSE_VER).tar.gz $(JLCHECKSUM) $< - mkdir -p SuiteSparse-$(SUITESPARSE_VER) - $(TAR) -C SuiteSparse-$(SUITESPARSE_VER) --strip-components 1 -zxf $< + mkdir -p $(dir $@) + $(TAR) -C $(dir $@) --strip-components 1 -zxf $< touch -c $@ ifeq ($(USE_ATLAS), 1) @@ -1562,17 +1596,15 @@ $(SUITESPARSE_OBJ_SOURCE): | $(OPENBLAS_OBJ_TARGET) else ifeq ($(USE_SYSTEM_LAPACK), 0) $(SUITESPARSE_OBJ_SOURCE): | $(LAPACK_OBJ_TARGET) endif -$(SUITESPARSE_OBJ_SOURCE): SuiteSparse-$(SUITESPARSE_VER)/Makefile - cd SuiteSparse-$(SUITESPARSE_VER) && \ - $(MAKE) library $(SUITESPARSE_MFLAGS) +$(SUITESPARSE_OBJ_SOURCE): $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/Makefile + $(MAKE) -C $(dir $<) library $(SUITESPARSE_MFLAGS) touch -c $@ -SuiteSparse-$(SUITESPARSE_VER)/checked: $(SUITESPARSE_OBJ_SOURCE) - cd SuiteSparse-$(SUITESPARSE_VER) && \ - $(MAKE) default $(SUITESPARSE_MFLAGS) +$(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/checked: $(SUITESPARSE_OBJ_SOURCE) + $(MAKE) -C $(dir $@) default $(SUITESPARSE_MFLAGS) touch $@ $(SUITESPARSE_OBJ_TARGET): $(SUITESPARSE_OBJ_SOURCE) - mkdir -p SuiteSparse-$(SUITESPARSE_VER)/lib && \ - cd SuiteSparse-$(SUITESPARSE_VER)/lib && \ + mkdir -p $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/lib && \ + cd $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/lib && \ rm -f *.a && \ cp -f `find .. -name libamd.a -o -name libcolamd.a -o -name libcamd.a -o -name libccolamd.a -o -name libcholmod.a -o -name libumfpack.a -o -name libsuitesparseconfig.a -o -name libspqr.a 2>/dev/null` . && \ $(CC) -shared $(WHOLE_ARCHIVE) libsuitesparseconfig.a $(NO_WHOLE_ARCHIVE) -o $(build_shlibdir)/libsuitesparseconfig.$(SHLIB_EXT) && \ @@ -1593,15 +1625,15 @@ $(SUITESPARSE_OBJ_TARGET): $(SUITESPARSE_OBJ_SOURCE) $(INSTALL_NAME_CMD)libspqr.$(SHLIB_EXT) $(build_shlibdir)/libspqr.$(SHLIB_EXT) clean-suitesparse: - -$(MAKE) -C SuiteSparse-$(SUITESPARSE_VER) clean - -rm -fr SuiteSparse-$(SUITESPARSE_VER)/lib + -$(MAKE) -C $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER) clean + -rm -fr $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/lib distclean-suitesparse: - -rm -rf SuiteSparse-$(SUITESPARSE_VER).tar.gz SuiteSparse-$(SUITESPARSE_VER) + -rm -rf $(SRCDIR)/srccache/SuiteSparse-$(SUITESPARSE_VER).tar.gz $(SRCDIR)/srccache/SuiteSparse-$(SUITESPARSE_VER) -get-suitesparse: SuiteSparse-$(SUITESPARSE_VER).tar.gz -configure-suitesparse: get-suitesparse +get-suitesparse: $(SRCDIR)/srccache/SuiteSparse-$(SUITESPARSE_VER).tar.gz +configure-suitesparse: $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/Makefile compile-suitesparse: $(SUITESPARSE_OBJ_SOURCE) -check-suitesparse: SuiteSparse-$(SUITESPARSE_VER)/checked +check-suitesparse: $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/checked install-suitesparse: $(SUITESPARSE_OBJ_TARGET) install-suitesparse-wrapper # SUITESPARSE WRAPPER @@ -1610,12 +1642,12 @@ ifeq ($(USE_SYSTEM_SUITESPARSE), 1) SUITESPARSE_INC = -I /usr/include/suitesparse SUITESPARSE_LIB = -lumfpack -lcholmod -lamd -lcamd -lcolamd -lspqr else -SUITESPARSE_INC = -I SuiteSparse-$(SUITESPARSE_VER)/CHOLMOD/Include -I SuiteSparse-$(SUITESPARSE_VER)/SuiteSparse_config -I SuiteSparse-$(SUITESPARSE_VER)/SPQR/Include +SUITESPARSE_INC = -I $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/CHOLMOD/Include -I $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/SuiteSparse_config -I $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/SPQR/Include SUITESPARSE_LIB = -L$(build_shlibdir) -lcholmod -lumfpack -lspqr $(RPATH_ORIGIN) $(build_shlibdir)/libsuitesparse_wrapper.$(SHLIB_EXT): $(SUITESPARSE_OBJ_TARGET) endif -$(build_shlibdir)/libsuitesparse_wrapper.$(SHLIB_EXT): SuiteSparse_wrapper.c +$(build_shlibdir)/libsuitesparse_wrapper.$(SHLIB_EXT): $(SRCDIR)/SuiteSparse_wrapper.c $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -O2 -shared $(fPIC) $(SUITESPARSE_INC) $< -o $@ $(SUITESPARSE_LIB) $(INSTALL_NAME_CMD)libsuitesparse_wrapper.$(SHLIB_EXT) $@ touch -c $@ @@ -1634,30 +1666,31 @@ install-suitesparse-wrapper: $(build_shlibdir)/libsuitesparse_wrapper.$(SHLIB_EX ## UNWIND ## LIBUNWIND_TARGET_OBJ = $(build_libdir)/libunwind.a -LIBUNWIND_TARGET_SOURCE = libunwind-$(UNWIND_VER)/src/.libs/libunwind.a +LIBUNWIND_TARGET_SOURCE = $(BUILDDIR)/libunwind-$(UNWIND_VER)/src/.libs/libunwind.a LIBUNWIND_CFLAGS = -U_FORTIFY_SOURCE $(fPIC) LIBUNWIND_CPPFLAGS = -libunwind-$(UNWIND_VER).tar.gz: - $(JLDOWNLOAD) $@ http://download.savannah.gnu.org/releases/libunwind/$@ -libunwind-$(UNWIND_VER)/configure: libunwind-$(UNWIND_VER).tar.gz +$(SRCDIR)/srccache/libunwind-$(UNWIND_VER).tar.gz: + $(JLDOWNLOAD) $@ http://download.savannah.gnu.org/releases/libunwind/$(notdir $@) +$(SRCDIR)/srccache/libunwind-$(UNWIND_VER)/configure: $(SRCDIR)/srccache/libunwind-$(UNWIND_VER).tar.gz $(JLCHECKSUM) $< - $(TAR) xfz $< - cd libunwind-$(UNWIND_VER) && patch -p1 < ../libunwind.patch + cd $(dir $<) && $(TAR) xfz $< + cd $(dir $<)/libunwind-$(UNWIND_VER) && patch -p1 < $(SRCDIR)/libunwind.patch touch -c $@ -libunwind-$(UNWIND_VER)/config.status: libunwind-$(UNWIND_VER)/configure - cd libunwind-$(UNWIND_VER) && \ - ./configure $(CONFIGURE_COMMON) CPPFLAGS="$(CPPFLAGS) $(LIBUNWIND_CPPFLAGS)" CFLAGS="$(CFLAGS) $(LIBUNWIND_CFLAGS)" --disable-shared --disable-minidebuginfo +$(BUILDDIR)/libunwind-$(UNWIND_VER)/config.status: $(SRCDIR)/srccache/libunwind-$(UNWIND_VER)/configure + mkdir -p $(dir $@) + cd $(dir $@) && \ + $< $(CONFIGURE_COMMON) CPPFLAGS="$(CPPFLAGS) $(LIBUNWIND_CPPFLAGS)" CFLAGS="$(CFLAGS) $(LIBUNWIND_CFLAGS)" --disable-shared --disable-minidebuginfo touch -c $@ -$(LIBUNWIND_TARGET_SOURCE): libunwind-$(UNWIND_VER)/config.status - $(MAKE) -C libunwind-$(UNWIND_VER) +$(LIBUNWIND_TARGET_SOURCE): $(BUILDDIR)/libunwind-$(UNWIND_VER)/config.status + $(MAKE) -C $(dir $<) touch -c $@ -libunwind-$(UNWIND_VER)/checked: $(LIBUNWIND_TARGET_SOURCE) +$(BUILDDIR)/libunwind-$(UNWIND_VER)/checked: $(LIBUNWIND_TARGET_SOURCE) ifeq ($(OS),$(BUILD_OS)) - $(MAKE) -C libunwind-$(UNWIND_VER) check + $(MAKE) -C $(dir $@) check endif echo 1 > $@ -#todo: libunwind tests known to fail +#todo: libunwind tests are known to fail, so they aren't run $(LIBUNWIND_TARGET_OBJ): $(LIBUNWIND_TARGET_SOURCE) $(call make-install,libunwind-$(UNWIND_VER),) ifeq ($(ARCH), ppc64) @@ -1667,15 +1700,15 @@ endif touch $@ clean-unwind: - -$(MAKE) -C libunwind-$(UNWIND_VER) clean + -$(MAKE) -C $(BUILDDIR)/libunwind-$(UNWIND_VER) clean -rm -f $(LIBUNWIND_TARGET_OBJ) $(LIBUNWIND_TARGET_SOURCE) distclean-unwind: - -rm -rf libunwind-$(UNWIND_VER).tar.gz libunwind-$(UNWIND_VER) + -rm -rf $(SRCDIR)/srccache/libunwind-$(UNWIND_VER).tar.gz $(SRCDIR)/srccache/libunwind-$(UNWIND_VER) -get-unwind: libunwind-$(UNWIND_VER).tar.gz -configure-unwind: libunwind-$(UNWIND_VER)/config.status +get-unwind: $(SRCDIR)/srccache/libunwind-$(UNWIND_VER).tar.gz +configure-unwind: $(BUILDDIR)/libunwind-$(UNWIND_VER)/config.status compile-unwind: $(LIBUNWIND_TARGET_SOURCE) -check-unwind: libunwind-$(UNWIND_VER)/checked +check-unwind: $(BUILDDIR)/libunwind-$(UNWIND_VER)/checked install-unwind: $(LIBUNWIND_TARGET_OBJ) ## OS X Unwind ## @@ -1683,30 +1716,30 @@ install-unwind: $(LIBUNWIND_TARGET_OBJ) OSXUNWIND_FLAGS = ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) CFLAGS="$(CFLAGS) -ggdb3 -O0" CXXFLAGS="$(CXXFLAGS) -ggdb3 -O0" SFLAGS="-ggdb3" LDFLAGS="$(LDFLAGS) -Wl,-macosx_version_min,10.7" OSXUNWIND_OBJ_TARGET = $(build_shlibdir)/libosxunwind.$(SHLIB_EXT) -OSXUNWIND_OBJ_SOURCE = libosxunwind-$(OSXUNWIND_VER)/libosxunwind.$(SHLIB_EXT) +OSXUNWIND_OBJ_SOURCE = $(BUILDDIR)/libosxunwind-$(OSXUNWIND_VER)/libosxunwind.$(SHLIB_EXT) -libosxunwind-$(OSXUNWIND_VER).tar.gz: +$(SRCDIR)/srccache/libosxunwind-$(OSXUNWIND_VER).tar.gz: $(JLDOWNLOAD) $@ https://github.com/JuliaLang/libosxunwind/archive/v$(OSXUNWIND_VER).tar.gz -libosxunwind-$(OSXUNWIND_VER)/Makefile: libosxunwind-$(OSXUNWIND_VER).tar.gz +$(BUILDDIR)/libosxunwind-$(OSXUNWIND_VER)/Makefile: $(SRCDIR)/srccache/libosxunwind-$(OSXUNWIND_VER).tar.gz $(JLCHECKSUM) $< - $(TAR) xfz $< + cd $(BUILDDIR) && $(TAR) xfz $< touch -c $@ -$(OSXUNWIND_OBJ_SOURCE): libosxunwind-$(OSXUNWIND_VER)/Makefile - $(MAKE) -C libosxunwind-$(OSXUNWIND_VER) $(OSXUNWIND_FLAGS) +$(OSXUNWIND_OBJ_SOURCE): $(BUILDDIR)/libosxunwind-$(OSXUNWIND_VER)/Makefile + $(MAKE) -C $(dir $<) $(OSXUNWIND_FLAGS) touch -c $@ $(OSXUNWIND_OBJ_TARGET): $(OSXUNWIND_OBJ_SOURCE) | $(build_shlibdir) - cp libosxunwind-$(OSXUNWIND_VER)/libosxunwind.a $(build_libdir)/libosxunwind.a + cp $(BUILDDIR)/libosxunwind-$(OSXUNWIND_VER)/libosxunwind.a $(build_libdir)/libosxunwind.a cp $< $@ - cp -R libosxunwind-$(OSXUNWIND_VER)/include/* $(build_includedir) + cp -R $(BUILDDIR)/libosxunwind-$(OSXUNWIND_VER)/include/* $(build_includedir) $(INSTALL_NAME_CMD)libosxunwind.$(SHLIB_EXT) $@ clean-osxunwind: - -$(MAKE) -C libosxunwind-$(OSXUNWIND_VER) distclean $(OSXUNWIND_FLAGS) + -$(MAKE) -C $(BUILDDIR)/libosxunwind-$(OSXUNWIND_VER) clean $(OSXUNWIND_FLAGS) -rm $(OSXUNWIND_OBJ_TARGET) distclean-osxunwind: - -rm -rf libosxunwind-$(OSXUNWIND_VER) + -rm -rf $(SRCDIR)/srccache/libosxunwind-$(OSXUNWIND_VER).tar.gz get-osxunwind: libosxunwind-$(OSXUNWIND_VER)/Makefile configure-osxunwind: get-osxunwind @@ -1716,49 +1749,48 @@ install-osxunwind: $(OSXUNWIND_OBJ_TARGET) ## GMP ## -GMP_SRC_TARGET = gmp-$(GMP_VER)/.libs/libgmp.$(SHLIB_EXT) +GMP_SRC_TARGET = $(BUILDDIR)/gmp-$(GMP_VER)/.libs/libgmp.$(SHLIB_EXT) GMP_OBJ_TARGET = $(build_shlibdir)/libgmp.$(SHLIB_EXT) ifeq ($(SANITIZE),1) GMP_CONFIGURE_OPTS += --disable-assembly endif -gmp-$(GMP_VER).tar.bz2: - $(JLDOWNLOAD) $@ https://gmplib.org/download/gmp/$@ -gmp-$(GMP_VER)/configure: gmp-$(GMP_VER).tar.bz2 +$(SRCDIR)/srccache/gmp-$(GMP_VER).tar.bz2: + $(JLDOWNLOAD) $@ https://gmplib.org/download/gmp/$(notdir $@) +$(SRCDIR)/srccache/gmp-$(GMP_VER)/configure: $(SRCDIR)/srccache/gmp-$(GMP_VER).tar.bz2 $(JLCHECKSUM) $< - $(TAR) jxf $< -ifeq ($(OS), Darwin) - cd gmp-$(GMP_VER) && patch -p1 < ../gmp_6.0.0_osx.patch -endif + cd $(dir $<) && $(TAR) jxf $< + cd $(dir $@) && patch -p1 < $(SRCDIR)/gmp_6.0.0_osx.patch touch -c $@ -gmp-$(GMP_VER)/config.status: gmp-$(GMP_VER)/configure - cd gmp-$(GMP_VER) && \ - ./configure $(CONFIGURE_COMMON) F77= --enable-shared --disable-static $(GMP_CONFIGURE_OPTS) +$(BUILDDIR)/gmp-$(GMP_VER)/config.status: $(SRCDIR)/srccache/gmp-$(GMP_VER)/configure + mkdir -p $(dir $@) + cd $(dir $@) && \ + $< $(CONFIGURE_COMMON) F77= --enable-shared --disable-static $(GMP_CONFIGURE_OPTS) touch -c $@ -$(GMP_SRC_TARGET): gmp-$(GMP_VER)/config.status - $(MAKE) -C gmp-$(GMP_VER) $(LIBTOOL_CCLD) -gmp-$(GMP_VER)/checked: $(GMP_SRC_TARGET) +$(GMP_SRC_TARGET): $(BUILDDIR)/gmp-$(GMP_VER)/config.status + $(MAKE) -C $(dir $<) $(LIBTOOL_CCLD) +$(BUILDDIR)/gmp-$(GMP_VER)/checked: $(GMP_SRC_TARGET) ifeq ($(OS),$(BUILD_OS)) - $(MAKE) -C gmp-$(GMP_VER) $(LIBTOOL_CCLD) check + $(MAKE) -C $(dir $@) $(LIBTOOL_CCLD) check endif echo 1 > $@ -$(GMP_OBJ_TARGET): $(GMP_SRC_TARGET) gmp-$(GMP_VER)/checked | $(build_shlibdir) $(build_includedir) - $(INSTALL_M) gmp-$(GMP_VER)/.libs/libgmp.*$(SHLIB_EXT)* $(build_shlibdir) - $(INSTALL_F) gmp-$(GMP_VER)/gmp.h $(build_includedir) +$(GMP_OBJ_TARGET): $(GMP_SRC_TARGET) $(BUILDDIR)/gmp-$(GMP_VER)/checked | $(build_shlibdir) $(build_includedir) + $(INSTALL_M) $(BUILDDIR)/gmp-$(GMP_VER)/.libs/libgmp.*$(SHLIB_EXT)* $(build_shlibdir) + $(INSTALL_F) $(BUILDDIR)/gmp-$(GMP_VER)/gmp.h $(build_includedir) $(INSTALL_NAME_CMD)libgmp.$(SHLIB_EXT) $@ touch -c $@ clean-gmp: - -$(MAKE) -C gmp-$(GMP_VER) clean + -$(MAKE) -C $(BUILDDIR)/gmp-$(GMP_VER) clean -rm -f $(GMP_OBJ_TARGET) distclean-gmp: - -rm -rf gmp-$(GMP_VER).tar.bz2 gmp-$(GMP_VER) + -rm -rf $(SRCDIR)/srccache/gmp-$(GMP_VER).tar.bz2 $(SRCDIR)/srccache/gmp-$(GMP_VER) -get-gmp: gmp-$(GMP_VER).tar.bz2 -configure-gmp: gmp-$(GMP_VER)/config.status +get-gmp: $(SRCDIR)/srccache/gmp-$(GMP_VER).tar.bz2 +configure-gmp: $(BUILDDIR)/gmp-$(GMP_VER)/config.status compile-gmp: $(GMP_SRC_TARGET) -check-gmp: gmp-$(GMP_VER)/checked +check-gmp: $(BUILDDIR)/gmp-$(GMP_VER)/checked install-gmp: $(GMP_OBJ_TARGET) ifeq ($(USE_SYSTEM_GMP), 0) @@ -1779,7 +1811,7 @@ endif endif -MPFR_SRC_TARGET = mpfr-$(MPFR_VER)/src/.libs/libmpfr.$(SHLIB_EXT) +MPFR_SRC_TARGET = $(BUILDDIR)/mpfr-$(MPFR_VER)/src/.libs/libmpfr.$(SHLIB_EXT) MPFR_OBJ_TARGET = $(build_shlibdir)/libmpfr.$(SHLIB_EXT) ifeq ($(OS),Darwin) MPFR_CHECK_MFLAGS = LDFLAGS="$(LDFLAGS) -Wl,-rpath,'$(build_libdir)'" @@ -1790,145 +1822,144 @@ ifeq ($(SANITIZE),1) MPFR_OPTS += --host=none-unknown-linux endif -mpfr-$(MPFR_VER).tar.bz2: - $(JLDOWNLOAD) $@ http://www.mpfr.org/mpfr-$(MPFR_VER)/$@ -mpfr-$(MPFR_VER)/configure: mpfr-$(MPFR_VER).tar.bz2 +$(SRCDIR)/srccache/mpfr-$(MPFR_VER).tar.bz2: + $(JLDOWNLOAD) $@ http://www.mpfr.org/mpfr-$(MPFR_VER)/$(notdir $@) +$(SRCDIR)/srccache/mpfr-$(MPFR_VER)/configure: $(SRCDIR)/srccache/mpfr-$(MPFR_VER).tar.bz2 $(JLCHECKSUM) $< - $(TAR) jxf $< + cd $(dir $<) && $(TAR) jxf $< touch -c $@ -mpfr-$(MPFR_VER)/config.status: mpfr-$(MPFR_VER)/configure | $(MPFR_DEPS) - cd mpfr-$(MPFR_VER) && \ - ./configure $(CONFIGURE_COMMON) $(MPFR_OPTS) F77= --enable-shared --disable-static +$(BUILDDIR)/mpfr-$(MPFR_VER)/config.status: $(SRCDIR)/srccache/mpfr-$(MPFR_VER)/configure | $(MPFR_DEPS) + mkdir -p $(dir $@) + cd $(dir $@) && \ + $< $(CONFIGURE_COMMON) $(MPFR_OPTS) F77= --enable-shared --disable-static touch -c $@ -$(MPFR_SRC_TARGET): mpfr-$(MPFR_VER)/config.status - $(MAKE) -C mpfr-$(MPFR_VER) $(LIBTOOL_CCLD) -mpfr-$(MPFR_VER)/checked: $(MPFR_SRC_TARGET) +$(MPFR_SRC_TARGET): $(BUILDDIR)/mpfr-$(MPFR_VER)/config.status + $(MAKE) -C $(dir $<) $(LIBTOOL_CCLD) +$(BUILDDIR)/mpfr-$(MPFR_VER)/checked: $(MPFR_SRC_TARGET) ifeq ($(OS),$(BUILD_OS)) - $(MAKE) -C mpfr-$(MPFR_VER) $(LIBTOOL_CCLD) check $(MPFR_CHECK_MFLAGS) + $(MAKE) -C $(dir $@) $(LIBTOOL_CCLD) check $(MPFR_CHECK_MFLAGS) endif echo 1 > $@ -$(MPFR_OBJ_TARGET): $(MPFR_SRC_TARGET) mpfr-$(MPFR_VER)/checked +$(MPFR_OBJ_TARGET): $(MPFR_SRC_TARGET) $(BUILDDIR)/mpfr-$(MPFR_VER)/checked $(call make-install,mpfr-$(MPFR_VER),$(LIBTOOL_CCLD)) $(INSTALL_NAME_CMD)libmpfr.$(SHLIB_EXT) $@ touch -c $@ clean-mpfr: - -$(MAKE) -C mpfr-$(MPFR_VER) clean + -$(MAKE) -C $(BUILDDIR)/mpfr-$(MPFR_VER) clean -rm -f $(MPFR_OBJ_TARGET) distclean-mpfr: - -rm -rf mpfr-$(MPFR_VER).tar.bz2 mpfr-$(MPFR_VER) + -rm -rf $(SRCDIR)/srccache/mpfr-$(MPFR_VER).tar.bz2 $(SRCDIR)/srccache/mpfr-$(MPFR_VER) -get-mpfr: mpfr-$(MPFR_VER).tar.bz2 -configure-mpfr: mpfr-$(MPFR_VER)/config.status +get-mpfr: $(SRCDIR)/srccache/mpfr-$(MPFR_VER).tar.bz2 +configure-mpfr: $(BUILDDIR)/mpfr-$(MPFR_VER)/config.status compile-mpfr: $(MPFR_SRC_TARGET) -check-mpfr: mpfr-$(MPFR_VER)/checked +check-mpfr: $(BUILDDIR)/mpfr-$(MPFR_VER)/checked install-mpfr: $(MPFR_OBJ_TARGET) ## patchelf ## -PATCHELF_SOURCE = patchelf-$(PATCHELF_VER)/src/patchelf +PATCHELF_SOURCE = $(BUILDDIR)/patchelf-$(PATCHELF_VER)/src/patchelf PATCHELF_TARGET = $(build_bindir)/patchelf -compile-patchelf: install-patchelf -install-patchelf: $(PATCHELF_TARGET) - -patchelf-$(PATCHELF_VER).tar.gz: +$(SRCDIR)/srccache/patchelf-$(PATCHELF_VER).tar.gz: $(JLDOWNLOAD) $@ http://nixos.org/releases/patchelf/patchelf-$(PATCHELF_VER)/patchelf-$(PATCHELF_VER).tar.gz -patchelf-$(PATCHELF_VER)/configure: patchelf-$(PATCHELF_VER).tar.gz +$(SRCDIR)/srccache/patchelf-$(PATCHELF_VER)/configure: $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER).tar.gz $(JLCHECKSUM) $< - $(TAR) zxf $< + cd $(dir $<) && $(TAR) zxf $< touch -c $@ -patchelf-$(PATCHELF_VER)/config.status: patchelf-$(PATCHELF_VER)/configure - cd patchelf-$(PATCHELF_VER) && \ - ./configure $(CONFIGURE_COMMON) +$(BUILDDIR)/patchelf-$(PATCHELF_VER)/config.status: $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER)/configure + mkdir $(dir $@) + cd $(dir $@) && \ + $< $(CONFIGURE_COMMON) touch -c $@ -$(PATCHELF_SOURCE): patchelf-$(PATCHELF_VER)/config.status - $(MAKE) -C patchelf-$(PATCHELF_VER) +$(PATCHELF_SOURCE): $(BUILDDIR)/patchelf-$(PATCHELF_VER)/config.status + $(MAKE) -C $(dir $<) touch -c $@ -patchelf-$(PATCHELF_VER)/checked: $(PATCHELF_SOURCE) +$(BUILDDIR)/patchelf-$(PATCHELF_VER)/checked: $(PATCHELF_SOURCE) ifeq ($(OS),$(BUILD_OS)) # disabled due to bug in v0.6 - #$(MAKE) -C patchelf-$(PATCHELF_VER) check + #$(MAKE) -C $(dir $@) check endif echo 1 > $@ -$(PATCHELF_TARGET): $(PATCHELF_SOURCE) patchelf-$(PATCHELF_VER)/checked +$(PATCHELF_TARGET): $(PATCHELF_SOURCE) $(BUILDDIR)/patchelf-$(PATCHELF_VER)/checked $(call make-install,patchelf-$(PATCHELF_VER),) touch -c $@ clean-patchelf: - -$(MAKE) -C patchelf-$(PATCHELF_VER) clean + -$(MAKE) -C $(BUILDDIR)/patchelf-$(PATCHELF_VER) clean -rm -f $(PATCHELF_OBJ_TARGET) distclean-patchelf: - -rm -rf patchelf-$(PATCHELF_VER).tar.gz patchelf-$(PATCHELF_VER) + -rm -rf $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER).tar.gz $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER) -get-patchelf: patchelf-$(PATCHELF_VER).tar.gz -configure-patchelf: patchelf-$(PATCHELF_VER)/config.status +get-patchelf: $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER).tar.gz +configure-patchelf: $(BUILDDIR)/patchelf-$(PATCHELF_VER)/config.status compile-patchelf: $(PATCHELF_SOURCE) -check-patchelf: patchelf-$(PATCHELF_VER)/checked +check-patchelf: $(BUILDDIR)/patchelf-$(PATCHELF_VER)/checked install-patchelf: $(PATCHELF_TARGET) ## Git # only used for the mac binaries in contrib/mac/app/Makefile -GIT_SOURCE = git-$(GIT_VER)/git +GIT_SOURCE = $(BUILDDIR)/git-$(GIT_VER)/git GIT_TARGET = $(build_libexecdir)/git -git-$(GIT_VER).tar.gz: +$(SRCDIR)/srccache/git-$(GIT_VER).tar.gz: $(JLDOWNLOAD) $@ https://www.kernel.org/pub/software/scm/git/git-$(GIT_VER).tar.gz -git-$(GIT_VER)/configure: git-$(GIT_VER).tar.gz +$(BUILDDIR)/git-$(GIT_VER)/configure: $(SRCDIR)/srccache/git-$(GIT_VER).tar.gz $(JLCHECKSUM) $< - $(TAR) zxf $< + cd $(BUILDDIR) && $(TAR) zxf $< touch -c $@ -git-$(GIT_VER)/config.status: git-$(GIT_VER)/configure - cd git-$(GIT_VER) && \ +$(BUILDDIR)/git-$(GIT_VER)/config.status: $(BUILDDIR)/git-$(GIT_VER)/configure + cd $(dir $@) && \ ./configure $(CONFIGURE_COMMON) --bindir="$(build_libexecdir)" touch -c $@ -$(GIT_SOURCE): git-$(GIT_VER)/config.status - $(MAKE) -C git-$(GIT_VER) +$(GIT_SOURCE): $(BUILDDIR)/git-$(GIT_VER)/config.status + $(MAKE) -C $(dir $<) touch -c $@ -git-$(GIT_VER)/checked: $(GIT_SOURCE) +$(BUILDDIR)/git-$(GIT_VER)/checked: $(GIT_SOURCE) echo 1 > $@ -$(GIT_TARGET): $(GIT_SOURCE) git-$(GIT_VER)/checked +$(GIT_TARGET): $(GIT_SOURCE) $(BUILDDIR)/git-$(GIT_VER)/checked $(call make-install,git-$(GIT_VER),NO_INSTALL_HARDLINKS=1 bindir="$(build_libexecdir)") touch -c $@ clean-git: - -$(MAKE) -C git-$(GIT_VER) clean + -$(MAKE) -C $(BUILDDIR)/git-$(GIT_VER) clean -rm -f $(GIT_OBJ_TARGET) distclean-git: - -rm -rf git-$(GIT_VER).tar.gz git-$(GIT_VER) + -rm -rf $(SRCDIR)/srccache/git-$(GIT_VER).tar.gz $(SRCDIR)/srccache/git-$(GIT_VER) -get-git: git-$(GIT_VER).tar.gz -configure-git: git-$(GIT_VER)/config.status +get-git: $(SRCDIR)/srccache/git-$(GIT_VER).tar.gz +configure-git: $(BUILDDIR)/git-$(GIT_VER)/config.status compile-git: $(GIT_SOURCE) -check-git: git-$(GIT_VER)/checked +check-git: $(BUILDDIR)/git-$(GIT_VER)/checked install-git: $(GIT_TARGET) ## virtualenv -VIRTUALENV_SOURCE = virtualenv-$(VIRTUALENV_VER)/virtualenv.py -VIRTUALENV_TARGET = julia-env +VIRTUALENV_SOURCE = $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER)/virtualenv.py +VIRTUALENV_TARGET = $(BUILDDIR)/julia-env -virtualenv-$(VIRTUALENV_VER).tar.gz: - $(JLDOWNLOAD) $@ https://pypi.python.org/packages/source/v/virtualenv/$@ -$(VIRTUALENV_SOURCE): virtualenv-$(VIRTUALENV_VER).tar.gz +$(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER).tar.gz: + $(JLDOWNLOAD) $@ https://pypi.python.org/packages/source/v/virtualenv/$(dir $@) +$(VIRTUALENV_SOURCE): $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER).tar.gz $(JLCHECKSUM) $< - $(TAR) zxf $< + cd $(dir $<) && $(TAR) zxf $< touch -c $@ $(VIRTUALENV_TARGET): $(VIRTUALENV_SOURCE) - "$(shell ./find_python2)" $< $@ + "$(shell $(SRCDIR)/find_python2)" $< $@ ifeq ($(BUILD_OS), WINNT) -[ -e $@/Scripts ] && ! [ -e $@/bin ] && cmd //C mklink //J $@\\bin $@\\Scripts endif touch -c $@ clean-virtualenv: - -$(MAKE) -C virtualenv-$(VIRTUALENV_VER) clean + -$(MAKE) -C $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER) clean -rm -rf $(VIRTUALENV_TARGET) distclean-virtualenv: clean-virtualenv - -rm -rf virtualenv-$(VIRTUALENV_VER).tar.gz virtualenv-$(VIRTUALENV_VER) + -rm -rf $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER).tar.gz $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER) -get-virtualenv: virtualenv-$(VIRTUALENV_VER).tar.gz +get-virtualenv: $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER).tar.gz configure-virtualenv: get-virtualenv compile-virtualenv: $(VIRTUALENV_SOURCE) check-virtualenv: compile-virtualenv @@ -1938,9 +1969,9 @@ install-virtualenv: $(VIRTUALENV_TARGET) LIBGIT2_GIT_URL = git://github.com/libgit2/libgit2.git LIBGIT2_TAR_URL = https://api.github.com/repos/libgit2/libgit2/tarball/$1 -$(eval $(call git-external,libgit2,LIBGIT2,CMakeLists.txt,build/libgit2.$(SHLIB_EXT))) +$(eval $(call git-external,libgit2,LIBGIT2,CMakeLists.txt,build/libgit2.$(SHLIB_EXT),$(SRCDIR)/srccache)) -LIBGIT2_OBJ_SOURCE = $(LIBGIT2_SRC_DIR)/build/libgit2.$(SHLIB_EXT) +LIBGIT2_OBJ_SOURCE = $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/libgit2.$(SHLIB_EXT) LIBGIT2_OBJ_TARGET = $(build_shlibdir)/libgit2.$(SHLIB_EXT) LIBGIT2_OPTS = $(CMAKE_COMMON) -DTHREADSAFE=ON @@ -1957,22 +1988,22 @@ LIBGIT2_OPTS += -DCMAKE_FIND_ROOT_PATH=/usr/$(XC_HOST) -DCMAKE_FIND_ROOT_PATH_MO endif endif -$(LIBGIT2_SRC_DIR)/build/Makefile: $(LIBGIT2_SRC_DIR)/CMakeLists.txt - mkdir -p $(LIBGIT2_SRC_DIR)/build - cd $(LIBGIT2_SRC_DIR)/build/ && \ - $(CMAKE) .. $(LIBGIT2_OPTS) +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt + mkdir -p $(dir $@) + cd $(dir $@) && \ + $(CMAKE) $(dir $<) $(LIBGIT2_OPTS) touch -c $@ -$(LIBGIT2_OBJ_SOURCE): $(LIBGIT2_SRC_DIR)/build/Makefile - $(MAKE) -C $(LIBGIT2_SRC_DIR)/build +$(LIBGIT2_OBJ_SOURCE): $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile + $(MAKE) -C $(dir $<) touch -c $@ -$(LIBGIT2_SRC_DIR)/build/checked: $(LIBGIT2_OBJ_SOURCE) +$(LIBGIT2_SRC_DIR)/checked: $(LIBGIT2_OBJ_SOURCE) ifeq ($(OS),$(BUILD_OS)) - $(MAKE) -C $(LIBGIT2_SRC_DIR)/build test + $(MAKE) -C $(dir $@) test endif echo 1 > $@ $(LIBGIT2_OBJ_TARGET): $(LIBGIT2_OBJ_SOURCE) | $(build_shlibdir) cp $< $@ - #$$(call make-install,$(LIBGIT2_SRC_DIR)/build,) + #$$(call make-install,$(LIBGIT2_SRC_DIR),) # currently don't need the full install ifeq ($(OS),Linux) # If we're on linux, copy over libssl and libcrypto for libgit2 -LIBGIT_LIBS=$$(ldd "$@" | tail -n +2 | awk '{print $$(NF-1)}'); \ @@ -1985,13 +2016,13 @@ endif touch -c $@ clean-libgit2: - -rm -rf $(LIBGIT2_SRC_DIR)/build/ + -rm -rf $(BUILDDIR)/$(LIBGIT2_SRC_DIR) -rm -f $(LIBGIT2_OBJ_TARGET) -get-libgit2: $(LIBGIT2_SRC_DIR)/CMakeLists.txt -configure-libgit2: $(LIBGIT2_SRC_DIR)/build/Makefile +get-libgit2: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt +configure-libgit2: $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile compile-libgit2: $(LIBGIT2_OBJ_SOURCE) -check-libgit2: $(LIBGIT2_SRC_DIR)/build/checked +check-libgit2: $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/checked install-libgit2: $(LIBGIT2_OBJ_TARGET) ## phony targets ## diff --git a/deps/jlchecksum b/deps/jlchecksum index 57120b5e0368c..3e64f693573c2 100755 --- a/deps/jlchecksum +++ b/deps/jlchecksum @@ -36,7 +36,7 @@ checksum_error() echo " But \`$CHECKSUM_PROG\` results in:" >&2 print_hash "$CURR_CHECKSUM" echo " This can happen due to bad downloads or network proxies, please check your" >&2 - echo " network proxy/firewall settings and delete deps/$BASENAME" >&2 + echo " network proxy/firewall settings and delete deps/srccache/$BASENAME" >&2 echo " to force a redownload when you are ready" >&2 echo "===============================================================================" >&2 exit 2 @@ -66,15 +66,15 @@ MD5_PROG="" find_checksum_progs() { if [ ! -z $(which sha512sum) ]; then - SHA512_PROG="sha512sum $BASENAME | awk '{ print \$1; }'" + SHA512_PROG="sha512sum $DIR/srccache/$BASENAME | awk '{ print \$1; }'" elif [ ! -z $(which shasum) ]; then - SHA512_PROG="shasum -a 512 $BASENAME | awk '{ print \$1; }'" + SHA512_PROG="shasum -a 512 $DIR/srccache/$BASENAME | awk '{ print \$1; }'" fi if [ ! -z $(which md5sum) ]; then - MD5_PROG="md5sum $BASENAME | awk '{ print \$1; }'" + MD5_PROG="md5sum $DIR/srccache/$BASENAME | awk '{ print \$1; }'" elif [ ! -z $(which md5) ]; then - MD5_PROG="md5 -q $BASENAME" + MD5_PROG="md5 -q $DIR/srccache/$BASENAME" fi } diff --git a/deps/osx-10.10.llvm-3.3.patch b/deps/osx-10.10.llvm-3.3.patch index 85897ae7f25c7..f22aaf03432f2 100644 --- a/deps/osx-10.10.llvm-3.3.patch +++ b/deps/osx-10.10.llvm-3.3.patch @@ -1,5 +1,5 @@ ---- Makefile.rules.old 2014-07-29 12:01:50.000000000 +0530 -+++ Makefile.rules 2014-07-29 12:02:10.000000000 +0530 +--- llvm-3.3.src/Makefile.rules.old 2014-07-29 12:01:50.000000000 +0530 ++++ llvm-3.3/Makefile.rules 2014-07-29 12:02:10.000000000 +0530 @@ -571,9 +571,9 @@ ifeq ($(HOST_OS),Darwin) DARWIN_VERSION := `sw_vers -productVersion` endif diff --git a/src/Makefile b/src/Makefile index f33e562c58d40..a17d5d325f685 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,4 +1,6 @@ -JULIAHOME = $(abspath ..) +SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +JULIAHOME := $(abspath $(SRCDIR)/..) +BUILDDIR ?= . include $(JULIAHOME)/deps/Versions.make include $(JULIAHOME)/Make.inc @@ -6,20 +8,23 @@ override CFLAGS += $(JCFLAGS) override CXXFLAGS += $(JCXXFLAGS) override CPPFLAGS += $(JCPPFLAGS) -BUILDDIR ?= . - SRCS = \ jltypes gf ast builtins module codegen disasm debuginfo interpreter \ alloc dlload sys init task array dump toplevel jl_uv jlapi signal-handling \ llvm-simdloop simplevector -HEADERS = julia.h julia_internal.h julia_version.h options.h $(wildcard support/*.h) $(LIBUV_INC)/uv.h +HEADERS = $(addprefix $(SRCDIR)/,julia.h julia_internal.h options.h) $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(LIBUV_INC)/uv.h +# -I BUILDDIR comes before -I SRCDIR so that the user can override on a per-build-directory basis +# for gcc/clang, suggested content is: +# #include_next +# #define ARGUMENT_TO_OVERRIDE 1 FLAGS = \ - -D_GNU_SOURCE -Iflisp -Isupport \ + -D_GNU_SOURCE -I$(BUILDDIR) -I$(SRCDIR) \ + -I$(SRCDIR)/flisp -I$(SRCDIR)/support \ -I$(call exec,$(LLVM_CONFIG) --includedir) \ -I$(LIBUV_INC) -I$(build_includedir) -DLIBRARY_EXPORTS \ - -I$(BUILDDIR) -I$(JULIAHOME)/deps/valgrind + -I$(JULIAHOME)/deps/valgrind ifneq ($(USEMSVC), 1) FLAGS += -Wall -Wno-strict-aliasing -fno-omit-frame-pointer -fvisibility=hidden -fno-common endif @@ -73,44 +78,44 @@ release debug: %: libjulia-% $(BUILDDIR): mkdir $(BUILDDIR) -$(BUILDDIR)/%.o: %.c $(HEADERS) | $(BUILDDIR) +$(BUILDDIR)/%.o: $(SRCDIR)/%.c $(HEADERS) | $(BUILDDIR) @$(call PRINT_CC, $(CC) $(CPPFLAGS) $(CFLAGS) $(SHIPFLAGS) $(DISABLE_ASSERTIONS) -c $< -o $@) -$(BUILDDIR)/%.dbg.obj: %.c $(HEADERS) | $(BUILDDIR) +$(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.c $(HEADERS) | $(BUILDDIR) @$(call PRINT_CC, $(CC) $(CPPFLAGS) $(CFLAGS) $(DEBUGFLAGS) -c $< -o $@) -$(BUILDDIR)/%.o: %.cpp $(HEADERS) $(shell which $(LLVM_CONFIG)) | $(BUILDDIR) +$(BUILDDIR)/%.o: $(SRCDIR)/%.cpp $(HEADERS) $(shell which $(LLVM_CONFIG)) | $(BUILDDIR) @$(call PRINT_CC, $(CXX) $(call exec,$(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(SHIPFLAGS) -c $< -o $@) -$(BUILDDIR)/%.dbg.obj: %.cpp $(HEADERS) $(shell which $(LLVM_CONFIG)) | $(BUILDDIR) +$(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.cpp $(HEADERS) $(shell which $(LLVM_CONFIG)) | $(BUILDDIR) @$(call PRINT_CC, $(CXX) $(call exec,$(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(DEBUGFLAGS) -c $< -o $@) $(BUILDDIR)/julia_flisp.boot.inc: $(BUILDDIR)/julia_flisp.boot $(FLISP_EXECUTABLE) - @$(call PRINT_FLISP, $(call spawn,$(FLISP_EXECUTABLE)) ./bin2hex.scm < $< > $@) + @$(call PRINT_FLISP, $(call spawn,$(FLISP_EXECUTABLE)) $(SRCDIR)/bin2hex.scm < $< > $@) -export julia_flisp_boot=$(BUILDDIR)/julia_flisp.boot -$(julia_flisp_boot): julia-parser.scm julia-syntax.scm \ - match.scm utils.scm jlfrontend.scm mk_julia_flisp_boot.scm $(FLISP_EXECUTABLE) - @$(call PRINT_FLISP, $(call spawn,$(FLISP_EXECUTABLE)) ./mk_julia_flisp_boot.scm) +$(BUILDDIR)/julia_flisp.boot: $(addprefix $(SRCDIR)/,jlfrontend.scm \ + julia-parser.scm julia-syntax.scm match.scm utils.scm mk_julia_flisp_boot.scm) \ + $(FLISP_EXECUTABLE) + @$(call PRINT_FLISP, $(call spawn,$(FLISP_EXECUTABLE)) $(SRCDIR)/mk_julia_flisp_boot.scm $(dir $<) $(notdir $<) $@) -$(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc flisp/*.h -$(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: intrinsics.cpp cgutils.cpp ccall.cpp abi_*.cpp -$(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: table.c -$(BUILDDIR)/gc.o $(BUILDDIR)/gc.dbg.obj: gc-debug.c -$(BUILDDIR)/signal-handling.o $(BUILDDIR)/signal-handling.dbg.obj: signals-*.c +$(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc $(SRCDIR)/flisp/*.h +$(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,intrinsics.cpp cgutils.cpp ccall.cpp abi_*.cpp) +$(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: $(SRCDIR)/table.c +$(BUILDDIR)/gc.o $(BUILDDIR)/gc.dbg.obj: $(SRCDIR)/gc-debug.c +$(BUILDDIR)/signal-handling.o $(BUILDDIR)/signal-handling.dbg.obj: $(addprefix $(SRCDIR)/,signals-*.c) -$(BUILDDIR)/support/libsupport.a: support/*.h support/*.c - $(MAKE) -C support BUILDDIR='$(abspath $(BUILDDIR)/support)' +$(BUILDDIR)/support/libsupport.a: $(SRCDIR)/support/*.h $(SRCDIR)/support/*.c + $(MAKE) -C $(SRCDIR)/support BUILDDIR='$(abspath $(BUILDDIR)/support)' -$(BUILDDIR)/support/libsupport-debug.a: support/*.h support/*.c - $(MAKE) -C support debug BUILDDIR='$(abspath $(BUILDDIR)/support)' +$(BUILDDIR)/support/libsupport-debug.a: $(SRCDIR)/support/*.h $(SRCDIR)/support/*.c + $(MAKE) -C $(SRCDIR)/support debug BUILDDIR='$(abspath $(BUILDDIR)/support)' $(FLISP_EXECUTABLE_release): $(BUILDDIR)/flisp/libflisp.a -$(BUILDDIR)/flisp/libflisp.a: flisp/*.h flisp/*.c $(BUILDDIR)/support/libsupport.a - $(MAKE) -C flisp BUILDDIR='$(abspath $(BUILDDIR)/flisp)' +$(BUILDDIR)/flisp/libflisp.a: $(addprefix $(SRCDIR)/,flisp/*.h flisp/*.c) $(BUILDDIR)/support/libsupport.a + $(MAKE) -C $(SRCDIR)/flisp BUILDDIR='$(abspath $(BUILDDIR)/flisp)' $(FLISP_EXECUTABLE_debug): $(BUILDDIR)/flisp/libflisp-debug.a -$(BUILDDIR)/flisp/libflisp-debug.a: flisp/*.h flisp/*.c $(BUILDDIR)/support/libsupport-debug.a - $(MAKE) -C flisp debug BUILDDIR='$(abspath $(BUILDDIR)/flisp)' +$(BUILDDIR)/flisp/libflisp-debug.a: $(addprefix $(SRCDIR)/,flisp/*.h flisp/*.c) $(BUILDDIR)/support/libsupport-debug.a + $(MAKE) -C $(SRCDIR)/flisp debug BUILDDIR='$(abspath $(BUILDDIR)/flisp)' -julia_version.h: ../VERSION +$(BUILDDIR)/julia_version.h: $(JULIAHOME)/VERSION @echo "// This is an autogenerated header file" > $@.$(JULIA_BUILD_MODE).tmp @echo "#ifndef JULIA_VERSION_H" >> $@.$(JULIA_BUILD_MODE).tmp @echo "#define JULIA_VERSION_H" >> $@.$(JULIA_BUILD_MODE).tmp @@ -132,11 +137,11 @@ else CXXLD = $(LD) -dll -export:jl_setjmp -export:jl_longjmp endif -$(build_shlibdir)/libjulia-debug.$(SHLIB_EXT): julia.expmap $(DOBJS) $(BUILDDIR)/flisp/libflisp-debug.a $(BUILDDIR)/support/libsupport-debug.a $(LIBUV) +$(build_shlibdir)/libjulia-debug.$(SHLIB_EXT): $(SRCDIR)/julia.expmap $(DOBJS) $(BUILDDIR)/flisp/libflisp-debug.a $(BUILDDIR)/support/libsupport-debug.a $(LIBUV) @$(call PRINT_LINK, $(CXXLD) $(CXXFLAGS) $(CXXLDFLAGS) $(DEBUGFLAGS) $(DOBJS) $(RPATH_ORIGIN) -o $@ $(LDFLAGS) $(JLIBLDFLAGS) $(DEBUG_LIBS)) $(INSTALL_NAME_CMD)libjulia-debug.$(SHLIB_EXT) $@ $(DSYMUTIL) $@ -$(BUILDDIR)/libjulia-debug.a: julia.expmap $(DOBJS) $(BUILDDIR)/flisp/libflisp-debug.a $(BUILDDIR)/support/libsupport-debug.a +$(BUILDDIR)/libjulia-debug.a: $(SRCDIR)/julia.expmap $(DOBJS) $(BUILDDIR)/flisp/libflisp-debug.a $(BUILDDIR)/support/libsupport-debug.a rm -f $@ @$(call PRINT_LINK, ar -rcs $@ $(DOBJS)) libjulia-debug: $(build_shlibdir)/libjulia-debug.$(SHLIB_EXT) @@ -147,7 +152,7 @@ else SONAME = endif -$(build_shlibdir)/libjulia.$(SHLIB_EXT): julia.expmap $(OBJS) $(BUILDDIR)/flisp/libflisp.a $(BUILDDIR)/support/libsupport.a $(LIBUV) +$(build_shlibdir)/libjulia.$(SHLIB_EXT): $(SRCDIR)/julia.expmap $(OBJS) $(BUILDDIR)/flisp/libflisp.a $(BUILDDIR)/support/libsupport.a $(LIBUV) @$(call PRINT_LINK, $(CXXLD) $(CXXFLAGS) $(CXXLDFLAGS) $(SHIPFLAGS) $(OBJS) $(RPATH_ORIGIN) -o $@ $(LDFLAGS) $(JLIBLDFLAGS) $(RELEASE_LIBS) $(SONAME)) $(CXXLDFLAGS) $(INSTALL_NAME_CMD)libjulia.$(SHLIB_EXT) $@ $(DSYMUTIL) $@ @@ -159,14 +164,14 @@ libjulia-release: $(build_shlibdir)/libjulia.$(SHLIB_EXT) clean: -rm -f $(build_shlibdir)/libjulia* -rm -f $(BUILDDIR)/julia_flisp.boot $(BUILDDIR)/julia_flisp.boot.inc - -rm -f $(BUILDDIR)/*.dbg.obj $(BUILDDIR)/*.o *~ $(BUILDDIR)/*.$(SHLIB_EXT) $(BUILDDIR)/*.a *# + -rm -f $(BUILDDIR)/*.dbg.obj $(BUILDDIR)/*.o $(BUILDDIR)/*.$(SHLIB_EXT) $(BUILDDIR)/*.a -rm -f $(BUILDDIR)/julia_version.h clean-flisp: - -$(MAKE) -C flisp clean BUILDDIR='$(abspath $(BUILDDIR)/flisp)' + -$(MAKE) -C $(SRCDIR)/flisp clean BUILDDIR='$(abspath $(BUILDDIR)/flisp)' clean-support: - -$(MAKE) -C support clean BUILDDIR='$(abspath $(BUILDDIR)/support)' + -$(MAKE) -C $(SRCDIR)/support clean BUILDDIR='$(abspath $(BUILDDIR)/support)' cleanall: clean clean-flisp clean-support diff --git a/src/julia.h b/src/julia.h index 22e19de7a6b84..73ffa40fd5b75 100644 --- a/src/julia.h +++ b/src/julia.h @@ -7,7 +7,11 @@ extern "C" { #endif -#include "options.h" +//** Configuration options that affect the Julia ABI **// +// if this is not defined, only individual dimension sizes are +// stored and not total length, to save space. +#define STORE_ARRAY_LEN +//** End Configuration options **// #include "libsupport.h" #include @@ -1567,7 +1571,7 @@ DLLEXPORT int jl_generating_output(void); #define JL_OPTIONS_USE_PRECOMPILED_NO 0 // Version information -#include "julia_version.h" +#include DLLEXPORT extern int jl_ver_major(void); DLLEXPORT extern int jl_ver_minor(void); diff --git a/src/julia_internal.h b/src/julia_internal.h index 230bcd9f6de5a..4a230bf7d31d9 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -3,8 +3,8 @@ #ifndef JULIA_INTERNAL_H #define JULIA_INTERNAL_H -#include "options.h" -#include "uv.h" +#include +#include #ifdef __cplusplus extern "C" { diff --git a/src/mk_julia_flisp_boot.scm b/src/mk_julia_flisp_boot.scm index 295474188e2c2..59985106d9517 100644 --- a/src/mk_julia_flisp_boot.scm +++ b/src/mk_julia_flisp_boot.scm @@ -1,2 +1,5 @@ -(load "jlfrontend.scm") -(make-system-image (os.getenv "julia_flisp_boot")) +(set! lastpwd (path.cwd)) +(path.cwd (cadr *argv*)) +(load (caddr *argv*)) +(path.cwd lastpwd) +(make-system-image (cadddr *argv*)) diff --git a/src/options.h b/src/options.h index c17495befef20..5cf154df152da 100644 --- a/src/options.h +++ b/src/options.h @@ -3,15 +3,14 @@ #ifndef JL_OPTIONS_H #define JL_OPTIONS_H +// Options in here are NOT allowed to affect the jlapi, since that would require this header to be installed + // Build-time options for debugging, tweaking, and selecting alternative // implementations of core features. +// // object layout options ------------------------------------------------------ -// if this is not defined, only individual dimension sizes are -// stored and not total length, to save space. -#define STORE_ARRAY_LEN - // how much space we're willing to waste if an array outgrows its // original object #define ARRAY_INLINE_NBYTES (2048*sizeof(void*)) diff --git a/ui/Makefile b/ui/Makefile index c9b5d0ea982f8..aaa6c904bbb97 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -1,4 +1,6 @@ -JULIAHOME = $(abspath ..) +SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +JULIAHOME := $(abspath $(SRCDIR)/..) +BUILDDIR ?= . include $(JULIAHOME)/deps/Versions.make include $(JULIAHOME)/Make.inc @@ -16,8 +18,8 @@ ifneq ($(USEMSVC), 1) FLAGS += -Wall -Wno-strict-aliasing -fno-omit-frame-pointer endif -OBJS = $(SRCS:%=%.o) -DOBJS = $(SRCS:%=%.dbg.obj) +OBJS = $(SRCS:%=$(BUILDDIR)/%.o) +DOBJS = $(SRCS:%=$(BUILDDIR)/%.dbg.obj) DEBUGFLAGS += $(FLAGS) SHIPFLAGS += $(FLAGS) ifeq ($(USE_LLVM_SHLIB),1) @@ -45,14 +47,14 @@ default: release all: release debug release debug : % : julia-% -%.o: %.c +$(BUILDDIR)/%.o: $(SRCDIR)/%.c @$(call PRINT_CC, $(CC) $(CPPFLAGS) $(CFLAGS) $(SHIPFLAGS) -c $< -o $@) -%.dbg.obj: %.c +$(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.c @$(call PRINT_CC, $(CC) $(CPPFLAGS) $(CFLAGS) $(DEBUGFLAGS) -c $< -o $@) ifeq ($(OS),WINNT) ifneq ($(USEMSVC), 1) -julia_res.o: $(JULIAHOME)/contrib/windows/julia.rc +$(BUILDDIR)/julia_res.o: $(JULIAHOME)/contrib/windows/julia.rc JLVER=`cat $(JULIAHOME)/VERSION` && \ JLVERi=`echo $$JLVER | perl -nle \ '/^(\d+)\.?(\d*)\.?(\d*)/ && \ From 16d38127293b7ae2635ef7cc9e8e7aebcd1b978f Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 4 Aug 2015 15:22:33 -0400 Subject: [PATCH 0045/1938] support O= output directory on toplevel make respects both $(JULIAHOME)/Make.user and $(O)/Make.user --- Make.inc | 24 +++- Makefile | 219 ++++++++++++++++++---------------- base/Makefile | 2 +- base/client.jl | 2 + base/fs.jl | 2 +- base/libc.jl | 2 +- base/pcre.jl | 2 +- base/rounding.jl | 6 +- base/stream.jl | 3 +- base/sysimg.jl | 8 +- contrib/windows/msys_build.sh | 1 + deps/Makefile | 116 ++++++++++-------- deps/jlchecksum | 9 +- doc/Makefile | 77 ++++++------ src/Makefile | 2 +- src/dump.c | 8 ++ src/flisp/Makefile | 5 +- src/support/Makefile | 5 +- test/Makefile | 21 ++-- test/perf/Makefile | 43 +++---- ui/repl.c | 6 +- 21 files changed, 313 insertions(+), 250 deletions(-) diff --git a/Make.inc b/Make.inc index 4b4a4cf4b365a..eee1e27f79b56 100644 --- a/Make.inc +++ b/Make.inc @@ -2,7 +2,8 @@ ## Note: ## It is generally preferable to change these options, for -## your local machine, in a file named `Make.user` in this directory +## your local machine, in a file named `Make.user` in the toplevel +## and build directories # OPENBLAS build options OPENBLAS_TARGET_ARCH= @@ -71,10 +72,26 @@ WITH_GC_DEBUG_ENV = 0 # Prevent picking up $ARCH from the environment variables ARCH= +# pick up BUILDROOT from O= if it isn't already set (from recursive make) +ifeq ($(BUILDROOT),) +ifeq ("$(origin O)", "command line") + BUILDROOT := $(abspath $O) + BUILDDIR := $(abspath $(BUILDROOT)/$(shell $(JULIAHOME)/contrib/relative_path.sh $(JULIAHOME) $(SRCDIR))) + $(info $(shell printf '\033[32;1mBuilding into $(BUILDROOT)\033[0m')) # use printf to expand the escape sequences +else + BUILDROOT:=$(JULIAHOME) +endif +endif +export BUILDROOT + # we include twice to pickup user definitions better +# include from JULIAHOME first so that BUILDROOT can override ifeq (exists, $(shell [ -e $(JULIAHOME)/Make.user ] && echo exists )) include $(JULIAHOME)/Make.user endif +ifeq (exists, $(shell [ -e $(BUILDROOT)/Make.user ] && echo exists )) +include $(BUILDROOT)/Make.user +endif # disable automatic Makefile rules .SUFFIXES: @@ -110,7 +127,7 @@ includedir = $(prefix)/include sysconfdir = $(prefix)/etc # Directories where things get built into -build_prefix = $(JULIAHOME)/usr +build_prefix = $(BUILDROOT)/usr build_staging = $(build_prefix)-staging build_bindir = $(build_prefix)/bin build_libdir = $(build_prefix)/lib @@ -399,6 +416,9 @@ endif ifeq (exists, $(shell [ -e $(JULIAHOME)/Make.user ] && echo exists )) include $(JULIAHOME)/Make.user endif +ifeq (exists, $(shell [ -e $(BUILDROOT)/Make.user ] && echo exists )) +include $(BUILDROOT)/Make.user +endif ifeq ($(SANITIZE),1) ifeq ($(SANITIZE_MEMORY),1) diff --git a/Makefile b/Makefile index 1904d62cc009e..1121a62647e9e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -JULIAHOME := $(abspath .) +JULIAHOME := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) include $(JULIAHOME)/Make.inc # TODO: Code bundled with Julia should be installed into a versioned directory, @@ -8,7 +8,7 @@ include $(JULIAHOME)/Make.inc # prefix/share/julia/site/VERSDIR (not prefix/share/julia/VERSDIR/site ... # so that prefix/share/julia/VERSDIR can be overwritten without touching # third-party code). -VERSDIR = v`cut -d. -f1-2 < VERSION` +VERSDIR = v`cut -d. -f1-2 < $(JULIAHOME)/VERSION` #file name of make binary-dist result ifeq ($(JULIA_BINARYDIST_TARNAME),) @@ -19,57 +19,66 @@ default: $(JULIA_BUILD_MODE) # contains either "debug" or "release" all: debug release # sort is used to remove potential duplicates -DIRS = $(sort $(build_bindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_man1dir)) +DIRS := $(sort $(build_bindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_man1dir)) +ifneq ($(BUILDROOT),$(JULIAHOME)) +BUILDDIRS := $(addprefix $(BUILDROOT)/,. base src ui doc deps test test/perf) +BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS)) +DIRS := $(DIRS) $(BUILDDIRS) +$(BUILDDIRMAKE): | $(BUILDDIRS) + @# add Makefiles to the build directories for convenience (pointing back to the source location of each) + @echo '# -- This file is automatically generated in julia/Makefile -- #' > $@ + @echo 'BUILDROOT=$(BUILDROOT)' >> $@ + @echo 'include $(JULIAHOME)/$(patsubst $(BUILDROOT)/%,%,$(@:/Makefile=))/Makefile' >> $@ +julia-deps: | $(BUILDDIRMAKE) +endif $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) $(foreach link,base test,$(eval $(call symlink_target,$(link),$(build_datarootdir)/julia))) # Build the HTML docs (skipped if already exists, notably in tarballs) -doc/_build/html: - @$(MAKE) -C doc html +$(BUILDROOT)/doc/_build/html: + @$(MAKE) -C $(BUILDROOT)/doc html # doc needs to live under $(build_docdir), not under $(build_datarootdir)/julia/ CLEAN_TARGETS += clean-docdir clean-docdir: @-rm -fr $(abspath $(build_docdir)) -$(build_prefix)/.examples: $(wildcard $(JULIAHOME)/examples/*.jl) $(shell find $(JULIAHOME)/examples/clustermanager) - @echo Copying in usr/share/doc/julia/examples - @-rm -fr $(build_docdir)/examples - @mkdir -p $(build_docdir)/examples - @cp -R $(JULIAHOME)/examples/*.jl $(build_docdir)/examples/ - @cp -R $(JULIAHOME)/examples/clustermanager $(build_docdir)/examples/ - @echo 1 > $@ +$(subst $(abspath $(BUILDROOT))/,,$(abspath $(build_docdir))): $(build_docdir) +$(build_docdir): + @mkdir -p $@/examples + @cp -R examples/*.jl $@/examples/ + @cp -R examples/clustermanager $@/examples/ julia-symlink: julia-ui-$(JULIA_BUILD_MODE) ifneq ($(OS),WINNT) ifndef JULIA_VAGRANT_BUILD - @ln -sf "$(shell contrib/relative_path.sh "$(JULIAHOME)" "$(JULIA_EXECUTABLE)")" julia + @ln -sf "$(shell $(JULIAHOME)/contrib/relative_path.sh "$(BUILDROOT)" "$(JULIA_EXECUTABLE)")" $(BUILDROOT)/julia endif endif -julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test - @$(MAKE) $(QUIET_MAKE) -C deps +julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test $(build_docdir) $(build_sysconfdir)/julia/juliarc.jl $(build_man1dir)/julia.1 + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/deps -julia-base: julia-deps $(build_sysconfdir)/julia/juliarc.jl $(build_man1dir)/julia.1 - @$(MAKE) $(QUIET_MAKE) -C base +julia-base: julia-deps + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/base julia-libccalltest: - @$(MAKE) $(QUIET_MAKE) -C test libccalltest + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test libccalltest julia-src-release julia-src-debug : julia-src-% : julia-deps - @$(MAKE) $(QUIET_MAKE) -C src libjulia-$* + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src libjulia-$* julia-ui-release julia-ui-debug : julia-ui-% : julia-src-% - @$(MAKE) $(QUIET_MAKE) -C ui julia-$* + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/ui julia-$* -julia-inference : julia-base julia-ui-$(JULIA_BUILD_MODE) $(build_prefix)/.examples - @$(MAKE) $(QUIET_MAKE) $(build_private_libdir)/inference.ji JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) +julia-inference : julia-base julia-ui-$(JULIA_BUILD_MODE) + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) $(build_private_libdir)/inference.ji JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) julia-sysimg-release : julia-inference julia-ui-release - @$(MAKE) $(QUIET_MAKE) $(build_private_libdir)/sys.$(SHLIB_EXT) JULIA_BUILD_MODE=release + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) $(build_private_libdir)/sys.$(SHLIB_EXT) JULIA_BUILD_MODE=release julia-sysimg-debug : julia-inference julia-ui-debug - @$(MAKE) $(QUIET_MAKE) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) JULIA_BUILD_MODE=debug + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) JULIA_BUILD_MODE=debug julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest @@ -77,28 +86,28 @@ debug release : % : julia-% check-whitespace: ifneq ($(NO_GIT), 1) - @contrib/check-whitespace.sh + @$(JULIAHOME)/contrib/check-whitespace.sh else $(warn "Skipping whitespace check because git is unavailable") endif release-candidate: release testall - @$(JULIA_EXECUTABLE) contrib/add_license_to_files.jl #add license headers - @$(JULIA_EXECUTABLE) doc/genstdlib.jl + @$(JULIA_EXECUTABLE) $(JULIAHOME)/contrib/add_license_to_files.jl #add license headers + @$(JULIA_EXECUTABLE) $(JULIAHOME)/doc/genstdlib.jl @#Check documentation - @$(JULIA_EXECUTABLE) doc/NEWS-update.jl #Add missing cross-references to NEWS.md - @$(MAKE) -C doc unicode #Rebuild Unicode table if necessary - @$(JULIA_EXECUTABLE) doc/DocCheck.jl > doc/UNDOCUMENTED.rst 2>&1 #Check for undocumented items - @if [ -z "$(cat doc/UNDOCUMENTED.rst)" ]; then \ - rm doc/UNDOCUMENTED.rst; \ + @$(JULIA_EXECUTABLE) $(JULIAHOME)/doc/NEWS-update.jl #Add missing cross-references to NEWS.md + @$(MAKE) -C $(BUILDROOT)/doc unicode #Rebuild Unicode table if necessary + @$(JULIA_EXECUTABLE) $(JULIAHOME)/doc/DocCheck.jl > $(BUILDROOT)/doc/UNDOCUMENTED.rst 2>&1 #Check for undocumented items + @if [ -z "$(cat $(BUILDROOT)/doc/UNDOCUMENTED.rst)" ]; then \ + rm $(BUILDROOT)/doc/UNDOCUMENTED.rst; \ else \ echo "Undocumented functions found in doc/UNDOCUMENTED.rst; document them, then retry"; \ exit 1; \ fi - @$(MAKE) -C doc html SPHINXOPTS="-n" #Rebuild Julia HTML docs pedantically - @$(MAKE) -C doc latex SPHINXOPTS="-n" #Rebuild Julia PDF docs pedantically - @$(MAKE) -C doc doctest #Run Julia doctests - @$(MAKE) -C doc linkcheck #Check all links + @$(MAKE) -C $(BUILDROOT)/doc html SPHINXOPTS="-n" #Rebuild Julia HTML docs pedantically + @$(MAKE) -C $(BUILDROOT)/doc latex SPHINXOPTS="-n" #Rebuild Julia PDF docs pedantically + @$(MAKE) -C $(BUILDROOT)/doc doctest #Run Julia doctests + @$(MAKE) -C $(BUILDROOT)/doc linkcheck #Check all links @# Check to see if the above make invocations changed anything important @if [ -n "$$(git status --porcelain)" ]; then \ @@ -107,7 +116,7 @@ release-candidate: release testall fi @#Check that benchmarks work - @$(MAKE) -C test/perf + @$(MAKE) -C $(BUILDROOT)/test/perf @#Check that netload tests work @#for test in test/netload/*.jl; do julia $$test; if [ $$? -ne 0 ]; then exit 1; fi; done @echo @@ -127,16 +136,14 @@ release-candidate: release testall @echo $(build_man1dir)/julia.1: $(JULIAHOME)/doc/man/julia.1 | $(build_man1dir) - @echo Copying in usr/share/man/man1/julia.1 @mkdir -p $(build_man1dir) @cp $< $@ $(build_sysconfdir)/julia/juliarc.jl: $(JULIAHOME)/etc/juliarc.jl | $(build_sysconfdir)/julia - @echo Creating usr/etc/julia/juliarc.jl @cp $< $@ ifeq ($(OS), WINNT) - @cat ./contrib/windows/juliarc.jl >> $(build_sysconfdir)/julia/juliarc.jl -$(build_sysconfdir)/julia/juliarc.jl: contrib/windows/juliarc.jl + @cat $(JULIAHOME)/contrib/windows/juliarc.jl >> $(build_sysconfdir)/julia/juliarc.jl +$(build_sysconfdir)/julia/juliarc.jl: $(JULIAHOME)/contrib/windows/juliarc.jl endif $(build_private_libdir)/%.$(SHLIB_EXT): $(build_private_libdir)/%.o @@ -146,7 +153,8 @@ $(build_private_libdir)/%.$(SHLIB_EXT): $(build_private_libdir)/%.o @$(INSTALL_NAME_CMD)$(notdir $@) $@ @$(DSYMUTIL) $@ -CORE_SRCS := base/boot.jl base/coreimg.jl \ +CORE_SRCS := $(addprefix $(JULIAHOME)/, \ + base/boot.jl base/coreimg.jl \ base/abstractarray.jl \ base/array.jl \ base/bool.jl \ @@ -169,33 +177,34 @@ CORE_SRCS := base/boot.jl base/coreimg.jl \ base/range.jl \ base/reduce.jl \ base/reflection.jl \ - base/tuple.jl -BASE_SRCS := $(shell find base -name \*.jl) + base/tuple.jl) +BASE_SRCS := $(shell find $(JULIAHOME)/base -name \*.jl) $(build_private_libdir)/inference0.ji: $(CORE_SRCS) | $(build_private_libdir) - @$(call PRINT_JULIA, cd base && \ + @$(call PRINT_JULIA, cd $(JULIAHOME)/base && \ $(call spawn,$(JULIA_EXECUTABLE)) -C $(JULIA_CPU_TARGET) --output-ji $(call cygpath_w,$@) -f \ coreimg.jl) $(build_private_libdir)/inference.ji: $(build_private_libdir)/inference0.ji - @$(call PRINT_JULIA, cd base && \ + @$(call PRINT_JULIA, cd $(JULIAHOME)/base && \ $(call spawn,$(JULIA_EXECUTABLE)) -C $(JULIA_CPU_TARGET) --output-ji $(call cygpath_w,$@) -f \ -J $(call cygpath_w,$<) coreimg.jl) +RELBUILDROOT := $(shell $(JULIAHOME)/contrib/relative_path.sh "$(JULIAHOME)/base" "$(BUILDROOT)/base/") COMMA:=, define sysimg_builder -$$(build_private_libdir)/sys$1.o: $$(build_private_libdir)/inference.ji VERSION $$(BASE_SRCS) - @$$(call PRINT_JULIA, cd base && \ +$$(build_private_libdir)/sys$1.o: $$(build_private_libdir)/inference.ji $(JULIAHOME)/VERSION $$(BASE_SRCS) + @$$(call PRINT_JULIA, cd $(JULIAHOME)/base && \ $$(call spawn,$2) -C $$(JULIA_CPU_TARGET) --output-o $$(call cygpath_w,$$@) $$(JULIA_SYSIMG_BUILD_FLAGS) -f \ - -J $$(call cygpath_w,$$<) sysimg.jl \ + -J $$(call cygpath_w,$$<) sysimg.jl $(RELBUILDROOT) \ || { echo '*** This error is usually fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***' && false; } ) .SECONDARY: $(build_private_libdir)/sys$1.o endef $(eval $(call sysimg_builder,,$(JULIA_EXECUTABLE_release))) $(eval $(call sysimg_builder,-debug,$(JULIA_EXECUTABLE_debug))) -$(build_bindir)/stringreplace: contrib/stringreplace.c | $(build_bindir) - @$(call PRINT_CC, $(HOSTCC) -o $(build_bindir)/stringreplace contrib/stringreplace.c) +$(build_bindir)/stringreplace: $(JULIAHOME)/contrib/stringreplace.c | $(build_bindir) + @$(call PRINT_CC, $(HOSTCC) -o $(build_bindir)/stringreplace $(JULIAHOME)/contrib/stringreplace.c) # public libraries, that are installed in $(prefix)/lib @@ -278,7 +287,7 @@ define stringreplace $(build_bindir)/stringreplace $$(strings -t x - $1 | grep '$2' | awk '{print $$1;}') '$3' 255 "$(call cygpath_w,$1)" endef -install: $(build_bindir)/stringreplace doc/_build/html +install: $(build_bindir)/stringreplace $(BUILDROOT)/doc/_build/html @$(MAKE) $(QUIET_MAKE) all @for subdir in $(bindir) $(libexecdir) $(datarootdir)/julia/site/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir); do \ mkdir -p $(DESTDIR)$$subdir; \ @@ -325,22 +334,22 @@ ifeq ($(OS),WINNT) endif $(INSTALL_F) $(build_includedir)/uv* $(DESTDIR)$(includedir)/julia endif - $(INSTALL_F) src/julia.h src/julia_version.h src/support/*.h $(DESTDIR)$(includedir)/julia + $(INSTALL_F) $(addprefix $(JULIAHOME)/,src/julia.h src/julia_version.h src/support/*.h) $(DESTDIR)$(includedir)/julia # Copy system image -$(INSTALL_F) $(build_private_libdir)/sys.ji $(DESTDIR)$(private_libdir) $(INSTALL_M) $(build_private_libdir)/sys.$(SHLIB_EXT) $(DESTDIR)$(private_libdir) $(INSTALL_M) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) $(DESTDIR)$(private_libdir) # Copy in system image build script - $(INSTALL_M) contrib/build_sysimg.jl $(DESTDIR)$(datarootdir)/julia/ + $(INSTALL_M) $(JULIAHOME)/contrib/build_sysimg.jl $(DESTDIR)$(datarootdir)/julia/ # Copy in standalone executable build script - $(INSTALL_M) contrib/build_executable.jl $(DESTDIR)$(datarootdir)/julia/ + $(INSTALL_M) $(JULIAHOME)/contrib/build_executable.jl $(DESTDIR)$(datarootdir)/julia/ # Copy in standalone julia-config script - $(INSTALL_M) contrib/julia-config.jl $(DESTDIR)$(datarootdir)/julia/ + $(INSTALL_M) $(JULIAHOME)/contrib/julia-config.jl $(DESTDIR)$(datarootdir)/julia/ # Copy in all .jl sources as well cp -R -L $(build_datarootdir)/julia $(DESTDIR)$(datarootdir)/ # Copy documentation cp -R -L $(build_docdir)/* $(DESTDIR)$(docdir)/ - cp -R -L doc/_build/html $(DESTDIR)$(docdir)/ + cp -R -L $(BUILDROOT)/doc/_build/html $(DESTDIR)$(docdir)/ -rm $(DESTDIR)$(docdir)/html/.buildinfo # Remove perf suite -rm -rf $(DESTDIR)$(datarootdir)/julia/test/perf/ @@ -351,14 +360,14 @@ endif $(INSTALL_F) $(build_man1dir)/julia.1 $(DESTDIR)$(man1dir)/ # Copy icon and .desktop file mkdir -p $(DESTDIR)$(datarootdir)/icons/hicolor/scalable/apps/ - $(INSTALL_F) contrib/julia.svg $(DESTDIR)$(datarootdir)/icons/hicolor/scalable/apps/ + $(INSTALL_F) $(JULIAHOME)/contrib/julia.svg $(DESTDIR)$(datarootdir)/icons/hicolor/scalable/apps/ -touch -c $(DESTDIR)$(datarootdir)/icons/hicolor/ -gtk-update-icon-cache $(DESTDIR)$(datarootdir)/icons/hicolor/ mkdir -p $(DESTDIR)$(datarootdir)/applications/ - $(INSTALL_F) contrib/julia.desktop $(DESTDIR)$(datarootdir)/applications/ + $(INSTALL_F) $(JULIAHOME)/contrib/julia.desktop $(DESTDIR)$(datarootdir)/applications/ # Install appdata file mkdir -p $(DESTDIR)$(datarootdir)/appdata/ - $(INSTALL_F) contrib/julia.appdata.xml $(DESTDIR)$(datarootdir)/appdata/ + $(INSTALL_F) $(JULIAHOME)/contrib/julia.appdata.xml $(DESTDIR)$(datarootdir)/appdata/ # Update RPATH entries and JL_SYSTEM_IMAGE_PATH if $(private_libdir_rel) != $(build_private_libdir_rel) ifneq ($(private_libdir_rel),$(build_private_libdir_rel)) @@ -382,7 +391,7 @@ endif cp -R $(build_sysconfdir)/julia $(DESTDIR)$(sysconfdir)/ distclean dist-clean: - rm -fr julia-*.tar.gz julia*.exe julia-*.7z julia-$(JULIA_COMMIT) + rm -fr $(BUILDROOT)/julia-*.tar.gz $(BUILDROOT)/julia*.exe $(BUILDROOT)/julia-*.7z $(BUILDROOT)/julia-$(JULIA_COMMIT) dist: @echo \'dist\' target is deprecated: use \'binary-dist\' instead. @@ -402,23 +411,23 @@ endif ifneq ($(DESTDIR),) $(error DESTDIR must not be set for make binary-dist) endif - @$(MAKE) install - cp LICENSE.md $(prefix) + @$(MAKE) -C $(BUILDROOT) -f $(JULIAHOME)/Makefile install + cp $(JULIAHOME)/LICENSE.md $(prefix) ifneq ($(OS), WINNT) - -./contrib/fixup-libgfortran.sh $(DESTDIR)$(private_libdir) + -$(JULIAHOME)/contrib/fixup-libgfortran.sh $(DESTDIR)$(private_libdir) endif ifeq ($(OS), Linux) - -./contrib/fixup-libstdc++.sh $(DESTDIR)$(private_libdir) + -$(JULIAHOME)/contrib/fixup-libstdc++.sh $(DESTDIR)$(private_libdir) endif # Copy in juliarc.jl files per-platform for binary distributions as well # Note that we don't install to sysconfdir: we always install to $(DESTDIR)$(prefix)/etc. # If you want to make a distribution with a hardcoded path, you take care of installation ifeq ($(OS), Darwin) - -cat ./contrib/mac/juliarc.jl >> $(DESTDIR)$(prefix)/etc/julia/juliarc.jl + -cat $(JULIAHOME)/contrib/mac/juliarc.jl >> $(DESTDIR)$(prefix)/etc/julia/juliarc.jl endif ifeq ($(OS), WINNT) - [ ! -d dist-extras ] || ( cd dist-extras && \ + [ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \ cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(bindir) && \ mkdir $(DESTDIR)$(prefix)/Git && \ 7z x PortableGit.7z -o"$(DESTDIR)$(prefix)/Git" && \ @@ -432,20 +441,21 @@ ifeq ($(OS), WINNT) cd $(prefix) && find * | sed -e 's/\//\\/g' -e 's/$$/\r/g' > etc/uninstall.log # build nsis package - $(call spawn,./dist-extras/nsis/makensis.exe) -NOCD -DVersion=$(JULIA_VERSION) -DArch=$(ARCH) -DCommit=$(JULIA_COMMIT) ./contrib/windows/build-installer.nsi + $(call spawn,$(JULIAHOME)/dist-extras/nsis/makensis.exe) -NOCD -DVersion=$(JULIA_VERSION) -DArch=$(ARCH) -DCommit=$(JULIA_COMMIT) $(JULIAHOME)/contrib/windows/build-installer.nsi # compress nsis installer and combine with 7zip self-extracting header - ./dist-extras/7z a -mx9 "julia-install-$(JULIA_COMMIT)-$(ARCH).7z" julia-installer.exe - cat ./contrib/windows/7zS.sfx ./contrib/windows/7zSFX-config.txt "julia-install-$(JULIA_COMMIT)-$(ARCH).7z" > "julia-${JULIA_VERSION}-${ARCH}.exe" - -rm -f julia-installer.exe + cd $(BUILDROOT) && $(JULIAHOME)/dist-extras/7z a -mx9 "julia-install-$(JULIA_COMMIT)-$(ARCH).7z" julia-installer.exe + cd $(BUILDROOT) && cat $(JULIAHOME)/contrib/windows/7zS.sfx $(JULIAHOME)/contrib/windows/7zSFX-config.txt "julia-install-$(JULIA_COMMIT)-$(ARCH).7z" > "julia-${JULIA_VERSION}-${ARCH}.exe" + -rm -f $(BUILDROOT)/julia-installer.exe else - $(TAR) zcvf $(JULIA_BINARYDIST_TARNAME).tar.gz julia-$(JULIA_COMMIT) + cd $(BUILDROOT) && $(TAR) zcvf $(JULIA_BINARYDIST_TARNAME).tar.gz julia-$(JULIA_COMMIT) endif rm -fr $(prefix) -light-source-dist.tmp: doc/_build/html +# this target does not accept BUILDROOT +light-source-dist.tmp: $(JULIAHOME)/doc/_build/html # Save git information - -@$(MAKE) -C base version_git.jl.phony + -@$(MAKE) -C $(JULIAHOME)/base version_git.jl.phony # Create file light-source-dist.tmp to hold all the filenames that go into the tarball echo "base/version_git.jl" > light-source-dist.tmp @@ -453,6 +463,7 @@ light-source-dist.tmp: doc/_build/html find doc/_build/html >> light-source-dist.tmp # Make tarball with only Julia code +# this target does not accept BUILDROOT light-source-dist: light-source-dist.tmp # Prefix everything with the current directory name (usually "julia"), then create tarball DIRNAME=$$(basename $$(pwd)); \ @@ -463,13 +474,14 @@ source-dist: @echo \'source-dist\' target is deprecated: use \'full-source-dist\' instead. # Make tarball with Julia code plus all dependencies +# this target does not accept BUILDROOT full-source-dist: light-source-dist.tmp # Get all the dependencies downloaded @$(MAKE) -C deps getall NO_GIT=1 # Create file full-source-dist.tmp to hold all the filenames that go into the tarball cp light-source-dist.tmp full-source-dist.tmp - -ls deps/*.tar.gz deps/*.tar.bz2 deps/*.tar.xz deps/*.tgz deps/*.zip >> full-source-dist.tmp + -ls deps/srccache/*.tar.gz deps/srccache/*.tar.bz2 deps/srccache/*.tar.xz deps/srccache/*.tgz deps/srccache/*.zip >> full-source-dist.tmp # Prefix everything with the current directory name (usually "julia"), then create tarball DIRNAME=$$(basename $$(pwd)); \ @@ -477,30 +489,31 @@ full-source-dist: light-source-dist.tmp cd ../ && tar -cz -T $$DIRNAME/full-source-dist.tmp1 --no-recursion -f $$DIRNAME/julia-$(JULIA_VERSION)_$(JULIA_COMMIT)-full.tar.gz clean: | $(CLEAN_TARGETS) - @$(MAKE) -C base clean - @$(MAKE) -C doc clean - @$(MAKE) -C src clean - @$(MAKE) -C ui clean - @$(MAKE) -C test clean - @rm -f julia - @rm -f *~ *# *.tar.gz + @$(MAKE) -C $(BUILDROOT)/base clean + @$(MAKE) -C $(BUILDROOT)/doc clean + @$(MAKE) -C $(BUILDROOT)/src clean + @$(MAKE) -C $(BUILDROOT)/ui clean + @$(MAKE) -C $(BUILDROOT)/test clean + @rm -f $(BUILDROOT)/julia + @rm -f $(BUILDROOT)/*.tar.gz @rm -f $(build_bindir)/stringreplace \ - light-source-dist.tmp light-source-dist.tmp1 \ - full-source-dist.tmp full-source-dist.tmp1 + $(BUILDROOT)/light-source-dist.tmp $(BUILDROOT)/light-source-dist.tmp1 \ + $(BUILDROOT)/full-source-dist.tmp $(BUILDROOT)/full-source-dist.tmp1 @rm -fr $(build_private_libdir) - @rm -f $(build_prefix)/.examples +# Temporarily add this line to the Makefile to remove extras + @rm -fr $(build_datarootdir)/julia/extras cleanall: clean - @$(MAKE) -C src clean-flisp clean-support + @$(MAKE) -C $(BUILDROOT)/src clean-flisp clean-support @rm -fr $(build_shlibdir) ifeq ($(OS),WINNT) @rm -rf $(build_prefix)/lib endif - @$(MAKE) -C deps clean-libuv + @$(MAKE) -C $(BUILDROOT)/deps clean-libuv distcleanall: cleanall - @$(MAKE) -C deps distcleanall - @$(MAKE) -C doc cleanall + @$(MAKE) -C $(BUILDROOT)/deps distcleanall + @$(MAKE) -C $(BUILDROOT)/doc cleanall rm -fr $(build_prefix) $(build_staging) .PHONY: default debug release check-whitespace release-candidate \ @@ -513,53 +526,53 @@ distcleanall: cleanall dist full-source-dist source-dist test: check-whitespace $(JULIA_BUILD_MODE) - @$(MAKE) $(QUIET_MAKE) -C test default JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test default JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) testall: check-whitespace $(JULIA_BUILD_MODE) - cp $(build_prefix)/lib/julia/sys$(JULIA_LIBSUFFIX).$(SHLIB_EXT) local.$(SHLIB_EXT) && $(JULIA_EXECUTABLE) -J local.$(SHLIB_EXT) -e 'true' && rm local.$(SHLIB_EXT) - @$(MAKE) $(QUIET_MAKE) -C test all JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) + cp $(build_prefix)/lib/julia/sys$(JULIA_LIBSUFFIX).$(SHLIB_EXT) $(BUILDROOT)/local.$(SHLIB_EXT) && $(JULIA_EXECUTABLE) -J $(BUILDROOT)/local.$(SHLIB_EXT) -e 'true' && rm $(BUILDROOT)/local.$(SHLIB_EXT) + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test all JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) testall1: check-whitespace $(JULIA_BUILD_MODE) - @env JULIA_CPU_CORES=1 $(MAKE) $(QUIET_MAKE) -C test all JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) + @env JULIA_CPU_CORES=1 $(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test all JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) test-%: check-whitespace $(JULIA_BUILD_MODE) - @$(MAKE) $(QUIET_MAKE) -C test $* JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test $* JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) perf: release - @$(MAKE) $(QUIET_MAKE) -C test/perf JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test/perf JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) perf-%: release - @$(MAKE) $(QUIET_MAKE) -C test/perf $* JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test/perf $* JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) # download target for some hardcoded windows dependencies .PHONY: win-extras wine_path win-extras: - [ -d dist-extras ] || mkdir dist-extras + [ -d $(JULIAHOME)/dist-extras ] || mkdir $(JULIAHOME)/dist-extras ifneq ($(BUILD_OS),WINNT) ifeq (,$(findstring CYGWIN,$(BUILD_OS))) - cp /usr/lib/p7zip/7z /usr/lib/p7zip/7z.so dist-extras + cp /usr/lib/p7zip/7z /usr/lib/p7zip/7z.so $(JULIAHOME)/dist-extras endif endif ifneq (,$(filter $(ARCH), i386 i486 i586 i686)) - cd dist-extras && \ + cd $(JULIAHOME)/dist-extras && \ $(JLDOWNLOAD) http://downloads.sourceforge.net/sevenzip/7z920.exe && \ 7z x -y 7z920.exe 7z.exe 7z.dll && \ ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win32/openSUSE_13.1 \ - "mingw32-libgfortran3 mingw32-libquadmath0 mingw32-libstdc++6 mingw32-libgcc_s_sjlj1 mingw32-libssp0 mingw32-libexpat1 mingw32-zlib1" && \ + "mingw32-libgfortran3 mingw32-libquadmath0 mingw32-libstdc++6 mingw32-libgcc_s_sjlj1 mingw32-libssp0 mingw32-libexpat1 mingw32-zlib1" && \ cp usr/i686-w64-mingw32/sys-root/mingw/bin/*.dll . else ifeq ($(ARCH),x86_64) - cd dist-extras && \ + cd $(JULIAHOME)/dist-extras && \ $(JLDOWNLOAD) 7z920-x64.msi http://downloads.sourceforge.net/sevenzip/7z920-x64.msi && \ 7z x -y 7z920-x64.msi _7z.exe _7z.dll && \ mv _7z.dll 7z.dll && \ mv _7z.exe 7z.exe && \ ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win64/openSUSE_13.1 \ - "mingw64-libgfortran3 mingw64-libquadmath0 mingw64-libstdc++6 mingw64-libgcc_s_seh1 mingw64-libssp0 mingw64-libexpat1 mingw64-zlib1" && \ + "mingw64-libgfortran3 mingw64-libquadmath0 mingw64-libstdc++6 mingw64-libgcc_s_seh1 mingw64-libssp0 mingw64-libexpat1 mingw64-zlib1" && \ cp usr/x86_64-w64-mingw32/sys-root/mingw/bin/*.dll . else $(error no win-extras target for ARCH=$(ARCH)) endif - cd dist-extras && \ + cd $(JULIAHOME)/dist-extras && \ $(JLDOWNLOAD) http://downloads.sourceforge.net/sevenzip/7z920_extra.7z && \ $(JLDOWNLOAD) https://unsis.googlecode.com/files/nsis-2.46.5-Unicode-setup.exe && \ $(JLDOWNLOAD) busybox.exe http://frippery.org/files/busybox/busybox-w32-FRP-1-g9eb16cb.exe && \ diff --git a/base/Makefile b/base/Makefile index 57846b05ef23a..1c673d7db2004 100644 --- a/base/Makefile +++ b/base/Makefile @@ -1,6 +1,6 @@ SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +BUILDDIR := . JULIAHOME := $(abspath $(SRCDIR)/..) -BUILDDIR ?= . include $(JULIAHOME)/deps/Versions.make include $(JULIAHOME)/Make.inc diff --git a/base/client.jl b/base/client.jl index 664c7b65323da..edc452845069c 100644 --- a/base/client.jl +++ b/base/client.jl @@ -406,6 +406,8 @@ function _atreplinit(repl) end function _start() + empty!(ARGS) + append!(ARGS, Core.ARGS) opts = JLOptions() try (quiet,repl,startup,color_set,history_file) = process_options(opts,copy(ARGS)) diff --git a/base/fs.jl b/base/fs.jl index db991a47a9206..b2cac54d5ad8c 100644 --- a/base/fs.jl +++ b/base/fs.jl @@ -47,7 +47,7 @@ export File, import Base: uvtype, uvhandle, eventloop, fd, position, stat, close, write, read, read!, readbytes, isopen, _sizeof_uv_fs, uv_error -include("file_constants.jl") +include(string(length(Core.ARGS)>=2?Core.ARGS[2]:"","file_constants.jl")) # include($BUILDROOT/base/file_constants.jl) abstract AbstractFile <: IO diff --git a/base/libc.jl b/base/libc.jl index 26e1dc161d90c..fe0a2fe3b0467 100644 --- a/base/libc.jl +++ b/base/libc.jl @@ -6,7 +6,7 @@ export FILE, TmStruct, strftime, strptime, getpid, gethostname, free, malloc, ca errno, strerror, flush_cstdio, systemsleep, time @windows_only export GetLastError, FormatMessage -include("errno.jl") +include(string(length(Core.ARGS)>=2?Core.ARGS[2]:"","errno_h.jl")) # include($BUILDROOT/base/errno_h.jl) ## RawFD ## diff --git a/base/pcre.jl b/base/pcre.jl index ec7a865aca94c..aa24f8690145f 100644 --- a/base/pcre.jl +++ b/base/pcre.jl @@ -4,7 +4,7 @@ module PCRE -include("pcre_h.jl") +include(string(length(Core.ARGS)>=2?Core.ARGS[2]:"","pcre_h.jl")) # include($BUILDROOT/base/pcre_h.jl) const PCRE_LIB = "libpcre2-8" diff --git a/base/rounding.jl b/base/rounding.jl index 96490407452c2..f82821ae3632f 100644 --- a/base/rounding.jl +++ b/base/rounding.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license module Rounding -include("fenv_constants.jl") +include(UTF8String(vcat(length(Core.ARGS)>=2?Core.ARGS[2].data:"".data, "fenv_constants.jl".data))) # include($BUILDROOT/base/fenv_constants.jl) export RoundingMode, RoundNearest, RoundToZero, RoundUp, RoundDown, RoundFromZero, @@ -41,8 +41,8 @@ function from_fenv(r::Integer) end end -set_rounding_raw{T<:Union{Float32,Float64}}(::Type{T},i::Integer) = ccall(:fesetround, Cint, (Cint,), i) -get_rounding_raw{T<:Union{Float32,Float64}}(::Type{T}) = ccall(:fegetround, Cint, ()) +set_rounding_raw{T<:Union{Float32,Float64}}(::Type{T},i::Integer) = ccall(:fesetround, Int32, (Int32,), i) +get_rounding_raw{T<:Union{Float32,Float64}}(::Type{T}) = ccall(:fegetround, Int32, ()) set_rounding{T<:Union{Float32,Float64}}(::Type{T},r::RoundingMode) = set_rounding_raw(T,to_fenv(r)) get_rounding{T<:Union{Float32,Float64}}(::Type{T}) = from_fenv(get_rounding_raw(T)) diff --git a/base/stream.jl b/base/stream.jl index b254a1f5e8e76..c6b84c2539015 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -1,7 +1,6 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -#TODO: Move stdio detection from C to Julia (might require some Clang magic) -include("uv_constants.jl") +include(string(length(Core.ARGS)>=2?Core.ARGS[2]:"","uv_constants.jl")) # include($BUILDROOT/base/uv_constants.jl) import .Libc: RawFD, dup @windows_only import .Libc: WindowsRawSocket diff --git a/base/sysimg.jl b/base/sysimg.jl index 8e6d20f1a3341..f352779f71bbb 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -31,9 +31,6 @@ include("essentials.jl") include("docs/bootstrap.jl") include("base.jl") include("reflection.jl") -include("build_h.jl") -include("version_git.jl") -include("c.jl") include("options.jl") # core operations & types @@ -81,7 +78,10 @@ include("dict.jl") include("set.jl") include("iterator.jl") -# For OS specific stuff in I/O +# For OS specific stuff +include(UTF8String(vcat(length(Core.ARGS)>=2?Core.ARGS[2].data:"".data, "build_h.jl".data))) # include($BUILDROOT/base/build_h.jl) +include(UTF8String(vcat(length(Core.ARGS)>=2?Core.ARGS[2].data:"".data, "version_git.jl".data))) # include($BUILDROOT/base/version_git.jl) +include("c.jl") include("osutils.jl") # strings & printing diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index bb4b88c84cc2a..27caae65a75a7 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -110,6 +110,7 @@ done for i in share/julia/base/pcre_h.jl; do $SEVENZIP e -y julia-installer.exe "\$_OUTDIR/$i" -obase >> get-deps.log done +echo "override PCRE_INCL_PATH =" >> Make.user # suppress "bash.exe: warning: could not find /tmp, please create!" mkdir -p usr/Git/tmp # Remove libjulia.dll if it was copied from downloaded binary diff --git a/deps/Makefile b/deps/Makefile index ddfaf79259708..1cf751a5ceb95 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -1,7 +1,7 @@ ## high-level setup ## SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) JULIAHOME := $(abspath $(SRCDIR)/..) -BUILDDIR ?= . +BUILDDIR := . include $(SRCDIR)/Versions.make include $(JULIAHOME)/Make.inc @@ -67,7 +67,7 @@ endif # If the top-level Makefile is called with environment variables, # they will override the values passed above to ./configure -MAKE_COMMON = DESTDIR="" prefix=$(build_prefix) bindir=$(build_bindir) libdir=$(build_libdir) libexecdir=$(build_libexecdir) datarootdir=$(build_datarootdir) includedir=$(build_includedir) sysconfdir=$(build_sysconfdir) +MAKE_COMMON = DESTDIR="" prefix=$(build_prefix) bindir=$(build_bindir) libdir=$(build_libdir) libexecdir=$(build_libexecdir) datarootdir=$(build_datarootdir) includedir=$(build_includedir) sysconfdir=$(build_sysconfdir) O= ## Overall configuration of which rules exist and should be run by default ## @@ -248,6 +248,7 @@ endef # # this defines rules for: # VARNAME_SRC_DIR = source directory for the output, relative to $(SRCDIR)/srccache or $(BUILDDIR) +# VARNAME_SRC_FILE = target file for make get-VARNAME target # dirname: # dirname/file_from_download: # distclean-dirname: @@ -257,9 +258,10 @@ include $(SRCDIR)/$1.version ifneq ($(NO_GIT),1) $2_SRC_DIR = $1 -$$(SRCDIR)/srccache/$1.git: | $$(SRCDIR)/srccache +$2_SRC_FILE = $$(SRCDIR)/srccache/$1.git +$$($2_SRC_FILE): | $$(SRCDIR)/srccache git clone -q --mirror --depth=10 --branch $$($2_BRANCH) $$($2_GIT_URL) $$@ -$5/$1: | $$(SRCDIR)/srccache/$1.git +$5/$1: | $$($2_SRC_FILE) # try to update the cache, if that fails, attempt to continue anyways (the ref might already be local) -cd $$(SRCDIR)/srccache/$1.git && git fetch -q $$($2_GIT_URL) $$($2_BRANCH):remotes/origin/$$($2_BRANCH) git clone -q --branch $$($2_BRANCH) $$(SRCDIR)/srccache/$1.git $$@ @@ -287,22 +289,24 @@ $5/$1/$4: $$(JULIAHOME)/.git/modules/deps/$1/HEAD endif else #unconditionally add dependency on expected location $5/$1/$4: $5/$1/.git/HEAD +$5/$1/.git/HEAD: | $5/$1 endif else # NO_GIT $2_SRC_DIR = $1-$$($2_SHA1) -$$(SRCDIR)/$$($2_SRC_DIR).tar.gz: +$2_SRC_FILE = $$(SRCDIR)/srccache/$$($2_SRC_DIR).tar.gz +$$($2_SRC_FILE): | $$(SRCDIR)/srccache $$(JLDOWNLOAD) $$@ $$(call $2_TAR_URL,$$($2_SHA1)) -$5/$$($2_SRC_DIR)/$3: $$(SRCDIR)/$$($2_SRC_DIR).tar.gz - $$(JLCHECKSUM) $< +$5/$$($2_SRC_DIR)/$3: $$($2_SRC_FILE) + $$(JLCHECKSUM) $$< mkdir -p $$(dir $$@) && \ $(TAR) -C $$(dir $$@) --strip-components 1 -xf $$< touch -c $$@ -endif +endif # NO_GIT distclean-$1: - -rm -rf $5/$$($2_SRC_DIR) $$(SRCDIR)/srccache/$$($2_SRC_DIR).tar.gz + -rm -rf $5/$$($2_SRC_DIR) $$($2_SRC_FILE) endef @@ -495,25 +499,25 @@ LLVM_MFLAGS += OPTIONAL_DIRS= endif ifneq ($(LLVM_CLANG_TAR),) -$(LLVM_CLANG_TAR): +$(LLVM_CLANG_TAR): | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://llvm.org/releases/$(LLVM_VER)/$(notdir $@) endif ifneq ($(LLVM_COMPILER_RT_TAR),) -$(LLVM_COMPILER_RT_TAR): +$(LLVM_COMPILER_RT_TAR): | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://llvm.org/releases/$(LLVM_VER)/$(notdir $@) endif ifneq ($(LLVM_LIBCXX_TAR),) -$(LLVM_LIBCXX_TAR): +$(LLVM_LIBCXX_TAR): | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://llvm.org/releases/$(LLVM_VER)/$(notdir $@) endif ifneq ($(LLVM_VER),svn) -$(LLVM_TAR): +$(LLVM_TAR): | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://llvm.org/releases/$(LLVM_VER)/$(notdir $@) endif ifneq ($(LLVM_LLDB_TAR),) -$(LLVM_LLDB_TAR): +$(LLVM_LLDB_TAR): | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://llvm.org/releases/$(LLVM_VER)/$(notdir $@) endif ifeq ($(BUILD_LLDB),1) @@ -656,7 +660,7 @@ endif # LLVM_VER # Apply version-specific LLVM patches LLVM_PATCH_LIST= define LLVM_PATCH -$$(LLVM_SRC_DIR)/$1.patch-applied: $$(SRCDIR)/$1.patch +$$(LLVM_SRC_DIR)/$1.patch-applied: | $$(SRCDIR)/$1.patch cd $$(LLVM_SRC_DIR) && patch -p1 < $$< echo 1 > $$@ LLVM_PATCH_LIST += $$(LLVM_SRC_DIR)/$1.patch-applied @@ -789,6 +793,7 @@ $(BUILDDIR)/$(LIBUV_SRC_DIR)/config.status: $(SRCDIR)/srccache/$(LIBUV_SRC_DIR)/ touch -c $(SRCDIR)/srccache/$(LIBUV_SRC_DIR)/aclocal.m4 # touch a few files to prevent autogen from getting called touch -c $(SRCDIR)/srccache/$(LIBUV_SRC_DIR)/Makefile.in touch -c $(SRCDIR)/srccache/$(LIBUV_SRC_DIR)/configure + mkdir -p $(dir $@) cd $(dir $@) && \ $< --with-pic $(CONFIGURE_COMMON) $(UV_FLAGS) touch -c $@ @@ -808,7 +813,7 @@ clean-libuv: -$(MAKE) -C $(BUILDDIR)/$(LIBUV_SRC_DIR) clean -rm -rf $(build_libdir)/libuv.a $(build_libdir)/libuv.la $(build_includedir)/libuv.h $(build_includedir)/libuv-private -get-libuv: $(SRCDIR)/srccache/$(LIBUV_SRC_DIR)/configure +get-libuv: $(LIBUV_SRC_FILE) configure-libuv: $(BUILDDIR)/$(LIBUV_SRC_DIR)/config.status compile-libuv: $(UV_SRC_TARGET) check-libuv: $(BUILDDIR)/$(LIBUV_SRC_DIR)/checked @@ -890,7 +895,7 @@ clean-openlibm: -rm $(OPENLIBM_OBJ_TARGET) -rm $(build_libdir)/libopenlibm.a -get-openlibm: $(SRCDIR)/srccache/$(OPENLIBM_SRC_DIR).git +get-openlibm: $(OPENLIBM_SRC_FILE) configure-openlibm: $(BUILDDIR)/$(OPENLIBM_SRC_DIR)/Makefile compile-openlibm: $(OPENLIBM_OBJ_SOURCE) check-openlibm: compile-openlibm @@ -930,7 +935,7 @@ clean-openspecfun: -rm $(OPENSPECFUN_OBJ_TARGET) -rm $(build_libdir)/libopenspecfun.a -get-openspecfun: $(SRCDIR)/srccache/$(OPENSPECFUN_SRC_DIR).git +get-openspecfun: $(OPENSPECFUN_SRC_FILE) configure-openspecfun: $(BUILDDIR)/$(OPENSPECFUN_SRC_DIR)/Makefile compile-openspecfun: $(OPENSPECFUN_OBJ_SOURCE) check-openspecfun: compile-openspecfun @@ -953,7 +958,7 @@ ifeq ($(ARCH), x86_64) DSFMT_CFLAGS += -msse2 -DHAVE_SSE2 endif -$(SRCDIR)/srccache/dsfmt-$(DSFMT_VER).tar.gz: +$(SRCDIR)/srccache/dsfmt-$(DSFMT_VER).tar.gz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/dSFMT-src-$(DSFMT_VER).tar.gz touch -c $@ $(BUILDDIR)/dsfmt-$(DSFMT_VER)/config.status: $(SRCDIR)/srccache/dsfmt-$(DSFMT_VER).tar.gz @@ -997,9 +1002,9 @@ RMATH_JULIA_FLAGS += CC="$(CC)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) \ USE_DSFMT=1 DSFMT_libdir="$(build_shlibdir)" \ DSFMT_includedir="$(build_includedir)" -$(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER).tar.gz: +$(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER).tar.gz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://api.github.com/repos/JuliaLang/Rmath-julia/tarball/v$(RMATH_JULIA_VER) -$(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER)/Makefile: $(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER).tar.gz +$(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER)/Makefile: $(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER).tar.gz | $(SRCDIR)/srccache $(JLCHECKSUM) $< mkdir -p $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER) $(TAR) -C $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER) --strip-components 1 -xf $< @@ -1028,7 +1033,7 @@ install-Rmath-julia: $(RMATH_JULIA_OBJ_TARGET) OBJCONV_SOURCE = $(BUILDDIR)/objconv/objconv OBJCONV_TARGET = $(build_bindir)/objconv -$(SRCDIR)/srccache/objconv.zip: +$(SRCDIR)/srccache/objconv.zip: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://www.agner.org/optimize/objconv.zip $(BUILDDIR)/objconv/config.status: $(SRCDIR)/srccache/objconv.zip -rm -r $(dir $@) @@ -1143,7 +1148,7 @@ endif clean-openblas: -$(MAKE) -C $(BUILDDIR)/$(OPENBLAS_SRC_DIR) clean -get-openblas: $(SRCDIR)/srccache/$(OPENBLAS_SRC_DIR).git +get-openblas: $(OPENBLAS_SRC_FILE) configure-openblas: $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/config.status compile-openblas: $(OPENBLAS_OBJ_SOURCE) check-openblas: compile-openblas @@ -1246,7 +1251,7 @@ ifneq ($(OS),WINNT) LAPACK_MFLAGS += BLASLIB="-Wl,-rpath,'$(build_libdir)' $(LIBBLAS)" endif -$(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz: +$(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://www.netlib.org/lapack/$(notdir $@) $(BUILDDIR)/lapack-$(LAPACK_VER)/make.inc: $(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz $(JLCHECKSUM) $< @@ -1325,10 +1330,10 @@ ARPACK_FLAGS += LDFLAGS="$(LDFLAGS) -Wl,-rpath,'$(build_libdir)'" endif # ARPACK-NG upstream keeps changing their download filenames -$(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER).tar.gz: +$(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER).tar.gz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://github.com/opencollab/arpack-ng/archive/$(ARPACK_VER).tar.gz touch -c $@ -$(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)-testA.mtx: +$(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)-testA.mtx: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://raw.githubusercontent.com/opencollab/arpack-ng/$(ARPACK_VER)/TESTS/testA.mtx touch -c $@ $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)/configure: $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER).tar.gz @@ -1354,8 +1359,9 @@ $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/config.status: $(SRCDIR)/srccache/arpack-ng- $(ARPACK_OBJ_SOURCE): $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/config.status $(MAKE) -C $(dir $<) $(ARPACK_MFLAGS) touch -c $@ -$(BUILDDIR)/arpack-ng-$(ARPACK_VER)/checked: $(ARPACK_OBJ_SOURCE) $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)-testA.mtx - cp $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)-testA.mtx $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/TESTS +$(BUILDDIR)/arpack-ng-$(ARPACK_VER)/checked: $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)-testA.mtx $(ARPACK_OBJ_SOURCE) + $(JLCHECKSUM) $< + cp $< $(dir $@)/TESTS/testA.mtx $(MAKE) -C $(dir $@) check $(ARPACK_MFLAGS) echo 1 > $@ $(ARPACK_OBJ_TARGET): $(ARPACK_OBJ_SOURCE) $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/checked | $(build_shlibdir) @@ -1416,7 +1422,7 @@ FFTW_ENABLE_single = --enable-single FFTW_ENABLE_double = FFTW_CONFIG += $(FFTW_ENABLE_$*) -$(SRCDIR)/srccache/fftw-$(FFTW_VER).tar.gz: +$(SRCDIR)/srccache/fftw-$(FFTW_VER).tar.gz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://www.fftw.org/$(notdir $@) $(SRCDIR)/srccache/fftw-$(FFTW_VER)/configure: $(SRCDIR)/srccache/fftw-$(FFTW_VER).tar.gz $(JLCHECKSUM) $< @@ -1425,7 +1431,7 @@ $(SRCDIR)/srccache/fftw-$(FFTW_VER)/configure: $(SRCDIR)/srccache/fftw-$(FFTW_VE touch -c $@ $(BUILDDIR)/fftw-$(FFTW_VER)-%/config.status: $(SRCDIR)/srccache/fftw-$(FFTW_VER)/configure mkdir -p $(dir $@) - # try to configure with avx support. if that fails, try again without it + @# try to configure with avx support. if that fails, try again without it cd $(dir $@) && \ ($< $(CONFIGURE_COMMON) $(FFTW_CONFIG) --enable-avx || \ $< $(CONFIGURE_COMMON) $(FFTW_CONFIG)) @@ -1498,8 +1504,8 @@ configure-fftw: configure-fftw-single configure-fftw-double compile-fftw: compile-fftw-single compile-fftw-double check-fftw: check-fftw-single check-fftw-double install-fftw: check-fftw-single check-fftw-double # do these serially, to prevent write conflicts - @$(MAKE) -s install-fftw-single - @$(MAKE) -s install-fftw-double + @$(MAKE) -s -f $(JULIAHOME)/deps/Makefile install-fftw-single + @$(MAKE) -s -f $(JULIAHOME)/deps/Makefile install-fftw-double get-fftw-single: $(SRCDIR)/srccache/fftw-$(FFTW_VER).tar.gz configure-fftw-single: $(BUILDDIR)/fftw-$(FFTW_VER)-single/config.status @@ -1523,16 +1529,18 @@ UTF8PROC_OBJ_LIB = $(build_libdir)/libutf8proc.a UTF8PROC_OBJ_HEADER = $(build_includedir)/utf8proc.h UTF8PROC_OBJ_TARGET = $(UTF8PROC_OBJ_LIB) $(UTF8PROC_OBJ_HEADER) UTF8PROC_CFLAGS = -O2 +UTF8PROC_MFLAGS = CC="$(CC) $(DEPS_CFLAGS)" CFLAGS="$(CFLAGS) $(UTF8PROC_CFLAGS)" PICFLAG="$(fPIC)" AR="$(AR)" $(UTF8PROC_SRC_TARGET): $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/Makefile - $(MAKE) -C $(dir $<) CC="$(CC) $(DEPS_CFLAGS)" CFLAGS="$(CFLAGS) $(UTF8PROC_CFLAGS)" PICFLAG="$(fPIC)" AR="$(AR)" libutf8proc.a + $(MAKE) -C $(dir $<) $(UTF8PROC_MFLAGS) libutf8proc.a touch -c $@ +# make check doesn't actually compile (missing prototype / implementation for getline) $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/checked: $(UTF8PROC_SRC_TARGET) ifeq ($(OS),$(BUILD_OS)) - $(MAKE) -C $(dir $@) check + $(MAKE) -C $(dir $@) $(UTF8PROC_MFLAGS) check endif echo 1 > $@ -$(UTF8PROC_OBJ_LIB): $(UTF8PROC_SRC_TARGET) $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/checked +$(UTF8PROC_OBJ_LIB): $(UTF8PROC_SRC_TARGET) cp -f $< $@ $(UTF8PROC_OBJ_HEADER): $(UTF8PROC_SRC_TARGET) cp -f $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/utf8proc.h $@ @@ -1541,7 +1549,7 @@ clean-utf8proc: -$(MAKE) -C $(BUILDDIR)/$(UTF8PROC_SRC_DIR) clean -rm -rf $(build_libdir)/libutf8proc.a $(build_includedir)/utf8proc.h -get-utf8proc: $(SRCDIR)/srccache/$(UTF8PROC_SRC_DIR).git +get-utf8proc: $(UTF8PROC_SRC_FILE) configure-utf8proc: $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/Makefile compile-utf8proc: $(UTF8PROC_SRC_TARGET) check-utf8proc: $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/checked @@ -1579,7 +1587,7 @@ SUITESPARSE_MFLAGS = CC="$(CC)" CXX="$(CXX)" F77="$(FC)" AR="$(AR)" RANLIB="$(RA INSTALL_LIB="$(build_libdir)" INSTALL_INCLUDE="$(build_includedir)" LIB="$(SUITE_SPARSE_LIB)" \ UMFPACK_CONFIG="$(UMFPACK_CONFIG)" CHOLMOD_CONFIG="$(CHOLMOD_CONFIG)" SPQR_CONFIG="$(SPQR_CONFIG)" -$(SRCDIR)/srccache/SuiteSparse-$(SUITESPARSE_VER).tar.gz: +$(SRCDIR)/srccache/SuiteSparse-$(SUITESPARSE_VER).tar.gz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://faculty.cse.tamu.edu/davis/SuiteSparse/$(notdir $@) $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/Makefile: $(SRCDIR)/srccache/SuiteSparse-$(SUITESPARSE_VER).tar.gz $(JLCHECKSUM) $< @@ -1670,7 +1678,7 @@ LIBUNWIND_TARGET_SOURCE = $(BUILDDIR)/libunwind-$(UNWIND_VER)/src/.libs/libunwin LIBUNWIND_CFLAGS = -U_FORTIFY_SOURCE $(fPIC) LIBUNWIND_CPPFLAGS = -$(SRCDIR)/srccache/libunwind-$(UNWIND_VER).tar.gz: +$(SRCDIR)/srccache/libunwind-$(UNWIND_VER).tar.gz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://download.savannah.gnu.org/releases/libunwind/$(notdir $@) $(SRCDIR)/srccache/libunwind-$(UNWIND_VER)/configure: $(SRCDIR)/srccache/libunwind-$(UNWIND_VER).tar.gz $(JLCHECKSUM) $< @@ -1694,7 +1702,7 @@ endif $(LIBUNWIND_TARGET_OBJ): $(LIBUNWIND_TARGET_SOURCE) $(call make-install,libunwind-$(UNWIND_VER),) ifeq ($(ARCH), ppc64) - # workaround for configure script bug + @# workaround for configure script bug mv $(build_prefix)/lib64/libunwind*.a $(build_libdir) endif touch $@ @@ -1718,7 +1726,7 @@ OSXUNWIND_FLAGS = ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USE OSXUNWIND_OBJ_TARGET = $(build_shlibdir)/libosxunwind.$(SHLIB_EXT) OSXUNWIND_OBJ_SOURCE = $(BUILDDIR)/libosxunwind-$(OSXUNWIND_VER)/libosxunwind.$(SHLIB_EXT) -$(SRCDIR)/srccache/libosxunwind-$(OSXUNWIND_VER).tar.gz: +$(SRCDIR)/srccache/libosxunwind-$(OSXUNWIND_VER).tar.gz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://github.com/JuliaLang/libosxunwind/archive/v$(OSXUNWIND_VER).tar.gz $(BUILDDIR)/libosxunwind-$(OSXUNWIND_VER)/Makefile: $(SRCDIR)/srccache/libosxunwind-$(OSXUNWIND_VER).tar.gz @@ -1756,14 +1764,16 @@ ifeq ($(SANITIZE),1) GMP_CONFIGURE_OPTS += --disable-assembly endif -$(SRCDIR)/srccache/gmp-$(GMP_VER).tar.bz2: +$(SRCDIR)/srccache/gmp-$(GMP_VER).tar.bz2: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://gmplib.org/download/gmp/$(notdir $@) $(SRCDIR)/srccache/gmp-$(GMP_VER)/configure: $(SRCDIR)/srccache/gmp-$(GMP_VER).tar.bz2 $(JLCHECKSUM) $< cd $(dir $<) && $(TAR) jxf $< - cd $(dir $@) && patch -p1 < $(SRCDIR)/gmp_6.0.0_osx.patch touch -c $@ -$(BUILDDIR)/gmp-$(GMP_VER)/config.status: $(SRCDIR)/srccache/gmp-$(GMP_VER)/configure +$(SRCDIR)/srccache/gmp-$(GMP_VER)/patched: $(SRCDIR)/srccache/gmp-$(GMP_VER)/configure + cd $(dir $@) && patch -p1 < $(SRCDIR)/gmp_6.0.0_osx.patch + echo 1 > $@ +$(BUILDDIR)/gmp-$(GMP_VER)/config.status: $(SRCDIR)/srccache/gmp-$(GMP_VER)/configure $(SRCDIR)/srccache/gmp-$(GMP_VER)/patched mkdir -p $(dir $@) cd $(dir $@) && \ $< $(CONFIGURE_COMMON) F77= --enable-shared --disable-static $(GMP_CONFIGURE_OPTS) @@ -1822,7 +1832,7 @@ ifeq ($(SANITIZE),1) MPFR_OPTS += --host=none-unknown-linux endif -$(SRCDIR)/srccache/mpfr-$(MPFR_VER).tar.bz2: +$(SRCDIR)/srccache/mpfr-$(MPFR_VER).tar.bz2: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://www.mpfr.org/mpfr-$(MPFR_VER)/$(notdir $@) $(SRCDIR)/srccache/mpfr-$(MPFR_VER)/configure: $(SRCDIR)/srccache/mpfr-$(MPFR_VER).tar.bz2 $(JLCHECKSUM) $< @@ -1862,7 +1872,7 @@ install-mpfr: $(MPFR_OBJ_TARGET) PATCHELF_SOURCE = $(BUILDDIR)/patchelf-$(PATCHELF_VER)/src/patchelf PATCHELF_TARGET = $(build_bindir)/patchelf -$(SRCDIR)/srccache/patchelf-$(PATCHELF_VER).tar.gz: +$(SRCDIR)/srccache/patchelf-$(PATCHELF_VER).tar.gz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://nixos.org/releases/patchelf/patchelf-$(PATCHELF_VER)/patchelf-$(PATCHELF_VER).tar.gz $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER)/configure: $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER).tar.gz $(JLCHECKSUM) $< @@ -1904,7 +1914,7 @@ install-patchelf: $(PATCHELF_TARGET) GIT_SOURCE = $(BUILDDIR)/git-$(GIT_VER)/git GIT_TARGET = $(build_libexecdir)/git -$(SRCDIR)/srccache/git-$(GIT_VER).tar.gz: +$(SRCDIR)/srccache/git-$(GIT_VER).tar.gz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://www.kernel.org/pub/software/scm/git/git-$(GIT_VER).tar.gz $(BUILDDIR)/git-$(GIT_VER)/configure: $(SRCDIR)/srccache/git-$(GIT_VER).tar.gz $(JLCHECKSUM) $< @@ -1940,8 +1950,8 @@ install-git: $(GIT_TARGET) VIRTUALENV_SOURCE = $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER)/virtualenv.py VIRTUALENV_TARGET = $(BUILDDIR)/julia-env -$(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER).tar.gz: - $(JLDOWNLOAD) $@ https://pypi.python.org/packages/source/v/virtualenv/$(dir $@) +$(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER).tar.gz: | $(SRCDIR)/srccache + $(JLDOWNLOAD) $@ https://pypi.python.org/packages/source/v/virtualenv/$(notdir $@) $(VIRTUALENV_SOURCE): $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER).tar.gz $(JLCHECKSUM) $< cd $(dir $<) && $(TAR) zxf $< @@ -1994,18 +2004,18 @@ $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/C $(CMAKE) $(dir $<) $(LIBGIT2_OPTS) touch -c $@ $(LIBGIT2_OBJ_SOURCE): $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile - $(MAKE) -C $(dir $<) + $(CMAKE) --build $(dir $<) touch -c $@ $(LIBGIT2_SRC_DIR)/checked: $(LIBGIT2_OBJ_SOURCE) ifeq ($(OS),$(BUILD_OS)) - $(MAKE) -C $(dir $@) test + $(CMAKE) --build $(dir $@) --target test endif echo 1 > $@ $(LIBGIT2_OBJ_TARGET): $(LIBGIT2_OBJ_SOURCE) | $(build_shlibdir) cp $< $@ - #$$(call make-install,$(LIBGIT2_SRC_DIR),) # currently don't need the full install + @#$$(call make-install,$(LIBGIT2_SRC_DIR),) # currently don't need the full install ifeq ($(OS),Linux) - # If we're on linux, copy over libssl and libcrypto for libgit2 + @# If we're on linux, copy over libssl and libcrypto for libgit2 -LIBGIT_LIBS=$$(ldd "$@" | tail -n +2 | awk '{print $$(NF-1)}'); \ for LIB in libssl libcrypto; do \ LIB_PATH=$$(echo "$$LIBGIT_LIBS" | grep "$$LIB"); \ @@ -2019,7 +2029,7 @@ clean-libgit2: -rm -rf $(BUILDDIR)/$(LIBGIT2_SRC_DIR) -rm -f $(LIBGIT2_OBJ_TARGET) -get-libgit2: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt +get-libgit2: $(LIBGIT2_SRC_FILE) configure-libgit2: $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile compile-libgit2: $(LIBGIT2_OBJ_SOURCE) check-libgit2: $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/checked diff --git a/deps/jlchecksum b/deps/jlchecksum index 3e64f693573c2..7def72031f865 100755 --- a/deps/jlchecksum +++ b/deps/jlchecksum @@ -12,6 +12,7 @@ fi DIR="$( cd "$( dirname "$0" )" && pwd )" # Get the basename of the file we're trying to checksum +ARG1=$1 BASENAME=$(basename $1) # Print out a hash, and wrap around if we're longer than 64 characters @@ -66,15 +67,15 @@ MD5_PROG="" find_checksum_progs() { if [ ! -z $(which sha512sum) ]; then - SHA512_PROG="sha512sum $DIR/srccache/$BASENAME | awk '{ print \$1; }'" + SHA512_PROG="sha512sum $ARG1 | awk '{ print \$1; }'" elif [ ! -z $(which shasum) ]; then - SHA512_PROG="shasum -a 512 $DIR/srccache/$BASENAME | awk '{ print \$1; }'" + SHA512_PROG="shasum -a 512 $ARG1 | awk '{ print \$1; }'" fi if [ ! -z $(which md5sum) ]; then - MD5_PROG="md5sum $DIR/srccache/$BASENAME | awk '{ print \$1; }'" + MD5_PROG="md5sum $ARG1 | awk '{ print \$1; }'" elif [ ! -z $(which md5) ]; then - MD5_PROG="md5 -q $DIR/srccache/$BASENAME" + MD5_PROG="md5 -q $ARG1" fi } diff --git a/doc/Makefile b/doc/Makefile index 08ba8dd25ff9a..e7e83b23f1bd4 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -3,29 +3,31 @@ default: html # You can set these variables from the command line. -SPHINXOPTS = -PAPER = -JULIAHOME = $(abspath ..) -JULIA_EXECUTABLE = $(JULIAHOME)/usr/bin/julia +SPHINXOPTS := +PAPER := +SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +BUILDDIR := . +JULIAHOME := $(abspath $(SRCDIR)/..) +JULIA_EXECUTABLE := $(JULIAHOME)/usr/bin/julia # Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d _build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +PAPEROPT_a4 := -D latex_paper_size=a4 +PAPEROPT_letter := -D latex_paper_size=letter +ALLSPHINXOPTS := -d $(BUILDDIR)/_build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(SRCDIR) # the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +I18NSPHINXOPTS := $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -JULIA_ENV = $(JULIAHOME)/deps/julia-env -ACTIVATE = $(JULIA_ENV)/bin/activate -SPHINX_BUILD = $(JULIA_ENV)/bin/sphinx-build +JULIA_ENV := $(BUILDDIR)/../deps/julia-env +ACTIVATE := $(JULIA_ENV)/bin/activate +SPHINX_BUILD := $(JULIA_ENV)/bin/sphinx-build $(ACTIVATE): - $(MAKE) -C $(JULIAHOME)/deps install-virtualenv + $(MAKE) -C $(BUILDDIR)/../deps install-virtualenv touch -c $@ -$(SPHINX_BUILD): $(ACTIVATE) requirements.txt +$(SPHINX_BUILD): $(SRCDIR)/requirements.txt $(ACTIVATE) . $(ACTIVATE) && pip install sphinx==1.3.1 \ - && pip install -r requirements.txt + && pip install -r $< touch -c $@ SPHINXBUILD = . $(ACTIVATE) && sphinx-build @@ -61,38 +63,38 @@ clean: cleanall: clean html: $(SPHINX_BUILD) - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) _build/html + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/_build/html @echo @echo "Build finished. The HTML pages are in _build/html." dirhtml: $(SPHINX_BUILD) - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) _build/dirhtml + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/_build/dirhtml @echo @echo "Build finished. The HTML pages are in _build/dirhtml." singlehtml: $(SPHINX_BUILD) - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) _build/singlehtml + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/_build/singlehtml @echo @echo "Build finished. The HTML page is in _build/singlehtml." pickle: $(SPHINX_BUILD) - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) _build/pickle + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/_build/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINX_BUILD) - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) _build/json + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/_build/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINX_BUILD) - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) _build/htmlhelp + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/_build/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in _build/htmlhelp." qthelp: $(SPHINX_BUILD) - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) _build/qthelp + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/_build/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in _build/qthelp, like this:" @@ -101,7 +103,7 @@ qthelp: $(SPHINX_BUILD) @echo "# assistant -collectionFile _build/qthelp/JuliaLanguage.qhc" devhelp: $(SPHINX_BUILD) - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) _build/devhelp + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/_build/devhelp @echo @echo "Build finished." @echo "To view the help file:" @@ -110,69 +112,72 @@ devhelp: $(SPHINX_BUILD) @echo "# devhelp" epub: $(SPHINX_BUILD) - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) _build/epub + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/_build/epub @echo @echo "Build finished. The epub file is in _build/epub." latex: $(SPHINX_BUILD) - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) _build/latex + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/_build/latex @echo @echo "Build finished; the LaTeX files are in _build/latex." @echo "Run 'make' in that directory to run these through (pdf)latex" \ "(use 'make latexpdf' here to do that automatically)." latexpdf: $(SPHINX_BUILD) - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) _build/latex + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/_build/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C _build/latex all-pdf @echo "pdflatex finished; the PDF files are in _build/latex." text: $(SPHINX_BUILD) - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) _build/text + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/_build/text @echo @echo "Build finished. The text files are in _build/text." man: $(SPHINX_BUILD) - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) _build/man + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/_build/man @echo @echo "Build finished. The manual pages are in _build/man." texinfo: $(SPHINX_BUILD) - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) _build/texinfo + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/_build/texinfo @echo @echo "Build finished. The Texinfo files are in _build/texinfo." @echo "Run 'make' in that directory to run these through makeinfo" \ "(use 'make info' here to do that automatically)." info: $(SPHINX_BUILD) - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) _build/texinfo + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/_build/texinfo @echo "Running Texinfo files through makeinfo..." make -C _build/texinfo info @echo "makeinfo finished; the Info files are in _build/texinfo." gettext: $(SPHINX_BUILD) - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) _build/locale + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/_build/locale @echo @echo "Build finished. The message catalogs are in _build/locale." changes: $(SPHINX_BUILD) - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) _build/changes + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/_build/changes @echo @echo "The overview file is in _build/changes." linkcheck: $(SPHINX_BUILD) - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) _build/linkcheck + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/_build/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in _build/linkcheck/output.txt." doctest: $(SPHINX_BUILD) - PATH="$(PATH):$(build_bindir)" $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) _build/doctest + PATH="$(build_bindir):$(PATH)" $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/_build/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in _build/doctest/output.txt." -manual/unicode-input-table.rst: $(JULIAHOME)/base/latex_symbols.jl - $(JULIA_EXECUTABLE) tabcomplete.jl > manual/unicode-input-table.rst +$(BUILDDIR)/manual: + mkdir -p $@ -unicode: manual/unicode-input-table.rst +$(BUILDDIR)/manual/unicode-input-table.rst: $(SRCDIR)/tabcomplete.jl $(JULIAHOME)/base/latex_symbols.jl | $(BUILDDIR)/manual + $(JULIA_EXECUTABLE) $< > $@ + +unicode: $(BUILDDIR)/manual/unicode-input-table.rst diff --git a/src/Makefile b/src/Makefile index a17d5d325f685..da8f8758b097a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,6 +1,6 @@ SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) JULIAHOME := $(abspath $(SRCDIR)/..) -BUILDDIR ?= . +BUILDDIR := . include $(JULIAHOME)/deps/Versions.make include $(JULIAHOME)/Make.inc diff --git a/src/dump.c b/src/dump.c index ce2426576c8b0..2929477a555aa 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1756,6 +1756,14 @@ void jl_save_system_image_to_stream(ios_t *f) // orphan old Base module if present jl_base_module = (jl_module_t*)jl_get_global(jl_main_module, jl_symbol("Base")); + // empty!(Core.ARGS) + if (jl_core_module != NULL) { + jl_array_t *args = (jl_array_t*)jl_get_global(jl_core_module, jl_symbol("ARGS")); + if (args != NULL) { + jl_array_del_end(args, jl_array_len(args)); + } + } + jl_idtable_type = jl_base_module ? jl_get_global(jl_base_module, jl_symbol("ObjectIdDict")) : NULL; jl_serialize_value(f, jl_main_module); diff --git a/src/flisp/Makefile b/src/flisp/Makefile index bec862493544c..1dfc7a52ed318 100644 --- a/src/flisp/Makefile +++ b/src/flisp/Makefile @@ -1,12 +1,11 @@ -JULIAHOME = $(abspath ../..) +JULIAHOME := $(abspath ../..) +BUILDDIR := . include $(JULIAHOME)/Make.inc override CFLAGS += $(JCFLAGS) override CXXFLAGS += $(JCXXFLAGS) override CPPFLAGS += $(JCPPFLAGS) -BUILDDIR ?= . - NAME = flisp EXENAME = $(NAME) LIBTARGET = lib$(NAME) diff --git a/src/support/Makefile b/src/support/Makefile index 382377e62aa5e..c03767c2dbafe 100644 --- a/src/support/Makefile +++ b/src/support/Makefile @@ -1,12 +1,11 @@ -JULIAHOME = $(abspath ../..) +JULIAHOME := $(abspath ../..) +BUILDDIR := . include $(JULIAHOME)/Make.inc override CFLAGS += $(JCFLAGS) override CXXFLAGS += $(JCXXFLAGS) override CPPFLAGS += $(JCPPFLAGS) -BUILDDIR ?= . - SRCS = hashing timefuncs ptrhash operators \ utf8 ios htable bitvector \ int2str libsupportinit arraylist strtod diff --git a/test/Makefile b/test/Makefile index c7848e0644c45..1da9f0242edfb 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,25 +1,28 @@ -JULIAHOME = $(abspath ..) -include ../Make.inc +SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +JULIAHOME := $(abspath $(SRCDIR)/..) +BUILDDIR := . +include $(JULIAHOME)/Make.inc +# TODO: this Makefile ignores BUILDDIR, except for computing JULIA_EXECUTABLE TESTS = all linalg $(filter-out TestHelpers runtests testdefs,$(subst .jl,,$(wildcard *.jl linalg/*.jl))) default: all $(TESTS): - @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no ./runtests.jl $@) + @cd $(SRCDIR) && $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no ./runtests.jl $@) perf: - @$(MAKE) -C perf all + @$(MAKE) -C $(SRCDIR)/perf all clean: - @$(MAKE) -C perf $@ - -rm -f libccalltest.$(SHLIB_EXT) ccalltest + @$(MAKE) -C $(SRCDIR)/perf clean + -rm -f $(SRCDIR)/libccalltest.$(SHLIB_EXT) $(SRCDIR)/ccalltest .PHONY: $(TESTS) perf clean libccalltest -all ccall libccalltest: libccalltest.$(SHLIB_EXT) -libccalltest.$(SHLIB_EXT): ccalltest.c +all ccall libccalltest: $(SRCDIR)/libccalltest.$(SHLIB_EXT) +$(SRCDIR)/libccalltest.$(SHLIB_EXT): $(SRCDIR)/ccalltest.c @$(call PRINT_CC, $(CC) $(CFLAGS) $(DEBUGFLAGS) -O3 $< $(fPIC) -shared -o $@ $(LDFLAGS) -DCC="$(CC)") -ccalltest: ccalltest.c libccalltest.$(SHLIB_EXT) +$(SRCDIR)/ccalltest: $(SRCDIR)/ccalltest.c $(SRCDIR)/libccalltest.$(SHLIB_EXT) @$(call PRINT_CC, $(CC) $(CFLAGS) $(DEBUGFLAGS) -O3 $< -o $@ $(LDFLAGS) -DCC="$(CC)") diff --git a/test/perf/Makefile b/test/perf/Makefile index 848dfbcbe3f28..0ff723aa70f7b 100644 --- a/test/perf/Makefile +++ b/test/perf/Makefile @@ -1,34 +1,37 @@ -JULIAHOME = $(abspath ../..) -include ../../Make.inc +SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +JULIAHOME := $(abspath $(SRCDIR)/../..) +BUILDDIR := . +include $(JULIAHOME)/Make.inc +# TODO: this Makefile ignores BUILDDIR, except for computing JULIA_EXECUTABLE + all: micro kernel cat shootout blas lapack simd sort spell sparse micro kernel cat shootout blas lapack simd sort spell sparse: - @$(MAKE) $(QUIET_MAKE) -C shootout + @$(MAKE) $(QUIET_MAKE) -C $(SRCDIR)/shootout ifneq ($(OS),WINNT) - @$(call spawn,$(JULIA_EXECUTABLE)) $@/perf.jl | perl -nle '@_=split/,/; printf "%-18s %8.3f %8.3f %8.3f %8.3f\n", $$_[1], $$_[2], $$_[3], $$_[4], $$_[5]' + @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/$@/perf.jl | perl -nle '@_=split/,/; printf "%-18s %8.3f %8.3f %8.3f %8.3f\n", $$_[1], $$_[2], $$_[3], $$_[4], $$_[5]' else - @$(call spawn,$(JULIA_EXECUTABLE)) $@/perf.jl 2> /dev/null + @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/$@/perf.jl 2> /dev/null endif codespeed: - @$(MAKE) $(QUIET_MAKE) -C shootout - @$(call spawn,$(JULIA_EXECUTABLE)) micro/perf.jl codespeed - @$(call spawn,$(JULIA_EXECUTABLE)) kernel/perf.jl codespeed - @$(call spawn,$(JULIA_EXECUTABLE)) shootout/perf.jl codespeed -# @$(call spawn,$(JULIA_EXECUTABLE)) cat/perf.jl codespeed -# @$(call spawn,$(JULIA_EXECUTABLE)) blas/perf.jl codespeed -# @$(call spawn,$(JULIA_EXECUTABLE)) lapack/perf.jl codespeed -# @$(call spawn,$(JULIA_EXECUTABLE)) simd/perf.jl codespeed -# @$(call spawn,$(JULIA_EXECUTABLE)) sort/perf.jl codespeed - @$(call spawn,$(JULIA_EXECUTABLE)) spell/perf.jl codespeed - @$(call spawn,$(JULIA_EXECUTABLE)) sparse/perf.jl codespeed - @$(call spawn,$(JULIA_EXECUTABLE)) report.jl + @$(MAKE) $(QUIET_MAKE) -C $(SRCDIR)/shootout + @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/micro/perf.jl codespeed + @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/kernel/perf.jl codespeed + @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/shootout/perf.jl codespeed +# @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/cat/perf.jl codespeed +# @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/blas/perf.jl codespeed +# @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/lapack/perf.jl codespeed +# @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/simd/perf.jl codespeed +# @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/sort/perf.jl codespeed + @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/spell/perf.jl codespeed + @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/sparse/perf.jl codespeed + @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/report.jl clean: - rm -f *~ - $(MAKE) -C micro $@ - $(MAKE) -C shootout $@ + $(MAKE) -C $(SRCDIR)/micro clean + $(MAKE) -C $(SRCDIR)/shootout clean .PHONY: micro kernel cat shootout blas lapack simd sort spell sparse clean diff --git a/ui/repl.c b/ui/repl.c index a4c11ea2dc41b..5ef39003f7f6c 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -456,11 +456,11 @@ static void print_profile(void) static int true_main(int argc, char *argv[]) { - if (jl_base_module != NULL) { - jl_array_t *args = (jl_array_t*)jl_get_global(jl_base_module, jl_symbol("ARGS")); + if (jl_core_module != NULL) { + jl_array_t *args = (jl_array_t*)jl_get_global(jl_core_module, jl_symbol("ARGS")); if (args == NULL) { args = jl_alloc_cell_1d(0); - jl_set_const(jl_base_module, jl_symbol("ARGS"), (jl_value_t*)args); + jl_set_const(jl_core_module, jl_symbol("ARGS"), (jl_value_t*)args); } assert(jl_array_len(args) == 0); jl_array_grow_end(args, argc); From 8dc87440475e4fb891b2d8178013c32d63b6f429 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 5 Aug 2015 18:37:24 -0400 Subject: [PATCH 0046/1938] fix application of llvm patches --- deps/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index 1cf751a5ceb95..540660494e197 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -660,8 +660,8 @@ endif # LLVM_VER # Apply version-specific LLVM patches LLVM_PATCH_LIST= define LLVM_PATCH -$$(LLVM_SRC_DIR)/$1.patch-applied: | $$(SRCDIR)/$1.patch - cd $$(LLVM_SRC_DIR) && patch -p1 < $$< +$$(LLVM_SRC_DIR)/$1.patch-applied: $(LLVM_SRC_DIR)/configure | $$(SRCDIR)/$1.patch + cd $$(LLVM_SRC_DIR) && patch -p1 < $$(SRCDIR)/$1.patch echo 1 > $$@ LLVM_PATCH_LIST += $$(LLVM_SRC_DIR)/$1.patch-applied endef From 0e8b9c5e133d75823b944d6af9072ddc5ed1d8b5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 18 Aug 2015 16:16:21 -0400 Subject: [PATCH 0047/1938] add missing win64-int128 patch to llvm3.3 list --- deps/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/deps/Makefile b/deps/Makefile index 540660494e197..182d5c92e588f 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -670,6 +670,7 @@ $(eval $(call LLVM_PATCH,llvm-3.3)) $(eval $(call LLVM_PATCH,instcombine-llvm-3.3)) $(eval $(call LLVM_PATCH,int128-vector.llvm-3.3)) $(eval $(call LLVM_PATCH,osx-10.10.llvm-3.3)) +$(eval $(call LLVM_PATCH,win64-int128.llvm-3.3)) else ifeq ($(LLVM_VER),3.6.0) $(eval $(call LLVM_PATCH,zerosign-llvm-3.6.0)) $(eval $(call LLVM_PATCH,win64-allocas-llvm-3.6.0)) From bd764883fe655fa49bc6f994e91078ae4b258724 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 7 Aug 2015 22:57:31 -0400 Subject: [PATCH 0048/1938] add missing touch rules to prevent gmp, mpfr, and libuv from attempting to rebuild repeatedly --- deps/Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deps/Makefile b/deps/Makefile index 182d5c92e588f..ea006baf096e4 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -809,6 +809,7 @@ endif $(UV_OBJ_TARGET): $(UV_SRC_TARGET) $(call make-install,$(LIBUV_SRC_DIR),) $(INSTALL_NAME_CMD)libuv.$(SHLIB_EXT) $(build_shlibdir)/libuv.$(SHLIB_EXT) + touch -c $@ clean-libuv: -$(MAKE) -C $(BUILDDIR)/$(LIBUV_SRC_DIR) clean @@ -1441,6 +1442,7 @@ $(BUILDDIR)/fftw-$(FFTW_VER)-%/config.status: $(SRCDIR)/srccache/fftw-$(FFTW_VER $(FFTW_SINGLE_SRC_TARGET): $(BUILDDIR)/fftw-$(FFTW_VER)-single/config.status $(MAKE) -C $(dir $<) + touch -c $@ $(BUILDDIR)/fftw-$(FFTW_VER)-single/checked: $(FFTW_SINGLE_SRC_TARGET) ifeq ($(OS),$(BUILD_OS)) $(MAKE) -C $(dir $@) check @@ -1463,6 +1465,7 @@ endif $(FFTW_DOUBLE_SRC_TARGET): $(BUILDDIR)/fftw-$(FFTW_VER)-double/config.status $(MAKE) -C $(dir $<) + touch -c $@ $(BUILDDIR)/fftw-$(FFTW_VER)-double/checked: $(FFTW_DOUBLE_SRC_TARGET) ifeq ($(OS),$(BUILD_OS)) $(MAKE) -C $(dir $@) check @@ -1781,6 +1784,7 @@ $(BUILDDIR)/gmp-$(GMP_VER)/config.status: $(SRCDIR)/srccache/gmp-$(GMP_VER)/conf touch -c $@ $(GMP_SRC_TARGET): $(BUILDDIR)/gmp-$(GMP_VER)/config.status $(MAKE) -C $(dir $<) $(LIBTOOL_CCLD) + touch -c $@ $(BUILDDIR)/gmp-$(GMP_VER)/checked: $(GMP_SRC_TARGET) ifeq ($(OS),$(BUILD_OS)) $(MAKE) -C $(dir $@) $(LIBTOOL_CCLD) check @@ -1846,6 +1850,7 @@ $(BUILDDIR)/mpfr-$(MPFR_VER)/config.status: $(SRCDIR)/srccache/mpfr-$(MPFR_VER)/ touch -c $@ $(MPFR_SRC_TARGET): $(BUILDDIR)/mpfr-$(MPFR_VER)/config.status $(MAKE) -C $(dir $<) $(LIBTOOL_CCLD) + touch -c $@ $(BUILDDIR)/mpfr-$(MPFR_VER)/checked: $(MPFR_SRC_TARGET) ifeq ($(OS),$(BUILD_OS)) $(MAKE) -C $(dir $@) $(LIBTOOL_CCLD) check $(MPFR_CHECK_MFLAGS) From 922c89dbc3f2b1167d62fb01a46ae1b1cffbe578 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 8 Aug 2015 15:07:13 -0400 Subject: [PATCH 0049/1938] add checksums for arpack-ng testA.mtx file --- deps/checksums/arpack-ng-3.2.0-testA.mtx/md5 | 1 + deps/checksums/arpack-ng-3.2.0-testA.mtx/sha512 | 1 + 2 files changed, 2 insertions(+) create mode 100644 deps/checksums/arpack-ng-3.2.0-testA.mtx/md5 create mode 100644 deps/checksums/arpack-ng-3.2.0-testA.mtx/sha512 diff --git a/deps/checksums/arpack-ng-3.2.0-testA.mtx/md5 b/deps/checksums/arpack-ng-3.2.0-testA.mtx/md5 new file mode 100644 index 0000000000000..8e47827354ee3 --- /dev/null +++ b/deps/checksums/arpack-ng-3.2.0-testA.mtx/md5 @@ -0,0 +1 @@ +2826846e98bcb009d339fb69973951d3 diff --git a/deps/checksums/arpack-ng-3.2.0-testA.mtx/sha512 b/deps/checksums/arpack-ng-3.2.0-testA.mtx/sha512 new file mode 100644 index 0000000000000..70eec3327a719 --- /dev/null +++ b/deps/checksums/arpack-ng-3.2.0-testA.mtx/sha512 @@ -0,0 +1 @@ +00af7f2353441c4197c52d105d3670fe250a312b8e67ae2794246f2ce8cd0b63585e5c5ab764921d357efd9ad685fcc0ee5b8b8ee7ab9af2bea26ccbb97c50ba From 2bd1d66d03e3fe728387035d2be8185a168b1d06 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 5 Aug 2015 12:30:41 -0400 Subject: [PATCH 0050/1938] cleanup old, unused cross-compile code that depended on gnu extensions to xargs --- Make.inc | 16 +++++----------- Makefile | 8 ++------ deps/Makefile | 4 ++++ 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/Make.inc b/Make.inc index eee1e27f79b56..e9e05e0a35ce3 100644 --- a/Make.inc +++ b/Make.inc @@ -196,17 +196,16 @@ override OPENBLAS_DYNAMIC_ARCH = 1 override CROSS_COMPILE=$(XC_HOST)- ifneq (,$(findstring mingw,$(XC_HOST))) override OS := WINNT -ifneq (,$(findstring CYGWIN,$(BUILD_OS))) -export STD_LIB_PATH := $(shell $(CROSS_COMPILE)gcc -print-search-dirs | grep programs | sed -e "s/^programs: =//" -e "s!/lib/!/bin/!g") -export STD_LIB_PATH := $(STD_LIB_PATH):$(shell $(CROSS_COMPILE)gcc -print-search-dirs | grep libraries | sed -e "s/^libraries: =//" -e "s!/lib/!/bin/!g") -else -export STD_LIB_PATH := $(shell $(CROSS_COMPILE)gcc -print-search-dirs | grep programs | sed "s/^programs: =//" | xargs -d":" winepath -w | tr '\n' ';') -export STD_LIB_PATH := $(STD_LIB_PATH);$(shell $(CROSS_COMPILE)gcc -print-search-dirs | grep libraries | sed "s/^libraries: =//" | xargs -d":" winepath -w | tr '\n' ';') +STD_LIB_PATH := $(shell $(CROSS_COMPILE)gcc -print-search-dirs | grep programs | sed -e "s/^programs: =//") +STD_LIB_PATH := $(STD_LIB_PATH):$(shell $(CROSS_COMPILE)gcc -print-search-dirs | grep libraries | sed -e "s/^libraries: =//") +ifneq (,$(findstring CYGWIN,$(BUILD_OS))) # the cygwin-mingw32 compiler lies about it search directory paths +STD_LIB_PATH := $(shell echo '$(STD_LIB_PATH)' | sed -e "s!/lib/!/bin/!g") endif else $(error "unknown XC_HOST variable set") endif endif +STD_LIB_PATH ?= $(PATH) JLDOWNLOAD = $(JULIAHOME)/deps/jldownload JLCHECKSUM = $(JULIAHOME)/deps/jlchecksum @@ -924,11 +923,6 @@ endif exec = $(shell $(call spawn,$(1))) -ifneq (,$(findstring CYGWIN,$(BUILD_OS))) -wine_pathsearch = $(firstword $(wildcard $(addsuffix /$(1),$(subst :, ,$(2))))) -else -wine_pathsearch = $(firstword $(wildcard $(addsuffix /$(1),$(shell printf %s\n '$(2)' | xargs -d";" winepath -u | tr '\n' ' ')))) -endif pathsearch = $(firstword $(wildcard $(addsuffix /$(1),$(subst :, ,$(2))))) JULIA_BUILD_MODE = release diff --git a/Makefile b/Makefile index 1121a62647e9e..40af17fd9eedf 100644 --- a/Makefile +++ b/Makefile @@ -190,7 +190,7 @@ $(build_private_libdir)/inference.ji: $(build_private_libdir)/inference0.ji $(call spawn,$(JULIA_EXECUTABLE)) -C $(JULIA_CPU_TARGET) --output-ji $(call cygpath_w,$@) -f \ -J $(call cygpath_w,$<) coreimg.jl) -RELBUILDROOT := $(shell $(JULIAHOME)/contrib/relative_path.sh "$(JULIAHOME)/base" "$(BUILDROOT)/base/") +RELBUILDROOT := $(call cygpath_w,$(shell $(JULIAHOME)/contrib/relative_path.sh "$(JULIAHOME)/base" "$(BUILDROOT)/base/")) COMMA:=, define sysimg_builder $$(build_private_libdir)/sys$1.o: $$(build_private_libdir)/inference.ji $(JULIAHOME)/VERSION $$(BASE_SRCS) @@ -266,11 +266,7 @@ ifeq ($(OS),WINNT) define std_dll julia-deps: | $$(build_bindir)/lib$(1).dll $$(build_bindir)/lib$(1).dll: | $$(build_bindir) -ifeq ($$(BUILD_OS),$$(OS)) - cp $$(call pathsearch,lib$(1).dll,$$(PATH)) $$(build_bindir) ; -else - cp $$(call wine_pathsearch,lib$(1).dll,$$(STD_LIB_PATH)) $$(build_bindir) ; -endif + cp $$(call pathsearch,lib$(1).dll,$$(STD_LIB_PATH)) $$(build_bindir) ; JL_LIBS += $(1) endef $(eval $(call std_dll,gfortran-3)) diff --git a/deps/Makefile b/deps/Makefile index ea006baf096e4..b663c2c85897d 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -1269,7 +1269,9 @@ $(BUILDDIR)/lapack-$(LAPACK_VER)/liblapack.a: $(BUILDDIR)/lapack-$(LAPACK_VER)/m $(MAKE) -C $(dir $@) lapacklib $(LAPACK_MFLAGS) touch -c $@ $(BUILDDIR)/lapack-$(LAPACK_VER)/checked: $(BUILDDIR)/lapack-$(LAPACK_VER)/liblapack.a +ifeq ($(BUILD_OS),$(OS)) $(MAKE) -C $(dir $@) lapack_testing $(LAPACK_MFLAGS) -k +endif touch $@ $(LAPACK_OBJ_SOURCE): $(BUILDDIR)/lapack-$(LAPACK_VER)/liblapack.a $(FC) -shared $(FFLAGS) $(JFFLAGS) $(dir $<)/SRC/*.o $(dir $<)/INSTALL/dlamch.o $(dir $<)/INSTALL/dsecnd_INT_ETIME.o $(dir $<)/INSTALL/ilaver.o $(dir $<)/INSTALL/slamch.o $(LIBBLAS) -o $@ @@ -1364,7 +1366,9 @@ $(ARPACK_OBJ_SOURCE): $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/config.status $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/checked: $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)-testA.mtx $(ARPACK_OBJ_SOURCE) $(JLCHECKSUM) $< cp $< $(dir $@)/TESTS/testA.mtx +ifeq ($(OS),$(BUILD_OS)) $(MAKE) -C $(dir $@) check $(ARPACK_MFLAGS) +endif echo 1 > $@ $(ARPACK_OBJ_TARGET): $(ARPACK_OBJ_SOURCE) $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/checked | $(build_shlibdir) $(call make-install,arpack-ng-$(ARPACK_VER),$(ARPACK_MFLAGS)) From 17474b7e8ca46514378bc2431cbe40b0ae86aa92 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 5 Aug 2015 15:27:39 -0400 Subject: [PATCH 0051/1938] prefer shelling out to the llvm-config-host binary, rather than trying to indirectly execute the target platform llvm-config --- Make.inc | 6 +++++- src/Makefile | 12 ++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Make.inc b/Make.inc index e9e05e0a35ce3..57bc6d3fe7b52 100644 --- a/Make.inc +++ b/Make.inc @@ -605,7 +605,11 @@ LLVM_CONFIG ?= llvm-config$(EXE) JCPPFLAGS+=-DSYSTEM_LLVM LLVM_VER = $(shell $(LLVM_CONFIG) --version) else -LLVM_CONFIG=$(build_bindir)/llvm-config$(EXE) +ifeq ($(BUILD_OS),$(OS)) +LLVM_CONFIG = $(build_bindir)/llvm-config$(EXE) +else +LLVM_CONFIG = $(build_bindir)/llvm-config-host +endif endif ifeq ($(USE_SYSTEM_PCRE), 1) diff --git a/src/Makefile b/src/Makefile index da8f8758b097a..cc4e8066d0452 100644 --- a/src/Makefile +++ b/src/Makefile @@ -22,7 +22,7 @@ HEADERS = $(addprefix $(SRCDIR)/,julia.h julia_internal.h options.h) $(BUILDDIR) FLAGS = \ -D_GNU_SOURCE -I$(BUILDDIR) -I$(SRCDIR) \ -I$(SRCDIR)/flisp -I$(SRCDIR)/support \ - -I$(call exec,$(LLVM_CONFIG) --includedir) \ + -I$(shell $(LLVM_CONFIG) --includedir) \ -I$(LIBUV_INC) -I$(build_includedir) -DLIBRARY_EXPORTS \ -I$(JULIAHOME)/deps/valgrind ifneq ($(USEMSVC), 1) @@ -31,12 +31,12 @@ endif # In LLVM < 3.4, --ldflags includes both options and libraries, so use it both before and after --libs # In LLVM >= 3.4, --ldflags has only options, and --system-libs has the libraries. -LLVMLINK = $(call exec,$(LLVM_CONFIG) --ldflags) $(call exec,$(LLVM_CONFIG) --libs) $(call exec,$(LLVM_CONFIG) --ldflags) $(call exec,$(LLVM_CONFIG) --system-libs 2> /dev/null) +LLVMLINK = $(shell $(LLVM_CONFIG) --ldflags) $(shell $(LLVM_CONFIG) --libs) $(shell $(LLVM_CONFIG) --ldflags) $(shell $(LLVM_CONFIG) --system-libs 2> /dev/null) ifeq ($(USE_LLVM_SHLIB),1) ifeq ($(LLVM_USE_CMAKE),1) -LLVMLINK = $(call exec,$(LLVM_CONFIG) --ldflags) -lLLVM +LLVMLINK = $(shell $(LLVM_CONFIG) --ldflags) -lLLVM else -LLVMLINK = $(call exec,$(LLVM_CONFIG) --ldflags) -lLLVM-$(call exec,$(LLVM_CONFIG) --version) +LLVMLINK = $(shell $(LLVM_CONFIG) --ldflags) -lLLVM-$(shell $(LLVM_CONFIG) --version) endif FLAGS += -DLLVM_SHLIB endif @@ -83,9 +83,9 @@ $(BUILDDIR)/%.o: $(SRCDIR)/%.c $(HEADERS) | $(BUILDDIR) $(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.c $(HEADERS) | $(BUILDDIR) @$(call PRINT_CC, $(CC) $(CPPFLAGS) $(CFLAGS) $(DEBUGFLAGS) -c $< -o $@) $(BUILDDIR)/%.o: $(SRCDIR)/%.cpp $(HEADERS) $(shell which $(LLVM_CONFIG)) | $(BUILDDIR) - @$(call PRINT_CC, $(CXX) $(call exec,$(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(SHIPFLAGS) -c $< -o $@) + @$(call PRINT_CC, $(CXX) $(shell $(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(SHIPFLAGS) -c $< -o $@) $(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.cpp $(HEADERS) $(shell which $(LLVM_CONFIG)) | $(BUILDDIR) - @$(call PRINT_CC, $(CXX) $(call exec,$(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(DEBUGFLAGS) -c $< -o $@) + @$(call PRINT_CC, $(CXX) $(shell $(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(DEBUGFLAGS) -c $< -o $@) $(BUILDDIR)/julia_flisp.boot.inc: $(BUILDDIR)/julia_flisp.boot $(FLISP_EXECUTABLE) @$(call PRINT_FLISP, $(call spawn,$(FLISP_EXECUTABLE)) $(SRCDIR)/bin2hex.scm < $< > $@) From 178de58e45ed8918ac34f2b4bf39559b0122cf9d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 8 Aug 2015 00:57:30 -0400 Subject: [PATCH 0052/1938] fix cygwin build of cpp files that depend on llvm-config --- Make.inc | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Make.inc b/Make.inc index 57bc6d3fe7b52..452db0a48c90d 100644 --- a/Make.inc +++ b/Make.inc @@ -226,11 +226,16 @@ ifneq (,$(findstring MSYS,$(BUILD_OS))) override BUILD_OS := WINNT endif -ifeq ($(OS), WINNT) -fPIC = ifeq ($(BUILD_OS), WINNT) -PATH := $(PATH):$(build_libdir):$(build_private_libdir):/c/Program Files/7-zip +PATH := $(PATH):$(build_libdir):$(build_private_libdir):/c/Program Files/7-zip:$(JULIAHOME)/dist-extras +HOST_EXE = .exe +else ifneq (,$(findstring CYGWIN,$(BUILD_OS))) +HOST_EXE = .exe +else +HOST_EXE = endif +ifeq ($(OS), WINNT) +fPIC = EXE = .exe else fPIC = -fPIC @@ -601,14 +606,14 @@ endif endif ifeq ($(USE_SYSTEM_LLVM), 1) -LLVM_CONFIG ?= llvm-config$(EXE) +LLVM_CONFIG ?= llvm-config$(HOST_EXE) JCPPFLAGS+=-DSYSTEM_LLVM LLVM_VER = $(shell $(LLVM_CONFIG) --version) else ifeq ($(BUILD_OS),$(OS)) -LLVM_CONFIG = $(build_bindir)/llvm-config$(EXE) +LLVM_CONFIG = $(build_bindir)/llvm-config$(HOST_EXE) else -LLVM_CONFIG = $(build_bindir)/llvm-config-host +LLVM_CONFIG = $(build_bindir)/llvm-config-host$(HOST_EXE) endif endif From b4507f23ec75b7e0efbbd51cbd7234bd029d1020 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 7 Aug 2015 22:58:51 -0400 Subject: [PATCH 0053/1938] add missing cygpath_w conversion for mk_julia_flisp_boot --- src/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index cc4e8066d0452..82e34f97b7cfc 100644 --- a/src/Makefile +++ b/src/Makefile @@ -88,12 +88,13 @@ $(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.cpp $(HEADERS) $(shell which $(LLVM_CONFIG)) @$(call PRINT_CC, $(CXX) $(shell $(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(DEBUGFLAGS) -c $< -o $@) $(BUILDDIR)/julia_flisp.boot.inc: $(BUILDDIR)/julia_flisp.boot $(FLISP_EXECUTABLE) - @$(call PRINT_FLISP, $(call spawn,$(FLISP_EXECUTABLE)) $(SRCDIR)/bin2hex.scm < $< > $@) + @$(call PRINT_FLISP, $(call spawn,$(FLISP_EXECUTABLE)) $(call cygpath_w,$(SRCDIR)/bin2hex.scm) < $< > $@) $(BUILDDIR)/julia_flisp.boot: $(addprefix $(SRCDIR)/,jlfrontend.scm \ julia-parser.scm julia-syntax.scm match.scm utils.scm mk_julia_flisp_boot.scm) \ $(FLISP_EXECUTABLE) - @$(call PRINT_FLISP, $(call spawn,$(FLISP_EXECUTABLE)) $(SRCDIR)/mk_julia_flisp_boot.scm $(dir $<) $(notdir $<) $@) + @$(call PRINT_FLISP, $(call spawn,$(FLISP_EXECUTABLE)) \ + $(call cygpath_w,$(SRCDIR)/mk_julia_flisp_boot.scm) $(call cygpath_w,$(dir $<)) $(notdir $<) $(call cygpath_w,$@)) $(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc $(SRCDIR)/flisp/*.h $(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,intrinsics.cpp cgutils.cpp ccall.cpp abi_*.cpp) From 11a82f956fd38f500a4e2b11d409601301c84547 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 7 Aug 2015 22:40:35 -0400 Subject: [PATCH 0054/1938] switch to := in Makefiles almost exclusively as per recommendation of Chapter 9 "Makefile language" of the developer information for linux kernel makefiles: https://www.kernel.org/doc/Documentation/kbuild/makefiles.txt --- Make.inc | 591 ++++++++++++++++++++++--------------------- Makefile | 8 +- base/Makefile | 12 +- deps/Makefile | 323 ++++++++++++----------- src/Makefile | 58 ++--- src/flisp/Makefile | 32 +-- src/support/Makefile | 10 +- ui/Makefile | 18 +- 8 files changed, 526 insertions(+), 526 deletions(-) diff --git a/Make.inc b/Make.inc index 452db0a48c90d..f2b62b7be76c6 100644 --- a/Make.inc +++ b/Make.inc @@ -6,40 +6,40 @@ ## and build directories # OPENBLAS build options -OPENBLAS_TARGET_ARCH= -OPENBLAS_SYMBOLSUFFIX= +OPENBLAS_TARGET_ARCH:= +OPENBLAS_SYMBOLSUFFIX:= # If OPENBLAS_TARGET_ARCH is set, we default to disabling OPENBLAS_DYNAMIC_ARCH ifneq ($(OPENBLAS_TARGET_ARCH),) -OPENBLAS_DYNAMIC_ARCH=0 +OPENBLAS_DYNAMIC_ARCH:=0 else -OPENBLAS_DYNAMIC_ARCH=1 +OPENBLAS_DYNAMIC_ARCH:=1 endif -OPENBLAS_USE_THREAD=1 +OPENBLAS_USE_THREAD:=1 # Use libraries available on the system instead of building them -USE_SYSTEM_LLVM=0 -USE_SYSTEM_LIBUNWIND=0 -USE_SYSTEM_PCRE=0 -USE_SYSTEM_LIBM=0 -USE_SYSTEM_OPENLIBM=0 -UNTRUSTED_SYSTEM_LIBM=0 -USE_SYSTEM_OPENSPECFUN=0 -USE_SYSTEM_DSFMT=0 -USE_SYSTEM_BLAS=0 -USE_SYSTEM_LAPACK=0 -USE_SYSTEM_FFTW=0 -USE_SYSTEM_GMP=0 -USE_SYSTEM_MPFR=0 -USE_SYSTEM_ARPACK=0 -USE_SYSTEM_SUITESPARSE=0 -USE_SYSTEM_LIBUV=0 -USE_SYSTEM_UTF8PROC=0 -USE_SYSTEM_LIBGIT2=0 -USE_SYSTEM_PATCHELF=0 +USE_SYSTEM_LLVM:=0 +USE_SYSTEM_LIBUNWIND:=0 +USE_SYSTEM_PCRE:=0 +USE_SYSTEM_LIBM:=0 +USE_SYSTEM_OPENLIBM:=0 +UNTRUSTED_SYSTEM_LIBM:=0 +USE_SYSTEM_OPENSPECFUN:=0 +USE_SYSTEM_DSFMT:=0 +USE_SYSTEM_BLAS:=0 +USE_SYSTEM_LAPACK:=0 +USE_SYSTEM_FFTW:=0 +USE_SYSTEM_GMP:=0 +USE_SYSTEM_MPFR:=0 +USE_SYSTEM_ARPACK:=0 +USE_SYSTEM_SUITESPARSE:=0 +USE_SYSTEM_LIBUV:=0 +USE_SYSTEM_UTF8PROC:=0 +USE_SYSTEM_LIBGIT2:=0 +USE_SYSTEM_PATCHELF:=0 # Link to the LLVM shared library -USE_LLVM_SHLIB = 0 +USE_LLVM_SHLIB := 0 ## Settings for various Intel tools # Set to 1 to use MKL @@ -56,21 +56,21 @@ USEIFC ?= 0 ifeq ($(USE_MKL), 1) $(warning "The julia make variable USE_MKL has been renamed to USE_INTEL_MKL") -USE_INTEL_MKL = 1 +USE_INTEL_MKL := 1 endif # libc++ is standard on OS X 10.9, but not for earlier releases -USE_LIBCPP = 0 +USE_LIBCPP := 0 # assume we don't have LIBSSP support in our compiler, will enable later if likely true -HAVE_SSP = 0 +HAVE_SSP := 0 # GC debugging options -WITH_GC_VERIFY = 0 -WITH_GC_DEBUG_ENV = 0 +WITH_GC_VERIFY := 0 +WITH_GC_DEBUG_ENV := 0 # Prevent picking up $ARCH from the environment variables -ARCH= +ARCH:= # pick up BUILDROOT from O= if it isn't already set (from recursive make) ifeq ($(BUILDROOT),) @@ -98,16 +98,16 @@ endif # find out if git repository is available ifeq ($(shell [ -e $(JULIAHOME)/.git ] && echo true || echo "Warning: git information unavailable; versioning information limited" >&2), true) -NO_GIT = 0 +NO_GIT := 0 else -NO_GIT = 1 +NO_GIT := 1 endif -JULIA_VERSION = $(shell cat $(JULIAHOME)/VERSION) +JULIA_VERSION := $(shell cat $(JULIAHOME)/VERSION) ifneq ($(NO_GIT), 1) -JULIA_COMMIT = $(shell git rev-parse --short=10 HEAD) +JULIA_COMMIT := $(shell git rev-parse --short=10 HEAD) else -JULIA_COMMIT = $(JULIA_VERSION) +JULIA_COMMIT := $(JULIA_VERSION) endif # Whether to use GPL libraries or not. @@ -115,64 +115,64 @@ USE_GPL_LIBS ?= 1 # Directories where said libraries get installed to prefix ?= $(abspath julia-$(JULIA_COMMIT)) -bindir = $(prefix)/bin -libdir = $(prefix)/lib -private_libdir = $(libdir)/julia -libexecdir = $(prefix)/libexec -datarootdir = $(prefix)/share -docdir = $(datarootdir)/doc/julia -mandir = $(datarootdir)/man -man1dir = $(mandir)/man1 -includedir = $(prefix)/include -sysconfdir = $(prefix)/etc +bindir := $(prefix)/bin +libdir := $(prefix)/lib +private_libdir := $(libdir)/julia +libexecdir := $(prefix)/libexec +datarootdir := $(prefix)/share +docdir := $(datarootdir)/doc/julia +mandir := $(datarootdir)/man +man1dir := $(mandir)/man1 +includedir := $(prefix)/include +sysconfdir := $(prefix)/etc # Directories where things get built into -build_prefix = $(BUILDROOT)/usr -build_staging = $(build_prefix)-staging -build_bindir = $(build_prefix)/bin -build_libdir = $(build_prefix)/lib -build_private_libdir = $(build_prefix)/lib/julia -build_libexecdir = $(build_prefix)/libexec -build_datarootdir = $(build_prefix)/share -build_docdir = $(build_datarootdir)/doc/julia -build_mandir = $(build_datarootdir)/man -build_man1dir = $(build_mandir)/man1 -build_includedir = $(build_prefix)/include -build_sysconfdir = $(build_prefix)/etc +build_prefix := $(BUILDROOT)/usr +build_staging := $(build_prefix)-staging +build_bindir := $(build_prefix)/bin +build_libdir := $(build_prefix)/lib +build_private_libdir := $(build_prefix)/lib/julia +build_libexecdir := $(build_prefix)/libexec +build_datarootdir := $(build_prefix)/share +build_docdir := $(build_datarootdir)/doc/julia +build_mandir := $(build_datarootdir)/man +build_man1dir := $(build_mandir)/man1 +build_includedir := $(build_prefix)/include +build_sysconfdir := $(build_prefix)/etc # Calculate relative paths to libdir, private_libdir, datarootdir, and sysconfdir -build_libdir_rel = $(shell $(JULIAHOME)/contrib/relative_path.sh $(build_bindir) $(build_libdir)) -libdir_rel = $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(libdir)) -build_private_libdir_rel = $(shell $(JULIAHOME)/contrib/relative_path.sh $(build_bindir) $(build_private_libdir)) -private_libdir_rel = $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(private_libdir)) -datarootdir_rel = $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(datarootdir)) -docdir_rel = $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(docdir)) -sysconfdir_rel = $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(sysconfdir)) +build_libdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(build_bindir) $(build_libdir)) +libdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(libdir)) +build_private_libdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(build_bindir) $(build_private_libdir)) +private_libdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(private_libdir)) +datarootdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(datarootdir)) +docdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(docdir)) +sysconfdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(sysconfdir)) -INSTALL_F = $(JULIAHOME)/contrib/install.sh 644 -INSTALL_M = $(JULIAHOME)/contrib/install.sh 755 +INSTALL_F := $(JULIAHOME)/contrib/install.sh 644 +INSTALL_M := $(JULIAHOME)/contrib/install.sh 755 # This used for debian packaging, to conform to library layout guidelines ifeq ($(MULTIARCH_INSTALL), 1) -MULTIARCH = $(shell gcc -print-multiarch) -private_libdir = $(prefix)/lib/$(MULTIARCH)/julia -libdir = $(prefix)/lib/$(MULTIARCH)/ +MULTIARCH := $(shell gcc -print-multiarch) +private_libdir := $(prefix)/lib/$(MULTIARCH)/julia +libdir := $(prefix)/lib/$(MULTIARCH)/ endif # LLVM Options -LLVMROOT = $(build_prefix) -LLVM_ASSERTIONS = 0 -LLVM_DEBUG = 0 +LLVMROOT := $(build_prefix) +LLVM_ASSERTIONS := 0 +LLVM_DEBUG := 0 # set to 1 to get clang and compiler-rt -BUILD_LLVM_CLANG = 0 +BUILD_LLVM_CLANG := 0 # set to 1 to get lldb (often does not work, no chance with llvm3.2 and earlier) # see http://lldb.llvm.org/build.html for dependencies -BUILD_LLDB = 0 +BUILD_LLDB := 0 # Cross-compile -#XC_HOST = i686-w64-mingw32 -#XC_HOST = x86_64-w64-mingw32 +#XC_HOST := i686-w64-mingw32 +#XC_HOST := x86_64-w64-mingw32 # Path to cmake (override in Make.user if needed) CMAKE ?= cmake @@ -188,12 +188,12 @@ XC_HOST ?= $(shell uname -m)-w64-mingw32 endif ifeq ($(XC_HOST),) -CROSS_COMPILE= -HOSTCC = $(CC) +CROSS_COMPILE:= +HOSTCC := $(CC) else -HOSTCC = gcc -override OPENBLAS_DYNAMIC_ARCH = 1 -override CROSS_COMPILE=$(XC_HOST)- +HOSTCC := gcc +override OPENBLAS_DYNAMIC_ARCH := 1 +override CROSS_COMPILE:=$(XC_HOST)- ifneq (,$(findstring mingw,$(XC_HOST))) override OS := WINNT STD_LIB_PATH := $(shell $(CROSS_COMPILE)gcc -print-search-dirs | grep programs | sed -e "s/^programs: =//") @@ -207,8 +207,8 @@ endif endif STD_LIB_PATH ?= $(PATH) -JLDOWNLOAD = $(JULIAHOME)/deps/jldownload -JLCHECKSUM = $(JULIAHOME)/deps/jlchecksum +JLDOWNLOAD := $(JULIAHOME)/deps/jldownload +JLCHECKSUM := $(JULIAHOME)/deps/jlchecksum # Figure out OS and architecture OS := $(BUILD_OS) @@ -228,78 +228,78 @@ endif ifeq ($(BUILD_OS), WINNT) PATH := $(PATH):$(build_libdir):$(build_private_libdir):/c/Program Files/7-zip:$(JULIAHOME)/dist-extras -HOST_EXE = .exe +HOST_EXE := .exe else ifneq (,$(findstring CYGWIN,$(BUILD_OS))) -HOST_EXE = .exe +HOST_EXE := .exe else -HOST_EXE = +HOST_EXE := endif ifeq ($(OS), WINNT) -fPIC = -EXE = .exe +fPIC := +EXE := .exe else -fPIC = -fPIC -EXE = +fPIC := -fPIC +EXE := endif -JULIAGC = MARKSWEEP -USE_COPY_STACKS = 1 +JULIAGC := MARKSWEEP +USE_COPY_STACKS := 1 # flag for disabling assertions ifeq ($(FORCE_ASSERTIONS), 1) -DISABLE_ASSERTIONS = +DISABLE_ASSERTIONS := else -DISABLE_ASSERTIONS = -DNDEBUG +DISABLE_ASSERTIONS := -DNDEBUG endif # Compiler specific stuff ifeq ($(USEMSVC), 1) -USEGCC = 0 -USECLANG = 0 -USEICC = 0 +USEGCC := 0 +USECLANG := 0 +USEICC := 0 else ifeq ($(USECLANG), 1) -USEGCC = 0 -USEICC = 0 +USEGCC := 0 +USEICC := 0 else ifeq ($(USEICC), 1) -USEGCC = 0 -USECLANG = 0 +USEGCC := 0 +USECLANG := 0 else # default to gcc -USEGCC = 1 -USECLANG = 0 -USEICC = 0 +USEGCC := 1 +USECLANG := 0 +USEICC := 0 endif endif endif ifeq ($(USEIFC), 1) -FC = ifort +FC := ifort else -FC = $(CROSS_COMPILE)gfortran +FC := $(CROSS_COMPILE)gfortran endif -STDLIBCPP_FLAG = +STDLIBCPP_FLAG := ifeq ($(OS), Darwin) DARWINVER := $(shell uname -r | cut -b 1-2) DARWINVER_GTE13 := $(shell expr `uname -r | cut -b 1-2` \>= 13) ifeq ($(DARWINVER), 10) # Snow Leopard specific configuration -USEGCC = 1 -USECLANG = 0 -OPENBLAS_TARGET_ARCH=NEHALEM -OPENBLAS_DYNAMIC_ARCH=0 -USE_SYSTEM_LIBUNWIND=1 +USEGCC := 1 +USECLANG := 0 +OPENBLAS_TARGET_ARCH:=NEHALEM +OPENBLAS_DYNAMIC_ARCH:=0 +USE_SYSTEM_LIBUNWIND:=1 else ifeq ($(DARWINVER_GTE13),1) -USE_LIBCPP = 1 -STDLIBCPP_FLAG = -stdlib=libstdc++ +USE_LIBCPP := 1 +STDLIBCPP_FLAG := -stdlib=libstdc++ else -USE_LIBCPP = 0 +USE_LIBCPP := 0 endif -USEGCC = 0 -USECLANG = 1 +USEGCC := 0 +USECLANG := 1 endif endif @@ -310,23 +310,23 @@ endif ifeq ($(SANITIZE),1) $(error Address Sanitizer only supported with clang. Try setting SANITIZE=0) endif -CC = $(CROSS_COMPILE)gcc -CXX = $(CROSS_COMPILE)g++ -JCFLAGS = -std=gnu99 -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 -JCPPFLAGS = -JCXXFLAGS = -pipe $(fPIC) -fno-rtti -DEBUGFLAGS = -O0 -ggdb3 -DJL_DEBUG_BUILD -fstack-protector-all -SHIPFLAGS = -O3 -ggdb3 -falign-functions +CC := $(CROSS_COMPILE)gcc +CXX := $(CROSS_COMPILE)g++ +JCFLAGS := -std=gnu99 -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 +JCPPFLAGS := +JCXXFLAGS := -pipe $(fPIC) -fno-rtti +DEBUGFLAGS := -O0 -ggdb3 -DJL_DEBUG_BUILD -fstack-protector-all +SHIPFLAGS := -O3 -ggdb3 -falign-functions endif ifeq ($(USECLANG),1) -CC = $(CROSS_COMPILE)clang -CXX = $(CROSS_COMPILE)clang++ -JCFLAGS = -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 -JCPPFLAGS = -JCXXFLAGS = -pipe $(fPIC) -fno-rtti -DEBUGFLAGS = -O0 -g -DJL_DEBUG_BUILD -fstack-protector-all -SHIPFLAGS = -O3 -g +CC := $(CROSS_COMPILE)clang +CXX := $(CROSS_COMPILE)clang++ +JCFLAGS := -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 +JCPPFLAGS := +JCXXFLAGS := -pipe $(fPIC) -fno-rtti +DEBUGFLAGS := -O0 -g -DJL_DEBUG_BUILD -fstack-protector-all +SHIPFLAGS := -O3 -g ifeq ($(OS), Darwin) ifeq ($(USE_LIBCPP), 1) CC += -stdlib=libc++ -mmacosx-version-min=10.7 @@ -347,13 +347,13 @@ endif ifeq ($(SANITIZE),1) $(error Address Sanitizer only supported with clang. Try setting SANITIZE=0) endif -CC = icc -CXX = icpc -JCFLAGS = -std=gnu11 -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 -fp-model precise -fp-model except -no-ftz -JCPPFLAGS = -JCXXFLAGS = -pipe $(fPIC) -fno-rtti -DEBUGFLAGS = -O0 -g -DJL_DEBUG_BUILD -fstack-protector-all -SHIPFLAGS = -O3 -g -falign-functions +CC := icc +CXX := icpc +JCFLAGS := -std=gnu11 -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 -fp-model precise -fp-model except -no-ftz +JCPPFLAGS := +JCXXFLAGS := -pipe $(fPIC) -fno-rtti +DEBUGFLAGS := -O0 -g -DJL_DEBUG_BUILD -fstack-protector-all +SHIPFLAGS := -O3 -g -falign-functions endif ifeq ($(USECCACHE), 1) @@ -362,10 +362,10 @@ CC_ARG := $(CC) # Expand CC and CXX here already because we want CXX_ARG := $(CXX) # the original definition and not the ccache version. CC_FULL := ccache $(CC) # Expand CC and CXX here already to avoid recursive CXX_FULL := ccache $(CXX) # referencing. -CC = $(CC_FULL) # Add an extra indirection to make CC/CXX non-simple -CXX = $(CXX_FULL) # vars (because of how -m$(BINARY) is added later on). -CC_BASE = ccache -CXX_BASE = ccache +CC := $(CC_FULL) # Add an extra indirection to make CC/CXX non-simple +CXX := $(CXX_FULL) # vars (because of how -m$(BINARY) is added later on). +CC_BASE := ccache +CXX_BASE := ccache ifeq ($(USECLANG),1) # ccache and Clang don't do well together # http://petereisentraut.blogspot.be/2011/05/ccache-and-clang.html @@ -375,22 +375,22 @@ CXX += -Qunused-arguments -fcolor-diagnostics export CCACHE_CPP2 := yes endif else #USECCACHE -CC_BASE = $(shell echo $(CC) | cut -d' ' -f1) -CXX_BASE = $(shell echo $(CXX) | cut -d' ' -f1) +CC_BASE := $(shell echo $(CC) | cut -d' ' -f1) +CXX_BASE := $(shell echo $(CXX) | cut -d' ' -f1) endif ifeq ($(LLVM_VER),svn) JCXXFLAGS += -std=c++11 endif -JFFLAGS = -O2 $(fPIC) +JFFLAGS := -O2 $(fPIC) ifneq ($(USEMSVC),1) -CPP = $(CC) -E +CPP := $(CC) -E AR := $(CROSS_COMPILE)ar AS := $(CROSS_COMPILE)as LD := $(CROSS_COMPILE)ld else #USEMSVC -CPP = $(CC) -EP +CPP := $(CC) -EP AR := lib ifeq ($(ARCH),x86_64) AS := ml64 @@ -403,18 +403,18 @@ RANLIB := $(CROSS_COMPILE)ranlib # file extensions ifeq ($(OS), WINNT) - SHLIB_EXT = dll + SHLIB_EXT := dll else ifeq ($(OS), Darwin) - SHLIB_EXT = dylib + SHLIB_EXT := dylib else - SHLIB_EXT = so + SHLIB_EXT := so endif # On Windows, we want shared library files to end up in $(build_bindir), instead of $(build_libdir) ifeq ($(OS),WINNT) -build_shlibdir = $(build_bindir) +build_shlibdir := $(build_bindir) else -build_shlibdir = $(build_libdir) +build_shlibdir := $(build_libdir) endif ifeq (exists, $(shell [ -e $(JULIAHOME)/Make.user ] && echo exists )) @@ -426,9 +426,9 @@ endif ifeq ($(SANITIZE),1) ifeq ($(SANITIZE_MEMORY),1) -SANITIZE_OPTS = -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer +SANITIZE_OPTS := -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer else -SANITIZE_OPTS = -fsanitize=address -mllvm -asan-stack=0 +SANITIZE_OPTS := -fsanitize=address -mllvm -asan-stack=0 endif JCXXFLAGS += $(SANITIZE_OPTS) JCFLAGS += $(SANITIZE_OPTS) @@ -479,20 +479,20 @@ ifeq ($(BUILD_OS),Darwin) ## Mac is a rather amazing 64-bit user-space on 32-bit kernel architecture, so to determine arch we detect ## the capabilities of the hardware, rather than the compiler or kernel, and make a substitution ifeq ($(ARCH),x86_64) -override ARCH = i686 +override ARCH := i686 else ifeq ($(ARCH),i386) -override ARCH = i686 +override ARCH := i686 endif ifeq ($(ARCH),i686) ifeq ($(shell sysctl -n hw.cpu64bit_capable),1) -override ARCH = x86_64 +override ARCH := x86_64 endif BUILD_MACHINE := $(ARCH)$(shell echo $(BUILD_MACHINE) | sed "s/[^-]*\(.*\)$$/\1/") endif endif else XC_HOST := $(ARCH)$(shell echo $(BUILD_MACHINE) | sed "s/[^-]*\(.*\)$$/\1/") -MARCH = $(ARCH) +MARCH := $(ARCH) endif ifneq ($(MARCH),) @@ -526,33 +526,33 @@ endif # We map amd64 to x86_64 for compatibility with systems that identify 64-bit systems as such ifeq ($(ARCH),amd64) -override ARCH = x86_64 +override ARCH := x86_64 endif ifeq ($(ARCH),i386) -BINARY=32 -ISX86=1 +BINARY:=32 +ISX86:=1 else ifeq ($(ARCH),i387) -BINARY=32 -ISX86=1 +BINARY:=32 +ISX86:=1 else ifeq ($(ARCH),i486) -BINARY=32 -ISX86=1 +BINARY:=32 +ISX86:=1 else ifeq ($(ARCH),i586) -BINARY=32 -ISX86=1 +BINARY:=32 +ISX86:=1 else ifeq ($(ARCH),i686) -BINARY=32 -ISX86=1 +BINARY:=32 +ISX86:=1 else ifeq ($(ARCH),x86_64) -BINARY=64 -ISX86=1 +BINARY:=64 +ISX86:=1 else ifneq (,$(findstring arm,$(ARCH))) -ISX86=0 +ISX86:=0 else ifneq (,$(findstring powerpc,$(ARCH))) -ISX86=0 +ISX86:=0 else ifneq (,$(findstring ppc,$(ARCH))) -ISX86=0 +ISX86:=0 else $(error "unknown word-size for arch: $(ARCH)") endif @@ -563,11 +563,11 @@ JCFLAGS += -fsigned-char LLVM_ASSERTIONS=1 #LLVM_FLAGS+="--with-float=hard --with-abi=aapcs-vfp" -LLVM_VER=3.6.1 +LLVM_VER:=3.6.1 -override USE_BLAS64=0 -override OPENBLAS_DYNAMIC_ARCH=0 -override OPENBLAS_TARGET_ARCH=ARMV7 +override USE_BLAS64:=0 +override OPENBLAS_DYNAMIC_ARCH:=0 +override OPENBLAS_TARGET_ARCH:=ARMV7 endif @@ -589,18 +589,18 @@ endif endif ifeq ($(OS),WINNT) -LIBUNWIND= +LIBUNWIND:= else ifeq ($(USE_SYSTEM_LIBUNWIND), 1) ifneq ($(OS),Darwin) -LIBUNWIND=-lunwind-generic -lunwind +LIBUNWIND:=-lunwind-generic -lunwind endif else ifeq ($(OS),Darwin) -LIBUNWIND=$(build_libdir)/libosxunwind.a +LIBUNWIND:=$(build_libdir)/libosxunwind.a JCPPFLAGS+=-DLIBOSXUNWIND else -LIBUNWIND=$(build_libdir)/libunwind-generic.a $(build_libdir)/libunwind.a +LIBUNWIND:=$(build_libdir)/libunwind-generic.a $(build_libdir)/libunwind.a endif endif endif @@ -608,19 +608,19 @@ endif ifeq ($(USE_SYSTEM_LLVM), 1) LLVM_CONFIG ?= llvm-config$(HOST_EXE) JCPPFLAGS+=-DSYSTEM_LLVM -LLVM_VER = $(shell $(LLVM_CONFIG) --version) +LLVM_VER := $(shell $(LLVM_CONFIG) --version) else ifeq ($(BUILD_OS),$(OS)) -LLVM_CONFIG = $(build_bindir)/llvm-config$(HOST_EXE) +LLVM_CONFIG := $(build_bindir)/llvm-config$(HOST_EXE) else -LLVM_CONFIG = $(build_bindir)/llvm-config-host$(HOST_EXE) +LLVM_CONFIG := $(build_bindir)/llvm-config-host$(HOST_EXE) endif endif ifeq ($(USE_SYSTEM_PCRE), 1) -PCRE_CONFIG = pcre2-config +PCRE_CONFIG := pcre2-config else -PCRE_CONFIG = $(build_bindir)/pcre2-config +PCRE_CONFIG := $(build_bindir)/pcre2-config endif # Use ILP64 BLAS interface when building openblas from source on 64-bit architectures @@ -638,64 +638,64 @@ endif ifeq ($(USE_SYSTEM_BLAS), 1) ifeq ($(OS), Darwin) -USE_BLAS64 = 0 -USE_SYSTEM_LAPACK = 0 -LIBBLAS = -L$(build_libdir) -lgfortblas -LIBBLASNAME = libgfortblas +USE_BLAS64 := 0 +USE_SYSTEM_LAPACK := 0 +LIBBLAS := -L$(build_libdir) -lgfortblas +LIBBLASNAME := libgfortblas else LIBBLAS ?= -lblas LIBBLASNAME ?= libblas endif else -LIBBLAS = -L$(build_shlibdir) -lopenblas -LIBBLASNAME = libopenblas +LIBBLAS := -L$(build_shlibdir) -lopenblas +LIBBLASNAME := libopenblas endif # OpenBLAS builds LAPACK as part of its build. # We only need to build LAPACK if we are not using OpenBLAS. ifeq ($(USE_SYSTEM_BLAS), 0) -LIBLAPACK = $(LIBBLAS) -LIBLAPACKNAME = $(LIBBLASNAME) +LIBLAPACK := $(LIBBLAS) +LIBLAPACKNAME := $(LIBBLASNAME) else ifeq ($(USE_SYSTEM_LAPACK), 1) LIBLAPACK ?= -llapack LIBLAPACKNAME ?= liblapack else -LIBLAPACK = -L$(build_shlibdir) -llapack $(LIBBLAS) -LIBLAPACKNAME = liblapack +LIBLAPACK := -L$(build_shlibdir) -llapack $(LIBBLAS) +LIBLAPACKNAME := liblapack endif endif ifeq ($(OS), WINNT) -LIBFFTWNAME = libfftw3 -LIBFFTWFNAME = libfftw3f +LIBFFTWNAME := libfftw3 +LIBFFTWFNAME := libfftw3f else -LIBFFTWNAME = libfftw3_threads -LIBFFTWFNAME = libfftw3f_threads +LIBFFTWNAME := libfftw3_threads +LIBFFTWFNAME := libfftw3f_threads endif ifeq ($(USE_SYSTEM_LIBM), 1) -LIBM = -lm -LIBMNAME = libm +LIBM := -lm +LIBMNAME := libm else -LIBM = -lopenlibm -LIBMNAME = libopenlibm +LIBM := -lopenlibm +LIBMNAME := libopenlibm endif ifeq ($(USE_SYSTEM_LIBUV), 1) - LIBUV = /usr/lib/libuv-julia.a - LIBUV_INC = /usr/include + LIBUV := /usr/lib/libuv-julia.a + LIBUV_INC := /usr/include else - LIBUV = $(build_libdir)/libuv.a - LIBUV_INC = $(build_includedir) + LIBUV := $(build_libdir)/libuv.a + LIBUV_INC := $(build_includedir) endif ifeq ($(USE_SYSTEM_UTF8PROC), 1) - LIBUTF8PROC = -lutf8proc - UTF8PROC_INC = /usr/include + LIBUTF8PROC := -lutf8proc + UTF8PROC_INC := /usr/include else - LIBUTF8PROC = $(build_libdir)/libutf8proc.a - UTF8PROC_INC = $(build_includedir) + LIBUTF8PROC := $(build_libdir)/libutf8proc.a + UTF8PROC_INC := $(build_includedir) endif # OS specific stuff @@ -703,40 +703,40 @@ endif # install_name_tool ifeq ($(OS), Darwin) # must end with a / and have no trailing spaces - INSTALL_NAME_ID_DIR = @rpath/ - INSTALL_NAME_CMD = install_name_tool -id $(INSTALL_NAME_ID_DIR) - INSTALL_NAME_CHANGE_CMD = install_name_tool -change + INSTALL_NAME_ID_DIR := @rpath/ + INSTALL_NAME_CMD := install_name_tool -id $(INSTALL_NAME_ID_DIR) + INSTALL_NAME_CHANGE_CMD := install_name_tool -change ifeq ($(shell test `dsymutil -v | cut -d\- -f2 | cut -d. -f1` -gt 102 && echo yes), yes) - DSYMUTIL = dsymutil + DSYMUTIL := dsymutil else - DSYMUTIL = true -ignore + DSYMUTIL := true -ignore endif else - INSTALL_NAME_ID_DIR = - INSTALL_NAME_CMD = true -ignore - INSTALL_NAME_CHANGE_CMD = true -ignore - DSYMUTIL = true -ignore + INSTALL_NAME_ID_DIR := + INSTALL_NAME_CMD := true -ignore + INSTALL_NAME_CHANGE_CMD := true -ignore + DSYMUTIL := true -ignore endif # shared library runtime paths ifeq ($(OS), WINNT) - RPATH = - RPATH_ORIGIN = + RPATH := + RPATH_ORIGIN := else ifeq ($(OS), Darwin) - RPATH = -Wl,-rpath,'@executable_path/$(build_private_libdir_rel)' -Wl,-rpath,'@executable_path/$(build_libdir_rel)' - RPATH_ORIGIN = -Wl,-rpath,'@loader_path/' + RPATH := -Wl,-rpath,'@executable_path/$(build_private_libdir_rel)' -Wl,-rpath,'@executable_path/$(build_libdir_rel)' + RPATH_ORIGIN := -Wl,-rpath,'@loader_path/' else - RPATH = -Wl,-rpath,'$$ORIGIN/$(build_private_libdir_rel)' -Wl,-rpath,'$$ORIGIN/$(build_libdir_rel)' -Wl,-z,origin - RPATH_ORIGIN = -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin + RPATH := -Wl,-rpath,'$$ORIGIN/$(build_private_libdir_rel)' -Wl,-rpath,'$$ORIGIN/$(build_libdir_rel)' -Wl,-z,origin + RPATH_ORIGIN := -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin endif # --whole-archive ifeq ($(OS), Darwin) - WHOLE_ARCHIVE = -Xlinker -all_load - NO_WHOLE_ARCHIVE = + WHOLE_ARCHIVE := -Xlinker -all_load + NO_WHOLE_ARCHIVE := else ifneq ($(USEMSVC), 1) - WHOLE_ARCHIVE = -Wl,--whole-archive - NO_WHOLE_ARCHIVE = -Wl,--no-whole-archive + WHOLE_ARCHIVE := -Wl,--whole-archive + NO_WHOLE_ARCHIVE := -Wl,--no-whole-archive endif ifeq ($(OS), Linux) @@ -748,47 +748,47 @@ OSLIBS += -Wl,--version-script=$(JULIAHOME)/src/julia.expmap endif endif endif -JLDFLAGS = -Wl,-Bdynamic +JLDFLAGS := -Wl,-Bdynamic ifeq (-Bsymbolic-functions, $(shell $(LD) --help | grep -o -e "-Bsymbolic-functions")) -JLIBLDFLAGS = -Wl,-Bsymbolic-functions +JLIBLDFLAGS := -Wl,-Bsymbolic-functions else -JLIBLDFLAGS = +JLIBLDFLAGS := endif else #Linux -JLIBLDFLAGS = +JLIBLDFLAGS := endif ifeq ($(OS), FreeBSD) -JLDFLAGS = -Wl,-Bdynamic +JLDFLAGS := -Wl,-Bdynamic OSLIBS += -lelf -lkvm -lrt -Wl,--export-dynamic -Wl,--version-script=$(JULIAHOME)/src/julia.expmap $(NO_WHOLE_ARCHIVE) $(LIBUNWIND) endif ifeq ($(OS), Darwin) -INSTALL_NAME_CMD = install_name_tool -id $(INSTALL_NAME_ID_DIR) -INSTALL_NAME_CHANGE_CMD = install_name_tool -change -SHLIB_EXT = dylib +INSTALL_NAME_CMD := install_name_tool -id $(INSTALL_NAME_ID_DIR) +INSTALL_NAME_CHANGE_CMD := install_name_tool -change +SHLIB_EXT := dylib OSLIBS += -ldl -Wl,-w -framework CoreFoundation -framework CoreServices $(LIBUNWIND) -WHOLE_ARCHIVE = -Xlinker -all_load -NO_WHOLE_ARCHIVE = -JLDFLAGS = -HAVE_SSP = 1 +WHOLE_ARCHIVE := -Xlinker -all_load +NO_WHOLE_ARCHIVE := +JLDFLAGS := +HAVE_SSP := 1 endif ifeq ($(OS), WINNT) ifneq ($(USEMSVC), 1) -HAVE_SSP = 1 +HAVE_SSP := 1 OSLIBS += -Wl,--export-all-symbols -Wl,--version-script=$(JULIAHOME)/src/julia.expmap \ $(NO_WHOLE_ARCHIVE) -lpsapi -lkernel32 -lws2_32 -liphlpapi -lwinmm -ldbghelp -JLDFLAGS = -Wl,--stack,8388608 +JLDFLAGS := -Wl,--stack,8388608 ifeq ($(ARCH),i686) JLDFLAGS += -Wl,--large-address-aware endif else #USEMSVC OSLIBS += kernel32.lib ws2_32.lib psapi.lib advapi32.lib iphlpapi.lib shell32.lib winmm.lib -JLDFLAGS = -stack:8388608 +JLDFLAGS := -stack:8388608 endif JCPPFLAGS += -D_WIN32_WINNT=0x0502 -UNTRUSTED_SYSTEM_LIBM = 1 +UNTRUSTED_SYSTEM_LIBM := 1 endif # Intel VTune Amplifier @@ -799,38 +799,38 @@ endif # Intel libraries ifeq ($(USE_INTEL_LIBM), 1) -USE_SYSTEM_LIBM = 1 -LIBM = -L$(MKLROOT)/../compiler/lib/intel64 -limf -LIBMNAME = libimf +USE_SYSTEM_LIBM := 1 +LIBM := -L$(MKLROOT)/../compiler/lib/intel64 -limf +LIBMNAME := libimf endif ifeq ($(USE_INTEL_MKL), 1) ifeq ($(USE_BLAS64), 1) export MKL_INTERFACE_LAYER := ILP64 -MKLLIB = $(MKLROOT)/lib/intel64 +MKLLIB := $(MKLROOT)/lib/intel64 else -MKLLIB = $(MKLROOT)/lib/ia32 +MKLLIB := $(MKLROOT)/lib/ia32 endif -USE_SYSTEM_BLAS=1 -USE_SYSTEM_LAPACK=1 -LIBBLASNAME = libmkl_rt -LIBLAPACKNAME = libmkl_rt -MKL_LDFLAGS = -L$(MKLLIB) -lmkl_rt +USE_SYSTEM_BLAS:=1 +USE_SYSTEM_LAPACK:=1 +LIBBLASNAME := libmkl_rt +LIBLAPACKNAME := libmkl_rt +MKL_LDFLAGS := -L$(MKLLIB) -lmkl_rt ifneq ($(strip $(MKLLIB)),) ifeq ($(OS), Linux) - RPATH_MKL = -Wl,-rpath,$(MKLLIB) + RPATH_MKL := -Wl,-rpath,$(MKLLIB) RPATH += $(RPATH_MKL) MKL_LDFLAGS += $(RPATH_MKL) endif endif -LIBBLAS = $(MKL_LDFLAGS) -LIBLAPACK = $(MKL_LDFLAGS) +LIBBLAS := $(MKL_LDFLAGS) +LIBLAPACK := $(MKL_LDFLAGS) endif ifeq ($(USE_INTEL_MKL_FFT), 1) -USE_SYSTEM_FFTW = 1 -LIBFFTWNAME = libmkl_rt -LIBFFTWFNAME = libmkl_rt +USE_SYSTEM_FFTW := 1 +LIBFFTWNAME := libmkl_rt +LIBFFTWFNAME := libmkl_rt endif ifeq ($(HAVE_SSP),1) @@ -845,24 +845,24 @@ endif # ATLAS must have been previously built with "make -C deps compile-atlas" (without -jN), # or installed to usr/lib/libatlas from some another source (built as # a shared library, for your platform, single threaded) -USE_ATLAS = 0 -ATLAS_LIBDIR = $(build_libdir) -#or ATLAS_LIBDIR = /path/to/system/atlas/lib +USE_ATLAS := 0 +ATLAS_LIBDIR := $(build_libdir) +#or ATLAS_LIBDIR := /path/to/system/atlas/lib ifeq ($(USE_ATLAS), 1) -USE_BLAS64 = 0 -USE_SYSTEM_BLAS = 1 -USE_SYSTEM_LAPACK = 1 -LIBBLAS = -L$(ATLAS_LIBDIR) -lsatlas -LIBLAPACK = $(LIBBLAS) -LIBBLASNAME = libsatlas -LIBLAPACKNAME = $(LIBBLASNAME) +USE_BLAS64 := 0 +USE_SYSTEM_BLAS := 1 +USE_SYSTEM_LAPACK := 1 +LIBBLAS := -L$(ATLAS_LIBDIR) -lsatlas +LIBLAPACK := $(LIBBLAS) +LIBBLASNAME := libsatlas +LIBLAPACKNAME := $(LIBBLASNAME) endif # Renaming OpenBLAS symbols, see #4923 and #8734 ifeq ($(USE_SYSTEM_BLAS), 0) ifeq ($(USE_BLAS64), 1) -OPENBLAS_SYMBOLSUFFIX = 64_ +OPENBLAS_SYMBOLSUFFIX := 64_ endif endif @@ -871,7 +871,7 @@ ifeq ($(BUILD_CUSTOM_LIBCXX),1) LDFLAGS += -L$(build_libdir) -lc++abi CXXLDFLAGS += -L$(build_libdir) -lc++abi -stdlib=libc++ -lc++ CPPFLAGS += -I$(build_includedir)/c++/v1 -CUSTOM_LD_LIBRARY_PATH = LD_LIBRARY_PATH="$(build_libdir)" +CUSTOM_LD_LIBRARY_PATH := LD_LIBRARY_PATH="$(build_libdir)" ifeq ($(USEICC),1) CXXFLAGS += -cxxlib-nostd -static-intel CLDFLAGS += -static-intel @@ -914,6 +914,7 @@ else endif endef +# many of the following targets must be = not := because the expansion of the makefile functions (and $1) shouldn't happen until later ifeq ($(BUILD_OS), WINNT) # MSYS spawn = $(1) cygpath_w = $(1) @@ -934,40 +935,40 @@ exec = $(shell $(call spawn,$(1))) pathsearch = $(firstword $(wildcard $(addsuffix /$(1),$(subst :, ,$(2))))) -JULIA_BUILD_MODE = release -JULIA_LIBSUFFIX= +JULIA_BUILD_MODE := release +JULIA_LIBSUFFIX:= ifeq (,$(findstring release,$(MAKECMDGOALS))) ifneq (,$(findstring debug,$(MAKECMDGOALS))) -JULIA_BUILD_MODE = debug -JULIA_LIBSUFFIX=-debug +JULIA_BUILD_MODE := debug +JULIA_LIBSUFFIX:=-debug endif endif -JULIA_EXECUTABLE_debug = $(build_bindir)/julia-debug$(EXE) -JULIA_EXECUTABLE_release = $(build_bindir)/julia$(EXE) -JULIA_EXECUTABLE = $(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE)) +JULIA_EXECUTABLE_debug := $(build_bindir)/julia-debug$(EXE) +JULIA_EXECUTABLE_release := $(build_bindir)/julia$(EXE) +JULIA_EXECUTABLE := $(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE)) # Colors for make ifndef VERBOSE -VERBOSE = 0 +VERBOSE := 0 endif -WARNCOLOR="\033[33;1m" +WARNCOLOR:="\033[33;1m" ifeq ($(VERBOSE), 0) QUIET_MAKE = -s -CCCOLOR="\033[34m" -LINKCOLOR="\033[34;1m" -PERLCOLOR="\033[35m" -FLISPCOLOR="\033[32m" -JULIACOLOR="\033[32;1m" +CCCOLOR:="\033[34m" +LINKCOLOR:="\033[34;1m" +PERLCOLOR:="\033[35m" +FLISPCOLOR:="\033[32m" +JULIACOLOR:="\033[32;1m" -SRCCOLOR="\033[33m" -BINCOLOR="\033[37;1m" -JULCOLOR="\033[34;1m" -ENDCOLOR="\033[0m" +SRCCOLOR:="\033[33m" +BINCOLOR:="\033[37;1m" +JULCOLOR:="\033[34;1m" +ENDCOLOR:="\033[0m" GOAL=$(subst ','\'',$(subst $(abspath $(JULIAHOME))/,,$(abspath $@))) diff --git a/Makefile b/Makefile index 40af17fd9eedf..3be22db4b1244 100644 --- a/Makefile +++ b/Makefile @@ -8,11 +8,11 @@ include $(JULIAHOME)/Make.inc # prefix/share/julia/site/VERSDIR (not prefix/share/julia/VERSDIR/site ... # so that prefix/share/julia/VERSDIR can be overwritten without touching # third-party code). -VERSDIR = v`cut -d. -f1-2 < $(JULIAHOME)/VERSION` +VERSDIR := v`cut -d. -f1-2 < $(JULIAHOME)/VERSION` #file name of make binary-dist result ifeq ($(JULIA_BINARYDIST_TARNAME),) - JULIA_BINARYDIST_TARNAME = julia-$(JULIA_COMMIT)-$(OS)-$(ARCH) + JULIA_BINARYDIST_TARNAME := julia-$(JULIA_COMMIT)-$(OS)-$(ARCH) endif default: $(JULIA_BUILD_MODE) # contains either "debug" or "release" @@ -208,10 +208,10 @@ $(build_bindir)/stringreplace: $(JULIAHOME)/contrib/stringreplace.c | $(build_bi # public libraries, that are installed in $(prefix)/lib -JL_LIBS = julia julia-debug +JL_LIBS := julia julia-debug # private libraries, that are installed in $(prefix)/lib/julia -JL_PRIVATE_LIBS = suitesparse_wrapper Rmath +JL_PRIVATE_LIBS := suitesparse_wrapper Rmath ifeq ($(USE_SYSTEM_FFTW),0) JL_PRIVATE_LIBS += fftw3 fftw3f fftw3_threads fftw3f_threads endif diff --git a/base/Makefile b/base/Makefile index 1c673d7db2004..a33a55b739847 100644 --- a/base/Makefile +++ b/base/Makefile @@ -4,21 +4,21 @@ JULIAHOME := $(abspath $(SRCDIR)/..) include $(JULIAHOME)/deps/Versions.make include $(JULIAHOME)/Make.inc -TAGGED_RELEASE_BANNER = "" +TAGGED_RELEASE_BANNER := "" ifneq ($(USEMSVC), 1) -CPP_STDOUT = $(CPP) -P +CPP_STDOUT := $(CPP) -P else -CPP_STDOUT = $(CPP) -E +CPP_STDOUT := $(CPP) -E endif all: $(addprefix $(BUILDDIR)/,pcre_h.jl errno_h.jl build_h.jl.phony fenv_constants.jl file_constants.jl uv_constants.jl version_git.jl.phony) -PCRE_CONST = 0x[0-9a-fA-F]+|[0-9]+ +PCRE_CONST := 0x[0-9a-fA-F]+|[0-9]+ ifeq ($(USE_SYSTEM_PCRE), 1) - PCRE_INCL_PATH = $(shell $(PCRE_CONFIG) --prefix)/include/pcre2.h + PCRE_INCL_PATH := $(shell $(PCRE_CONFIG) --prefix)/include/pcre2.h else - PCRE_INCL_PATH = $(build_includedir)/pcre2.h + PCRE_INCL_PATH := $(build_includedir)/pcre2.h endif $(BUILDDIR)/pcre_h.jl: $(PCRE_INCL_PATH) diff --git a/deps/Makefile b/deps/Makefile index b663c2c85897d..0d8c66a846087 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -27,7 +27,7 @@ include $(JULIAHOME)/Make.inc ## Some shared configuration options ## -CONFIGURE_COMMON = --prefix=$(abspath $(build_prefix)) --build=$(BUILD_MACHINE) --libdir=$(abspath $(build_libdir)) $(CUSTOM_LD_LIBRARY_PATH) +CONFIGURE_COMMON := --prefix=$(abspath $(build_prefix)) --build=$(BUILD_MACHINE) --libdir=$(abspath $(build_libdir)) $(CUSTOM_LD_LIBRARY_PATH) ifneq ($(XC_HOST),) CONFIGURE_COMMON += --host=$(XC_HOST) endif @@ -38,10 +38,10 @@ endif endif CONFIGURE_COMMON += F77="$(FC)" CC="$(CC) $(DEPS_CFLAGS)" CXX="$(CXX) $(DEPS_CXXFLAGS)" -CMAKE_CC_ARG = $(CC_ARG) $(DEPS_CFLAGS) -CMAKE_CXX_ARG = $(CXX_ARG) $(DEPS_CXXFLAGS) +CMAKE_CC_ARG := $(CC_ARG) $(DEPS_CFLAGS) +CMAKE_CXX_ARG := $(CXX_ARG) $(DEPS_CXXFLAGS) -CMAKE_COMMON = -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_BUILD_TYPE=Release +CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_BUILD_TYPE=Release ifneq ($(VERBOSE), 0) CMAKE_COMMON += -DCMAKE_VERBOSE_MAKEFILE=ON endif @@ -57,9 +57,9 @@ endif # For now this is LLVM specific, but I expect it won't be in the future ifneq ($(LLVM_USE_CMAKE),) ifeq ($(CMAKE_GENERATOR),Ninja) -CMAKE_GENERATOR_COMMAND = -G Ninja +CMAKE_GENERATOR_COMMAND := -G Ninja else ifeq ($(CMAKE_GENERATOR),make) -CMAKE_GENERATOR_COMMAND = -G "Unix Makefiles" +CMAKE_GENERATOR_COMMAND := -G "Unix Makefiles" else $(error Unknown CMake generator '$(CMAKE_GENERATOR)'. Options are 'Ninja' and 'make') endif @@ -67,7 +67,7 @@ endif # If the top-level Makefile is called with environment variables, # they will override the values passed above to ./configure -MAKE_COMMON = DESTDIR="" prefix=$(build_prefix) bindir=$(build_bindir) libdir=$(build_libdir) libexecdir=$(build_libexecdir) datarootdir=$(build_datarootdir) includedir=$(build_includedir) sysconfdir=$(build_sysconfdir) O= +MAKE_COMMON := DESTDIR="" prefix=$(build_prefix) bindir=$(build_bindir) libdir=$(build_libdir) libexecdir=$(build_libexecdir) datarootdir=$(build_datarootdir) includedir=$(build_includedir) sysconfdir=$(build_sysconfdir) O= ## Overall configuration of which rules exist and should be run by default ## @@ -76,13 +76,13 @@ MAKE_COMMON = DESTDIR="" prefix=$(build_prefix) bindir=$(build_bindir) libdir=$( unexport CONFIG_SITE ifeq ($(USE_GPL_LIBS), 1) -STAGE1_DEPS = -STAGE2_DEPS = Rmath-julia -STAGE3_DEPS = suitesparse-wrapper +STAGE1_DEPS := +STAGE2_DEPS := Rmath-julia +STAGE3_DEPS := suitesparse-wrapper else -STAGE1_DEPS = -STAGE2_DEPS = -STAGE3_DEPS = +STAGE1_DEPS := +STAGE2_DEPS := +STAGE3_DEPS := endif ifeq ($(USE_SYSTEM_LIBUV), 0) @@ -102,9 +102,9 @@ endif ifeq ($(OS), Linux) ifeq ($(USE_SYSTEM_PATCHELF), 0) STAGE1_DEPS += patchelf -PATCHELF=$(build_bindir)/patchelf +PATCHELF:=$(build_bindir)/patchelf else -PATCHELF=patchelf +PATCHELF:=patchelf endif endif @@ -186,12 +186,12 @@ endif #Platform specific flags ifeq ($(OS), WINNT) -LIBTOOL_CCLD = CCLD="$(CC) -no-undefined -avoid-version" +LIBTOOL_CCLD := CCLD="$(CC) -no-undefined -avoid-version" endif ## Common build target prefixes -DEP_LIBS = $(STAGE1_DEPS) $(STAGE2_DEPS) $(STAGE3_DEPS) +DEP_LIBS := $(STAGE1_DEPS) $(STAGE2_DEPS) $(STAGE3_DEPS) default: install | $(build_prefix) get: $(addprefix get-, $(DEP_LIBS)) @@ -206,7 +206,7 @@ getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-R ## PATHS ## # sort is used to remove potential duplicates -DIRS = $(sort $(build_bindir) $(build_libdir) $(build_includedir) $(build_sysconfdir) $(build_datarootdir) $(build_staging)) +DIRS := $(sort $(build_bindir) $(build_libdir) $(build_includedir) $(build_sysconfdir) $(build_datarootdir) $(build_staging)) $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) @@ -257,8 +257,8 @@ define git-external include $(SRCDIR)/$1.version ifneq ($(NO_GIT),1) -$2_SRC_DIR = $1 -$2_SRC_FILE = $$(SRCDIR)/srccache/$1.git +$2_SRC_DIR := $1 +$2_SRC_FILE := $$(SRCDIR)/srccache/$1.git $$($2_SRC_FILE): | $$(SRCDIR)/srccache git clone -q --mirror --depth=10 --branch $$($2_BRANCH) $$($2_GIT_URL) $$@ $5/$1: | $$($2_SRC_FILE) @@ -294,8 +294,8 @@ endif else # NO_GIT -$2_SRC_DIR = $1-$$($2_SHA1) -$2_SRC_FILE = $$(SRCDIR)/srccache/$$($2_SRC_DIR).tar.gz +$2_SRC_DIR := $1-$$($2_SHA1) +$2_SRC_FILE := $$(SRCDIR)/srccache/$$($2_SRC_DIR).tar.gz $$($2_SRC_FILE): | $$(SRCDIR)/srccache $$(JLDOWNLOAD) $$@ $$(call $2_TAR_URL,$$($2_SHA1)) $5/$$($2_SRC_DIR)/$3: $$($2_SRC_FILE) @@ -320,14 +320,14 @@ LLVM_GIT_URL_LIBCXX ?= $(LLVM_GIT_URL_BASE)/libcxx.git LLVM_GIT_URL_LIBCXXABI ?= $(LLVM_GIT_URL_BASE)/libcxxabi.git ifeq ($(BUILD_LLDB), 1) -BUILD_LLVM_CLANG = 1 +BUILD_LLVM_CLANG := 1 # because it's a build requirement endif ifeq ($(LLVM_DEBUG),1) -LLVM_BUILDTYPE = Debug +LLVM_BUILDTYPE := Debug else -LLVM_BUILDTYPE = Release +LLVM_BUILDTYPE := Release endif ifeq ($(LLVM_ASSERTIONS),1) LLVM_BUILDTYPE := $(LLVM_BUILDTYPE)+Asserts @@ -341,60 +341,60 @@ LLVM_BUILDTYPE := $(LLVM_BUILDTYPE)+ASAN endif endif -LLVM_SRC_DIR=$(SRCDIR)/srccache/llvm-$(LLVM_VER) -LLVM_BUILD_DIR=$(BUILDDIR)/llvm-$(LLVM_VER) -LLVM_LIB_FILE = libLLVMCodeGen.a +LLVM_SRC_DIR:=$(SRCDIR)/srccache/llvm-$(LLVM_VER) +LLVM_BUILD_DIR:=$(BUILDDIR)/llvm-$(LLVM_VER) +LLVM_LIB_FILE := libLLVMCodeGen.a ifeq ($(LLVM_USE_CMAKE),1) -LLVM_OBJ_SOURCE = $(LLVM_BUILD_DIR)/build_$(LLVM_BUILDTYPE)/lib/$(LLVM_LIB_FILE) +LLVM_OBJ_SOURCE := $(LLVM_BUILD_DIR)/build_$(LLVM_BUILDTYPE)/lib/$(LLVM_LIB_FILE) else -LLVM_OBJ_SOURCE = $(LLVM_BUILD_DIR)/build_$(LLVM_BUILDTYPE)/$(LLVM_FLAVOR)/lib/$(LLVM_LIB_FILE) +LLVM_OBJ_SOURCE := $(LLVM_BUILD_DIR)/build_$(LLVM_BUILDTYPE)/$(LLVM_FLAVOR)/lib/$(LLVM_LIB_FILE) endif -LLVM_OBJ_TARGET = $(build_libdir)/$(LLVM_LIB_FILE) +LLVM_OBJ_TARGET := $(build_libdir)/$(LLVM_LIB_FILE) ifneq ($(LLVM_VER),svn) ifeq ($(LLVM_VER), 3.3) -LLVM_TAR=$(SRCDIR)/srccache/llvm-$(LLVM_VER).src.tar.gz +LLVM_TAR:=$(SRCDIR)/srccache/llvm-$(LLVM_VER).src.tar.gz else -LLVM_TAR=$(SRCDIR)/srccache/llvm-$(LLVM_VER).src.tar.xz +LLVM_TAR:=$(SRCDIR)/srccache/llvm-$(LLVM_VER).src.tar.xz endif ifeq ($(BUILD_LLDB),1) ifeq ($(LLVM_VER), 3.3) -LLVM_LLDB_TAR=$(SRCDIR)/srccache/lldb-$(LLVM_VER).src.tar.gz +LLVM_LLDB_TAR:=$(SRCDIR)/srccache/lldb-$(LLVM_VER).src.tar.gz else -LLVM_LLDB_TAR=$(SRCDIR)/srccache/lldb-$(LLVM_VER).src.tar.xz +LLVM_LLDB_TAR:=$(SRCDIR)/srccache/lldb-$(LLVM_VER).src.tar.xz endif # LLVM_VER == 3.3 endif # BUILD_LLDB ifeq ($(BUILD_LLVM_CLANG),1) ifeq ($(LLVM_VER), 3.3) -LLVM_CLANG_TAR=$(SRCDIR)/srccache/cfe-$(LLVM_VER).src.tar.gz -LLVM_COMPILER_RT_TAR=$(SRCDIR)/srccache/compiler-rt-$(LLVM_VER).src.tar.gz +LLVM_CLANG_TAR:=$(SRCDIR)/srccache/cfe-$(LLVM_VER).src.tar.gz +LLVM_COMPILER_RT_TAR:=$(SRCDIR)/srccache/compiler-rt-$(LLVM_VER).src.tar.gz else -LLVM_CLANG_TAR=$(SRCDIR)/srccache/cfe-$(LLVM_VER).src.tar.xz -LLVM_COMPILER_RT_TAR=$(SRCDIR)/srccache/compiler-rt-$(LLVM_VER).src.tar.xz +LLVM_CLANG_TAR:=$(SRCDIR)/srccache/cfe-$(LLVM_VER).src.tar.xz +LLVM_COMPILER_RT_TAR:=$(SRCDIR)/srccache/compiler-rt-$(LLVM_VER).src.tar.xz endif # LLVM_VER else -LLVM_CLANG_TAR= -LLVM_COMPILER_RT_TAR= -LLVM_LIBCXX_TAR= +LLVM_CLANG_TAR:= +LLVM_COMPILER_RT_TAR:= +LLVM_LIBCXX_TAR:= endif # BUILD_LLVM_CLANG ifeq ($(BUILD_CUSTOM_LIBCXX),1) -LLVM_LIBCXX_TAR=$(SRCDIR)/srccache/libcxx-$(LLVM_VER).src.tar.gz +LLVM_LIBCXX_TAR:=$(SRCDIR)/srccache/libcxx-$(LLVM_VER).src.tar.gz endif endif # LLVM_VER != svn -LLVM_CXXFLAGS = $(CXXFLAGS) -LLVM_CPPFLAGS = $(CPPFLAGS) -LLVM_LDFLAGS = $(LDFLAGS) -LLVM_TARGETS = host -LLVM_TARGET_FLAGS = --enable-targets=$(LLVM_TARGETS) +LLVM_CXXFLAGS := $(CXXFLAGS) +LLVM_CPPFLAGS := $(CPPFLAGS) +LLVM_LDFLAGS := $(LDFLAGS) +LLVM_TARGETS := host +LLVM_TARGET_FLAGS := --enable-targets=$(LLVM_TARGETS) LLVM_CMAKE += -DLLVM_TARGETS_TO_BUILD:STRING="$(LLVM_TARGETS)" -DCMAKE_BUILD_TYPE="$(LLVM_BUILDTYPE)" -DLLVM_BUILD_LLVM_DYLIB:BOOL=ON -DLLVM_DYLIB_EXPORT_ALL:BOOL=ON LLVM_FLAGS += --disable-profiling --enable-shared --enable-static $(LLVM_TARGET_FLAGS) --disable-bindings --disable-docs # LLVM has weird install prefixes (see llvm-$(LLVM_VER)/build_$(LLVM_BUILDTYPE)/Makefile.config for the full list) # We map them here to the "normal" ones, which means just prefixing "PROJ_" to the variable name. -LLVM_MFLAGS = PROJ_libdir=$(build_libdir) PROJ_bindir=$(build_bindir) PROJ_includedir=$(build_includedir) +LLVM_MFLAGS := PROJ_libdir=$(build_libdir) PROJ_bindir=$(build_bindir) PROJ_includedir=$(build_includedir) ifeq ($(LLVM_ASSERTIONS), 1) LLVM_FLAGS += --enable-assertions LLVM_CMAKE += -DLLVM_ENABLE_ASSERTIONS:BOOL=ON @@ -448,19 +448,19 @@ endif # ARCH == ppc64 ifeq ($(LLVM_SANITIZE),1) ifeq ($(SANITIZE_MEMORY),1) -LLVM_CC = CFLAGS="$(CFLAGS) -fsanitize=memory -fsanitize-memory-track-origins" +LLVM_CC := CFLAGS="$(CFLAGS) -fsanitize=memory -fsanitize-memory-track-origins" LLVM_LDFLAGS += -fsanitize=memory -fsanitize-memory-track-origins LLVM_CXXFLAGS += -fsanitize=memory -fsanitize-memory-track-origins LLVM_CMAKE += -DLLVM_USE_SANITIZER="MemoryWithOrigins" else -LLVM_CC = CFLAGS="$(CFLAGS) -fsanitize=address" +LLVM_CC := CFLAGS="$(CFLAGS) -fsanitize=address" LLVM_LDFLAGS += -fsanitize=address LLVM_CXXFLAGS += -fsanitize=address LLVM_CMAKE += -DLLVM_USE_SANITIZER="Address" endif LLVM_MFLAGS += TOOL_NO_EXPORTS= HAVE_LINK_VERSION_SCRIPT=0 else -LLVM_CC = +LLVM_CC := endif # LLVM_SANITIZE ifneq ($(LLVM_CXXFLAGS),) @@ -538,7 +538,7 @@ LLVM_FLAGS += --with-python="$(shell $(SRCDIR)/find_python2)" ifeq ($(BUILD_CUSTOM_LIBCXX),1) ifeq ($(USEICC),1) -LIBCXX_EXTRA_FLAGS = -Bstatic -lirc -Bdynamic +LIBCXX_EXTRA_FLAGS := -Bstatic -lirc -Bdynamic endif $(LLVM_SRC_DIR)/projects/libcxx: $(LLVM_LIBCXX_TAR) | $(LLVM_SRC_DIR)/configure @@ -579,7 +579,7 @@ install-libcxx: $(build_libdir)/libc++.so.1.0 endif ifeq ($(BUILD_CUSTOM_LIBCXX),1) -LIBCXX_DEPENDENCY = $(build_libdir)/libc++abi.so.1.0 $(build_libdir)/libc++.so.1.0 +LIBCXX_DEPENDENCY := $(build_libdir)/libc++abi.so.1.0 $(build_libdir)/libc++.so.1.0 get-llvm: get-libcxx get-libcxxabi endif @@ -658,7 +658,7 @@ endif # LLVM_VER touch -c $@ # Apply version-specific LLVM patches -LLVM_PATCH_LIST= +LLVM_PATCH_LIST:= define LLVM_PATCH $$(LLVM_SRC_DIR)/$1.patch-applied: $(LLVM_SRC_DIR)/configure | $$(SRCDIR)/$1.patch cd $$(LLVM_SRC_DIR) && patch -p1 < $$(SRCDIR)/$1.patch @@ -676,7 +676,7 @@ $(eval $(call LLVM_PATCH,zerosign-llvm-3.6.0)) $(eval $(call LLVM_PATCH,win64-allocas-llvm-3.6.0)) endif # LLVM_VER -LLVM_BUILDDIR_withtype = $(LLVM_BUILD_DIR)/build_$(LLVM_BUILDTYPE) +LLVM_BUILDDIR_withtype := $(LLVM_BUILD_DIR)/build_$(LLVM_BUILDTYPE) ifneq ($(LLVM_USE_CMAKE),) $(LLVM_BUILDDIR_withtype)/CMakeCache.txt: $(LLVM_SRC_DIR)/configure | $(llvm_python_workaround) $(LIBCXX_DEPENDENCY) @@ -710,9 +710,9 @@ $(LLVM_OBJ_SOURCE): $(LLVM_BUILDDIR_withtype)/config.status $(LLVM_PATCH_LIST) | endif # LLVM_USE_CMAKE ifeq ($(LLVM_USE_CMAKE),1) -CHECK_COMMAND = $(CMAKE) --build . check +CHECK_COMMAND := $(CMAKE) --build . check else -CHECK_COMMAND = $(MAKE) $(LLVM_MFLAGS) check +CHECK_COMMAND := $(MAKE) $(LLVM_MFLAGS) check endif $(LLVM_BUILDDIR_withtype)/checked: $(LLVM_OBJ_SOURCE) | $(llvm_python_workaround) ifeq ($(OS),$(BUILD_OS)) @@ -762,14 +762,14 @@ update-llvm: endif ## LIBUV ## -LIBUV_GIT_URL=git://github.com/JuliaLang/libuv.git +LIBUV_GIT_URL:=git://github.com/JuliaLang/libuv.git LIBUV_TAR_URL=https://api.github.com/repos/JuliaLang/libuv/tarball/$1 $(eval $(call git-external,libuv,LIBUV,configure,.libs/libuv.la,$(SRCDIR)/srccache)) -UV_SRC_TARGET = $(BUILDDIR)/$(LIBUV_SRC_DIR)/.libs/libuv.la -UV_OBJ_TARGET = $(build_libdir)/libuv.la +UV_SRC_TARGET := $(BUILDDIR)/$(LIBUV_SRC_DIR)/.libs/libuv.la +UV_OBJ_TARGET := $(build_libdir)/libuv.la -UV_CFLAGS = +UV_CFLAGS := ifeq ($(USEMSVC), 1) UV_CFLAGS += -DBUILDING_UV_SHARED endif @@ -785,9 +785,9 @@ ifneq ($(VERBOSE), 0) UV_MFLAGS += V=1 endif ifneq ($(USEMSVC), 1) -UV_FLAGS = $(UV_MFLAGS) +UV_FLAGS := $(UV_MFLAGS) else -UV_FLAGS = --disable-shared $(UV_MFLAGS) +UV_FLAGS := --disable-shared $(UV_MFLAGS) endif $(BUILDDIR)/$(LIBUV_SRC_DIR)/config.status: $(SRCDIR)/srccache/$(LIBUV_SRC_DIR)/configure @@ -824,13 +824,13 @@ install-libuv: $(UV_OBJ_TARGET) ## PCRE ## -PCRE_SRC_TARGET = $(BUILDDIR)/pcre2-$(PCRE_VER)/.libs/libpcre2-8.$(SHLIB_EXT) -PCRE_OBJ_TARGET = $(build_shlibdir)/libpcre2-8.$(SHLIB_EXT) +PCRE_SRC_TARGET := $(BUILDDIR)/pcre2-$(PCRE_VER)/.libs/libpcre2-8.$(SHLIB_EXT) +PCRE_OBJ_TARGET := $(build_shlibdir)/libpcre2-8.$(SHLIB_EXT) # Force optimization for PCRE flags (Issue #11668) -PCRE_CFLAGS = -O3 +PCRE_CFLAGS := -O3 ifneq ($(OS),WINNT) -PCRE_LDFLAGS = "-Wl,-rpath,'$(build_libdir)'" +PCRE_LDFLAGS := "-Wl,-rpath,'$(build_libdir)'" endif $(SRCDIR)/srccache/pcre2-$(PCRE_VER).tar.bz2: | $(SRCDIR)/srccache @@ -873,13 +873,13 @@ install-pcre: $(PCRE_OBJ_TARGET) ## openlibm ## -OPENLIBM_GIT_URL = git://github.com/JuliaLang/openlibm.git +OPENLIBM_GIT_URL := git://github.com/JuliaLang/openlibm.git OPENLIBM_TAR_URL = https://api.github.com/repos/JuliaLang/openlibm/tarball/$1 $(eval $(call git-external,openlibm,OPENLIBM,Makefile,libopenlibm.$(SHLIB_EXT),$(BUILDDIR))) -OPENLIBM_OBJ_TARGET = $(build_shlibdir)/libopenlibm.$(SHLIB_EXT) $(build_libdir)/libopenlibm.a -OPENLIBM_OBJ_SOURCE = $(BUILDDIR)/$(OPENLIBM_SRC_DIR)/libopenlibm.$(SHLIB_EXT) -OPENLIBM_FLAGS = ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) +OPENLIBM_OBJ_TARGET := $(build_shlibdir)/libopenlibm.$(SHLIB_EXT) $(build_libdir)/libopenlibm.a +OPENLIBM_OBJ_SOURCE := $(BUILDDIR)/$(OPENLIBM_SRC_DIR)/libopenlibm.$(SHLIB_EXT) +OPENLIBM_FLAGS := ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) ifeq (CYGWIN,$(findstring CYGWIN,$(BUILD_OS))) OPENLIBM_FLAGS += OPENLIBM_HOME="$(call cygpath_w,$(abspath $(OPENLIBM_SRC_DIR)))" endif @@ -905,19 +905,19 @@ install-openlibm: $(OPENLIBM_OBJ_TARGET) ## openspecfun ## -OPENSPECFUN_GIT_URL = git://github.com/JuliaLang/openspecfun.git +OPENSPECFUN_GIT_URL := git://github.com/JuliaLang/openspecfun.git OPENSPECFUN_TAR_URL = https://api.github.com/repos/JuliaLang/openspecfun/tarball/$1 $(eval $(call git-external,openspecfun,OPENSPECFUN,Makefile,libopenspecfun.$(SHLIB_EXT),$(BUILDDIR))) # issue 8799 -OPENSPECFUN_CFLAGS = -O3 -std=c99 +OPENSPECFUN_CFLAGS := -O3 -std=c99 ifeq ($(USEICC),1) OPENSPECFUN_CFLAGS += -fp-model precise endif -OPENSPECFUN_OBJ_TARGET = $(build_shlibdir)/libopenspecfun.$(SHLIB_EXT) -OPENSPECFUN_OBJ_SOURCE = $(BUILDDIR)/$(OPENSPECFUN_SRC_DIR)/libopenspecfun.$(SHLIB_EXT) -OPENSPECFUN_FLAGS = ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) FFLAGS="$(JFFLAGS)" CFLAGS="$(CFLAGS) $(OPENSPECFUN_CFLAGS)" +OPENSPECFUN_OBJ_TARGET := $(build_shlibdir)/libopenspecfun.$(SHLIB_EXT) +OPENSPECFUN_OBJ_SOURCE := $(BUILDDIR)/$(OPENSPECFUN_SRC_DIR)/libopenspecfun.$(SHLIB_EXT) +OPENSPECFUN_FLAGS := ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) FFLAGS="$(JFFLAGS)" CFLAGS="$(CFLAGS) $(OPENSPECFUN_CFLAGS)" ifeq ($(USE_SYSTEM_LIBM),0) OPENSPECFUN_FLAGS += USE_OPENLIBM=1 @@ -946,10 +946,10 @@ install-openspecfun: $(OPENSPECFUN_OBJ_TARGET) ## DSFMT ## -DSFMT_OBJ_TARGET = $(build_shlibdir)/libdSFMT.$(SHLIB_EXT) $(build_includedir)/dSFMT.h -DSFMT_OBJ_SOURCE = $(BUILDDIR)/dsfmt-$(DSFMT_VER)/libdSFMT.$(SHLIB_EXT) +DSFMT_OBJ_TARGET := $(build_shlibdir)/libdSFMT.$(SHLIB_EXT) $(build_includedir)/dSFMT.h +DSFMT_OBJ_SOURCE := $(BUILDDIR)/dsfmt-$(DSFMT_VER)/libdSFMT.$(SHLIB_EXT) -DSFMT_CFLAGS = $(CFLAGS) -DNDEBUG -DDSFMT_MEXP=19937 $(fPIC) -DDSFMT_DO_NOT_USE_OLD_NAMES +DSFMT_CFLAGS := $(CFLAGS) -DNDEBUG -DDSFMT_MEXP=19937 $(fPIC) -DDSFMT_DO_NOT_USE_OLD_NAMES ifneq ($(USEMSVC), 1) DSFMT_CFLAGS += -O3 -finline-functions -fomit-frame-pointer -fno-strict-aliasing \ --param max-inline-insns-single=1800 -Wmissing-prototypes -Wall -std=c99 -shared @@ -992,8 +992,8 @@ install-dsfmt: $(DSFMT_OBJ_TARGET) ## Rmath-julia ## -RMATH_JULIA_OBJ_TARGET = $(build_shlibdir)/libRmath-julia.$(SHLIB_EXT) -RMATH_JULIA_OBJ_SOURCE = $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER)/src/libRmath-julia.$(SHLIB_EXT) +RMATH_JULIA_OBJ_TARGET := $(build_shlibdir)/libRmath-julia.$(SHLIB_EXT) +RMATH_JULIA_OBJ_SOURCE := $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER)/src/libRmath-julia.$(SHLIB_EXT) ifeq ($(USE_SYSTEM_DSFMT),0) $(RMATH_JULIA_OBJ_SOURCE): $(DSFMT_OBJ_TARGET) @@ -1032,8 +1032,8 @@ install-Rmath-julia: $(RMATH_JULIA_OBJ_TARGET) ## objconv ## -OBJCONV_SOURCE = $(BUILDDIR)/objconv/objconv -OBJCONV_TARGET = $(build_bindir)/objconv +OBJCONV_SOURCE := $(BUILDDIR)/objconv/objconv +OBJCONV_TARGET := $(build_bindir)/objconv $(SRCDIR)/srccache/objconv.zip: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://www.agner.org/optimize/objconv.zip @@ -1061,13 +1061,13 @@ install-objconv: $(OBJCONV_TARGET) ## OpenBLAS ## # LAPACK is built into OpenBLAS by default -OPENBLAS_GIT_URL = git://github.com/xianyi/OpenBLAS.git +OPENBLAS_GIT_URL := git://github.com/xianyi/OpenBLAS.git OPENBLAS_TAR_URL = https://api.github.com/repos/xianyi/OpenBLAS/tarball/$1 $(eval $(call git-external,openblas,OPENBLAS,Makefile,libopenblas.$(SHLIB_EXT),$(BUILDDIR))) -OPENBLAS_OBJ_SOURCE = $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/libopenblas.$(SHLIB_EXT) -OPENBLAS_OBJ_TARGET = $(build_shlibdir)/libopenblas.$(SHLIB_EXT) -OPENBLAS_BUILD_OPTS = CC="$(CC)" FC="$(FC)" RANLIB="$(RANLIB)" FFLAGS="$(FFLAGS) $(JFFLAGS)" TARGET=$(OPENBLAS_TARGET_ARCH) BINARY=$(BINARY) +OPENBLAS_OBJ_SOURCE := $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/libopenblas.$(SHLIB_EXT) +OPENBLAS_OBJ_TARGET := $(build_shlibdir)/libopenblas.$(SHLIB_EXT) +OPENBLAS_BUILD_OPTS := CC="$(CC)" FC="$(FC)" RANLIB="$(RANLIB)" FFLAGS="$(FFLAGS) $(JFFLAGS)" TARGET=$(OPENBLAS_TARGET_ARCH) BINARY=$(BINARY) # Thread support ifeq ($(OPENBLAS_USE_THREAD), 1) @@ -1163,9 +1163,9 @@ install-openblas: $(OPENBLAS_OBJ_TARGET) # should always be compiled with (a real) gcc, it's # configure script will search for the best match # (gcc 4.7, gcc, clang,ICC/microsoft/others) -ATLAS_OBJ_SOURCE = $(BUILDDIR)/atlas/build/lib/libsatlas.$(SHLIB_EXT) -ATLAS_OBJ_TARGET = $(build_shlibdir)/libsatlas.$(SHLIB_EXT) -ATLAS_FLAGS = --shared --prefix=$(build_prefix) --cc=gcc -t 0 \ +ATLAS_OBJ_SOURCE := $(BUILDDIR)/atlas/build/lib/libsatlas.$(SHLIB_EXT) +ATLAS_OBJ_TARGET := $(build_shlibdir)/libsatlas.$(SHLIB_EXT) +ATLAS_FLAGS := --shared --prefix=$(build_prefix) --cc=gcc -t 0 \ --with-netlib-lapack-tarfile=$(JULIAHOME)/deps/lapack-$(LAPACK_VER).tgz ifeq ($(OS), WINNT) ATLAS_FLAGS += -b 32 @@ -1211,7 +1211,7 @@ check-atlas: compile-atlas install-atlas: $(ATLAS_OBJ_TARGET) ## Mac gfortran BLAS wrapper ## -GFORTBLAS_FFLAGS = +GFORTBLAS_FFLAGS := ifeq ($(OS),Darwin) ifeq ($(USE_SYSTEM_BLAS),1) ifeq ($(USE_SYSTEM_LAPACK),0) @@ -1241,14 +1241,14 @@ endif ## LAPACK ## ifeq ($(USE_SYSTEM_LAPACK), 0) -LAPACK_OBJ_TARGET = $(build_shlibdir)/liblapack.$(SHLIB_EXT) -LAPACK_OBJ_SOURCE = $(BUILDDIR)/lapack-$(LAPACK_VER)/liblapack.$(SHLIB_EXT) +LAPACK_OBJ_TARGET := $(build_shlibdir)/liblapack.$(SHLIB_EXT) +LAPACK_OBJ_SOURCE := $(BUILDDIR)/lapack-$(LAPACK_VER)/liblapack.$(SHLIB_EXT) else -LAPACK_OBJ_TARGET = -LAPACK_OBJ_SOURCE = +LAPACK_OBJ_TARGET := +LAPACK_OBJ_SOURCE := endif -LAPACK_MFLAGS = NOOPT="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS) -O0" OPTS="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS)" FORTRAN="$(FC)" LOADER="$(FC)" +LAPACK_MFLAGS := NOOPT="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS) -O0" OPTS="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS)" FORTRAN="$(FC)" LOADER="$(FC)" ifneq ($(OS),WINNT) LAPACK_MFLAGS += BLASLIB="-Wl,-rpath,'$(build_libdir)' $(LIBBLAS)" endif @@ -1293,8 +1293,8 @@ install-lapack: $(LAPACK_OBJ_TARGET) ## ARPACK ## -ARPACK_FFLAGS = $(GFORTBLAS_FFLAGS) -ARPACK_CFLAGS = +ARPACK_FFLAGS := $(GFORTBLAS_FFLAGS) +ARPACK_CFLAGS := ifeq ($(USE_BLAS64), 1) ifeq ($(USEIFC),1) @@ -1304,10 +1304,10 @@ ARPACK_FFLAGS += -fdefault-integer-8 ifeq ($(USE_SYSTEM_BLAS), 0) ifeq ($(OPENBLAS_SYMBOLSUFFIX), 64_) ARPACK_FFLAGS += -cpp -ffixed-line-length-none -ARPACK_OPENBLASFCNS1 = axpy copy gemv geqr2 lacpy lahqr lanhs larnv lartg lascl laset scal trevc trmm trsen -ARPACK_OPENBLASFCNS2 = dot ger labad laev2 lamch lanst lanv2 lapy2 larf larfg lasr nrm2 orm2r rot steqr swap -ARPACK_OPENBLASFCNS3 = dotc geru unm2r -ARPACK_OPENBLASFCNS4 = COPY LABAD LAMCH LANHS LANV2 LARFG ROT +ARPACK_OPENBLASFCNS1 := axpy copy gemv geqr2 lacpy lahqr lanhs larnv lartg lascl laset scal trevc trmm trsen +ARPACK_OPENBLASFCNS2 := dot ger labad laev2 lamch lanst lanv2 lapy2 larf larfg lasr nrm2 orm2r rot steqr swap +ARPACK_OPENBLASFCNS3 := dotc geru unm2r +ARPACK_OPENBLASFCNS4 := COPY LABAD LAMCH LANHS LANV2 LARFG ROT ARPACK_FFLAGS += $(foreach fcn, $(ARPACK_OPENBLASFCNS1) $(ARPACK_OPENBLASFCNS2), -Ds$(fcn)=s$(fcn)_64 -Dd$(fcn)=d$(fcn)_64) ARPACK_FFLAGS += $(foreach fcn, $(ARPACK_OPENBLASFCNS1) $(ARPACK_OPENBLASFCNS3), -Dc$(fcn)=c$(fcn)_64 -Dz$(fcn)=z$(fcn)_64) ARPACK_FFLAGS += $(foreach fcn, $(ARPACK_OPENBLASFCNS4), -DS$(fcn)=S$(fcn)_64 -DD$(fcn)=D$(fcn)_64) @@ -1320,15 +1320,15 @@ endif endif ifeq ($(OS),WINNT) -ARPACK_OBJ_SOURCE = $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/.libs/libarpack-2.$(SHLIB_EXT) +ARPACK_OBJ_SOURCE := $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/.libs/libarpack-2.$(SHLIB_EXT) else -ARPACK_OBJ_SOURCE = $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/.libs/libarpack.$(SHLIB_EXT) +ARPACK_OBJ_SOURCE := $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/.libs/libarpack.$(SHLIB_EXT) endif -ARPACK_OBJ_TARGET = $(build_shlibdir)/libarpack.$(SHLIB_EXT) +ARPACK_OBJ_TARGET := $(build_shlibdir)/libarpack.$(SHLIB_EXT) -ARPACK_MFLAGS = F77="$(FC)" MPIF77="$(FC)" +ARPACK_MFLAGS := F77="$(FC)" MPIF77="$(FC)" ARPACK_FFLAGS += $(FFLAGS) $(JFFLAGS) -ARPACK_FLAGS = --with-blas="$(LIBBLAS)" --with-lapack="$(LIBLAPACK)" --disable-mpi --enable-shared FFLAGS="$(ARPACK_FFLAGS)" CFLAGS="$(CFLAGS) $(ARPACK_CFLAGS)" +ARPACK_FLAGS := --with-blas="$(LIBBLAS)" --with-lapack="$(LIBLAPACK)" --disable-mpi --enable-shared FFLAGS="$(ARPACK_FFLAGS)" CFLAGS="$(CFLAGS) $(ARPACK_CFLAGS)" ifneq ($(OS),WINNT) ARPACK_FLAGS += LDFLAGS="$(LDFLAGS) -Wl,-rpath,'$(build_libdir)'" endif @@ -1401,16 +1401,16 @@ install-arpack: $(ARPACK_OBJ_TARGET) ## FFTW ## ifeq ($(OS),WINNT) -FFTW_SINGLE_SRC_TARGET = $(BUILDDIR)/fftw-$(FFTW_VER)-single/.libs/libfftw3f-3.$(SHLIB_EXT) -FFTW_DOUBLE_SRC_TARGET = $(BUILDDIR)/fftw-$(FFTW_VER)-double/.libs/libfftw3-3.$(SHLIB_EXT) +FFTW_SINGLE_SRC_TARGET := $(BUILDDIR)/fftw-$(FFTW_VER)-single/.libs/libfftw3f-3.$(SHLIB_EXT) +FFTW_DOUBLE_SRC_TARGET := $(BUILDDIR)/fftw-$(FFTW_VER)-double/.libs/libfftw3-3.$(SHLIB_EXT) else -FFTW_SINGLE_SRC_TARGET = $(BUILDDIR)/fftw-$(FFTW_VER)-single/.libs/libfftw3f.$(SHLIB_EXT) -FFTW_DOUBLE_SRC_TARGET = $(BUILDDIR)/fftw-$(FFTW_VER)-double/.libs/libfftw3.$(SHLIB_EXT) +FFTW_SINGLE_SRC_TARGET := $(BUILDDIR)/fftw-$(FFTW_VER)-single/.libs/libfftw3f.$(SHLIB_EXT) +FFTW_DOUBLE_SRC_TARGET := $(BUILDDIR)/fftw-$(FFTW_VER)-double/.libs/libfftw3.$(SHLIB_EXT) endif -FFTW_SINGLE_OBJ_TARGET = $(build_shlibdir)/libfftw3f.$(SHLIB_EXT) -FFTW_DOUBLE_OBJ_TARGET = $(build_shlibdir)/libfftw3.$(SHLIB_EXT) +FFTW_SINGLE_OBJ_TARGET := $(build_shlibdir)/libfftw3f.$(SHLIB_EXT) +FFTW_DOUBLE_OBJ_TARGET := $(build_shlibdir)/libfftw3.$(SHLIB_EXT) -FFTW_CONFIG = --enable-shared --disable-fortran --disable-mpi --enable-threads +FFTW_CONFIG := --enable-shared --disable-fortran --disable-mpi --enable-threads ifneq (,$(findstring arm,$(ARCH))) FFTW_CONFIG += else ifeq ($(ARCH), ppc) @@ -1424,9 +1424,8 @@ ifneq ($(ARCH),x86_64) FFTW_CONFIG += --with-incoming-stack-boundary=2 endif endif -FFTW_ENABLE_single = --enable-single -FFTW_ENABLE_double = -FFTW_CONFIG += $(FFTW_ENABLE_$*) +FFTW_ENABLE_single := --enable-single +FFTW_ENABLE_double := $(SRCDIR)/srccache/fftw-$(FFTW_VER).tar.gz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://www.fftw.org/$(notdir $@) @@ -1439,8 +1438,8 @@ $(BUILDDIR)/fftw-$(FFTW_VER)-%/config.status: $(SRCDIR)/srccache/fftw-$(FFTW_VER mkdir -p $(dir $@) @# try to configure with avx support. if that fails, try again without it cd $(dir $@) && \ - ($< $(CONFIGURE_COMMON) $(FFTW_CONFIG) --enable-avx || \ - $< $(CONFIGURE_COMMON) $(FFTW_CONFIG)) + ($< $(CONFIGURE_COMMON) $(FFTW_CONFIG) $(FFTW_ENABLE_$*) --enable-avx || \ + $< $(CONFIGURE_COMMON) $(FFTW_CONFIG) $(FFTW_ENABLE_$*)) $(MAKE) -C $(dir $@) clean touch -c $@ @@ -1528,16 +1527,16 @@ check-fftw-double: $(BUILDDIR)/fftw-$(FFTW_VER)-double/checked install-fftw-double: $(FFTW_DOUBLE_OBJ_TARGET) ## UTF8PROC ## -UTF8PROC_GIT_URL = git://github.com/JuliaLang/utf8proc.git +UTF8PROC_GIT_URL := git://github.com/JuliaLang/utf8proc.git UTF8PROC_TAR_URL = https://api.github.com/repos/JuliaLang/utf8proc/tarball/$1 $(eval $(call git-external,utf8proc,UTF8PROC,Makefile,libutf8proc.a,$(BUILDDIR))) -UTF8PROC_SRC_TARGET = $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/libutf8proc.a -UTF8PROC_OBJ_LIB = $(build_libdir)/libutf8proc.a -UTF8PROC_OBJ_HEADER = $(build_includedir)/utf8proc.h -UTF8PROC_OBJ_TARGET = $(UTF8PROC_OBJ_LIB) $(UTF8PROC_OBJ_HEADER) -UTF8PROC_CFLAGS = -O2 -UTF8PROC_MFLAGS = CC="$(CC) $(DEPS_CFLAGS)" CFLAGS="$(CFLAGS) $(UTF8PROC_CFLAGS)" PICFLAG="$(fPIC)" AR="$(AR)" +UTF8PROC_SRC_TARGET := $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/libutf8proc.a +UTF8PROC_OBJ_LIB := $(build_libdir)/libutf8proc.a +UTF8PROC_OBJ_HEADER := $(build_includedir)/utf8proc.h +UTF8PROC_OBJ_TARGET := $(UTF8PROC_OBJ_LIB) $(UTF8PROC_OBJ_HEADER) +UTF8PROC_CFLAGS := -O2 +UTF8PROC_MFLAGS := CC="$(CC) $(DEPS_CFLAGS)" CFLAGS="$(CFLAGS) $(UTF8PROC_CFLAGS)" PICFLAG="$(fPIC)" AR="$(AR)" $(UTF8PROC_SRC_TARGET): $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/Makefile $(MAKE) -C $(dir $<) $(UTF8PROC_MFLAGS) libutf8proc.a @@ -1566,13 +1565,13 @@ install-utf8proc: $(UTF8PROC_OBJ_TARGET) ## SUITESPARSE ## -SUITESPARSE_OBJ_SOURCE = $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/UMFPACK/Lib/libumfpack.a -SUITESPARSE_OBJ_TARGET = $(build_shlibdir)/libspqr.$(SHLIB_EXT) +SUITESPARSE_OBJ_SOURCE := $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/UMFPACK/Lib/libumfpack.a +SUITESPARSE_OBJ_TARGET := $(build_shlibdir)/libspqr.$(SHLIB_EXT) ifeq ($(USE_BLAS64), 1) -UMFPACK_CONFIG = -DLONGBLAS='long long' -CHOLMOD_CONFIG = -DLONGBLAS='long long' -SPQR_CONFIG = -DLONGBLAS='long long' +UMFPACK_CONFIG := -DLONGBLAS='long long' +CHOLMOD_CONFIG := -DLONGBLAS='long long' +SPQR_CONFIG := -DLONGBLAS='long long' ifeq ($(USE_SYSTEM_BLAS), 0) ifeq ($(OPENBLAS_SYMBOLSUFFIX), 64_) UMFPACK_CONFIG += -DSUN64 @@ -1582,7 +1581,7 @@ endif endif endif -SUITE_SPARSE_LIB = -lm +SUITE_SPARSE_LIB := -lm ifneq ($(OS), Darwin) ifneq ($(OS), WINNT) SUITE_SPARSE_LIB += -lrt @@ -1591,7 +1590,7 @@ endif ifneq ($(OS), WINNT) SUITE_SPARSE_LIB += -Wl,-rpath,'$(build_libdir)' endif -SUITESPARSE_MFLAGS = CC="$(CC)" CXX="$(CXX)" F77="$(FC)" AR="$(AR)" RANLIB="$(RANLIB)" BLAS="$(LIBBLAS)" LAPACK="$(LIBLAPACK)" \ +SUITESPARSE_MFLAGS := CC="$(CC)" CXX="$(CXX)" F77="$(FC)" AR="$(AR)" RANLIB="$(RANLIB)" BLAS="$(LIBBLAS)" LAPACK="$(LIBLAPACK)" \ INSTALL_LIB="$(build_libdir)" INSTALL_INCLUDE="$(build_includedir)" LIB="$(SUITE_SPARSE_LIB)" \ UMFPACK_CONFIG="$(UMFPACK_CONFIG)" CHOLMOD_CONFIG="$(CHOLMOD_CONFIG)" SPQR_CONFIG="$(SPQR_CONFIG)" @@ -1655,11 +1654,11 @@ install-suitesparse: $(SUITESPARSE_OBJ_TARGET) install-suitesparse-wrapper # SUITESPARSE WRAPPER ifeq ($(USE_SYSTEM_SUITESPARSE), 1) -SUITESPARSE_INC = -I /usr/include/suitesparse -SUITESPARSE_LIB = -lumfpack -lcholmod -lamd -lcamd -lcolamd -lspqr +SUITESPARSE_INC := -I /usr/include/suitesparse +SUITESPARSE_LIB := -lumfpack -lcholmod -lamd -lcamd -lcolamd -lspqr else -SUITESPARSE_INC = -I $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/CHOLMOD/Include -I $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/SuiteSparse_config -I $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/SPQR/Include -SUITESPARSE_LIB = -L$(build_shlibdir) -lcholmod -lumfpack -lspqr $(RPATH_ORIGIN) +SUITESPARSE_INC := -I $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/CHOLMOD/Include -I $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/SuiteSparse_config -I $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/SPQR/Include +SUITESPARSE_LIB := -L$(build_shlibdir) -lcholmod -lumfpack -lspqr $(RPATH_ORIGIN) $(build_shlibdir)/libsuitesparse_wrapper.$(SHLIB_EXT): $(SUITESPARSE_OBJ_TARGET) endif @@ -1681,10 +1680,10 @@ install-suitesparse-wrapper: $(build_shlibdir)/libsuitesparse_wrapper.$(SHLIB_EX ## UNWIND ## -LIBUNWIND_TARGET_OBJ = $(build_libdir)/libunwind.a -LIBUNWIND_TARGET_SOURCE = $(BUILDDIR)/libunwind-$(UNWIND_VER)/src/.libs/libunwind.a -LIBUNWIND_CFLAGS = -U_FORTIFY_SOURCE $(fPIC) -LIBUNWIND_CPPFLAGS = +LIBUNWIND_TARGET_OBJ := $(build_libdir)/libunwind.a +LIBUNWIND_TARGET_SOURCE := $(BUILDDIR)/libunwind-$(UNWIND_VER)/src/.libs/libunwind.a +LIBUNWIND_CFLAGS := -U_FORTIFY_SOURCE $(fPIC) +LIBUNWIND_CPPFLAGS := $(SRCDIR)/srccache/libunwind-$(UNWIND_VER).tar.gz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://download.savannah.gnu.org/releases/libunwind/$(notdir $@) @@ -1729,10 +1728,10 @@ install-unwind: $(LIBUNWIND_TARGET_OBJ) ## OS X Unwind ## -OSXUNWIND_FLAGS = ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) CFLAGS="$(CFLAGS) -ggdb3 -O0" CXXFLAGS="$(CXXFLAGS) -ggdb3 -O0" SFLAGS="-ggdb3" LDFLAGS="$(LDFLAGS) -Wl,-macosx_version_min,10.7" +OSXUNWIND_FLAGS := ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) CFLAGS="$(CFLAGS) -ggdb3 -O0" CXXFLAGS="$(CXXFLAGS) -ggdb3 -O0" SFLAGS="-ggdb3" LDFLAGS="$(LDFLAGS) -Wl,-macosx_version_min,10.7" -OSXUNWIND_OBJ_TARGET = $(build_shlibdir)/libosxunwind.$(SHLIB_EXT) -OSXUNWIND_OBJ_SOURCE = $(BUILDDIR)/libosxunwind-$(OSXUNWIND_VER)/libosxunwind.$(SHLIB_EXT) +OSXUNWIND_OBJ_TARGET := $(build_shlibdir)/libosxunwind.$(SHLIB_EXT) +OSXUNWIND_OBJ_SOURCE := $(BUILDDIR)/libosxunwind-$(OSXUNWIND_VER)/libosxunwind.$(SHLIB_EXT) $(SRCDIR)/srccache/libosxunwind-$(OSXUNWIND_VER).tar.gz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://github.com/JuliaLang/libosxunwind/archive/v$(OSXUNWIND_VER).tar.gz @@ -1765,8 +1764,8 @@ install-osxunwind: $(OSXUNWIND_OBJ_TARGET) ## GMP ## -GMP_SRC_TARGET = $(BUILDDIR)/gmp-$(GMP_VER)/.libs/libgmp.$(SHLIB_EXT) -GMP_OBJ_TARGET = $(build_shlibdir)/libgmp.$(SHLIB_EXT) +GMP_SRC_TARGET := $(BUILDDIR)/gmp-$(GMP_VER)/.libs/libgmp.$(SHLIB_EXT) +GMP_OBJ_TARGET := $(build_shlibdir)/libgmp.$(SHLIB_EXT) ifeq ($(SANITIZE),1) GMP_CONFIGURE_OPTS += --disable-assembly @@ -1813,14 +1812,14 @@ check-gmp: $(BUILDDIR)/gmp-$(GMP_VER)/checked install-gmp: $(GMP_OBJ_TARGET) ifeq ($(USE_SYSTEM_GMP), 0) -MPFR_DEPS = $(GMP_OBJ_TARGET) +MPFR_DEPS := $(GMP_OBJ_TARGET) endif ## MPFR ## ifeq ($(USE_SYSTEM_MPFR), 0) ifeq ($(USE_SYSTEM_GMP), 0) -MPFR_OPTS = --with-gmp-include=$(abspath $(build_includedir)) --with-gmp-lib=$(abspath $(build_shlibdir)) +MPFR_OPTS := --with-gmp-include=$(abspath $(build_includedir)) --with-gmp-lib=$(abspath $(build_shlibdir)) endif endif ifeq ($(BUILD_OS),WINNT) @@ -1830,10 +1829,10 @@ endif endif -MPFR_SRC_TARGET = $(BUILDDIR)/mpfr-$(MPFR_VER)/src/.libs/libmpfr.$(SHLIB_EXT) -MPFR_OBJ_TARGET = $(build_shlibdir)/libmpfr.$(SHLIB_EXT) +MPFR_SRC_TARGET := $(BUILDDIR)/mpfr-$(MPFR_VER)/src/.libs/libmpfr.$(SHLIB_EXT) +MPFR_OBJ_TARGET := $(build_shlibdir)/libmpfr.$(SHLIB_EXT) ifeq ($(OS),Darwin) -MPFR_CHECK_MFLAGS = LDFLAGS="$(LDFLAGS) -Wl,-rpath,'$(build_libdir)'" +MPFR_CHECK_MFLAGS := LDFLAGS="$(LDFLAGS) -Wl,-rpath,'$(build_libdir)'" endif ifeq ($(SANITIZE),1) @@ -1879,8 +1878,8 @@ install-mpfr: $(MPFR_OBJ_TARGET) ## patchelf ## -PATCHELF_SOURCE = $(BUILDDIR)/patchelf-$(PATCHELF_VER)/src/patchelf -PATCHELF_TARGET = $(build_bindir)/patchelf +PATCHELF_SOURCE := $(BUILDDIR)/patchelf-$(PATCHELF_VER)/src/patchelf +PATCHELF_TARGET := $(build_bindir)/patchelf $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER).tar.gz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://nixos.org/releases/patchelf/patchelf-$(PATCHELF_VER)/patchelf-$(PATCHELF_VER).tar.gz @@ -1921,8 +1920,8 @@ install-patchelf: $(PATCHELF_TARGET) ## Git # only used for the mac binaries in contrib/mac/app/Makefile -GIT_SOURCE = $(BUILDDIR)/git-$(GIT_VER)/git -GIT_TARGET = $(build_libexecdir)/git +GIT_SOURCE := $(BUILDDIR)/git-$(GIT_VER)/git +GIT_TARGET := $(build_libexecdir)/git $(SRCDIR)/srccache/git-$(GIT_VER).tar.gz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://www.kernel.org/pub/software/scm/git/git-$(GIT_VER).tar.gz @@ -1957,8 +1956,8 @@ install-git: $(GIT_TARGET) ## virtualenv -VIRTUALENV_SOURCE = $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER)/virtualenv.py -VIRTUALENV_TARGET = $(BUILDDIR)/julia-env +VIRTUALENV_SOURCE := $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER)/virtualenv.py +VIRTUALENV_TARGET := $(BUILDDIR)/julia-env $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER).tar.gz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://pypi.python.org/packages/source/v/virtualenv/$(notdir $@) @@ -1987,14 +1986,14 @@ install-virtualenv: $(VIRTUALENV_TARGET) ## libgit2 -LIBGIT2_GIT_URL = git://github.com/libgit2/libgit2.git +LIBGIT2_GIT_URL := git://github.com/libgit2/libgit2.git LIBGIT2_TAR_URL = https://api.github.com/repos/libgit2/libgit2/tarball/$1 $(eval $(call git-external,libgit2,LIBGIT2,CMakeLists.txt,build/libgit2.$(SHLIB_EXT),$(SRCDIR)/srccache)) -LIBGIT2_OBJ_SOURCE = $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/libgit2.$(SHLIB_EXT) -LIBGIT2_OBJ_TARGET = $(build_shlibdir)/libgit2.$(SHLIB_EXT) +LIBGIT2_OBJ_SOURCE := $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/libgit2.$(SHLIB_EXT) +LIBGIT2_OBJ_TARGET := $(build_shlibdir)/libgit2.$(SHLIB_EXT) -LIBGIT2_OPTS = $(CMAKE_COMMON) -DTHREADSAFE=ON +LIBGIT2_OPTS := $(CMAKE_COMMON) -DTHREADSAFE=ON ifeq ($(OS),WINNT) LIBGIT2_OPTS += -DWIN32=ON -DMINGW=ON -DUSE_SSH=OFF -DCMAKE_SYSTEM_NAME=Windows ifneq ($(ARCH),x86_64) diff --git a/src/Makefile b/src/Makefile index 82e34f97b7cfc..d326e9e230624 100644 --- a/src/Makefile +++ b/src/Makefile @@ -4,22 +4,28 @@ BUILDDIR := . include $(JULIAHOME)/deps/Versions.make include $(JULIAHOME)/Make.inc +ifeq ($(USE_COPY_STACKS),1) +JCFLAGS += -DCOPY_STACKS +endif override CFLAGS += $(JCFLAGS) override CXXFLAGS += $(JCXXFLAGS) override CPPFLAGS += $(JCPPFLAGS) -SRCS = \ +SRCS := \ jltypes gf ast builtins module codegen disasm debuginfo interpreter \ alloc dlload sys init task array dump toplevel jl_uv jlapi signal-handling \ llvm-simdloop simplevector +ifeq ($(JULIAGC),MARKSWEEP) +SRCS += gc +endif -HEADERS = $(addprefix $(SRCDIR)/,julia.h julia_internal.h options.h) $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(LIBUV_INC)/uv.h +HEADERS := $(addprefix $(SRCDIR)/,julia.h julia_internal.h options.h) $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(LIBUV_INC)/uv.h # -I BUILDDIR comes before -I SRCDIR so that the user can override on a per-build-directory basis # for gcc/clang, suggested content is: # #include_next # #define ARGUMENT_TO_OVERRIDE 1 -FLAGS = \ +FLAGS := \ -D_GNU_SOURCE -I$(BUILDDIR) -I$(SRCDIR) \ -I$(SRCDIR)/flisp -I$(SRCDIR)/support \ -I$(shell $(LLVM_CONFIG) --includedir) \ @@ -31,22 +37,22 @@ endif # In LLVM < 3.4, --ldflags includes both options and libraries, so use it both before and after --libs # In LLVM >= 3.4, --ldflags has only options, and --system-libs has the libraries. -LLVMLINK = $(shell $(LLVM_CONFIG) --ldflags) $(shell $(LLVM_CONFIG) --libs) $(shell $(LLVM_CONFIG) --ldflags) $(shell $(LLVM_CONFIG) --system-libs 2> /dev/null) +LLVMLINK := $(shell $(LLVM_CONFIG) --ldflags) $(shell $(LLVM_CONFIG) --libs) $(shell $(LLVM_CONFIG) --ldflags) $(shell $(LLVM_CONFIG) --system-libs 2> /dev/null) ifeq ($(USE_LLVM_SHLIB),1) ifeq ($(LLVM_USE_CMAKE),1) -LLVMLINK = $(shell $(LLVM_CONFIG) --ldflags) -lLLVM +LLVMLINK := $(shell $(LLVM_CONFIG) --ldflags) -lLLVM else -LLVMLINK = $(shell $(LLVM_CONFIG) --ldflags) -lLLVM-$(shell $(LLVM_CONFIG) --version) +LLVMLINK := $(shell $(LLVM_CONFIG) --ldflags) -lLLVM-$(shell $(LLVM_CONFIG) --version) endif FLAGS += -DLLVM_SHLIB endif -COMMON_LIBS = -L$(build_shlibdir) -L$(build_libdir) $(LIBUV) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LLVMLINK) $(OSLIBS) -DEBUG_LIBS = $(WHOLE_ARCHIVE) $(BUILDDIR)/flisp/libflisp-debug.a $(WHOLE_ARCHIVE) $(BUILDDIR)/support/libsupport-debug.a $(COMMON_LIBS) -RELEASE_LIBS = $(WHOLE_ARCHIVE) $(BUILDDIR)/flisp/libflisp.a $(WHOLE_ARCHIVE) $(BUILDDIR)/support/libsupport.a $(COMMON_LIBS) +COMMON_LIBS := -L$(build_shlibdir) -L$(build_libdir) $(LIBUV) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LLVMLINK) $(OSLIBS) +DEBUG_LIBS := $(WHOLE_ARCHIVE) $(BUILDDIR)/flisp/libflisp-debug.a $(WHOLE_ARCHIVE) $(BUILDDIR)/support/libsupport-debug.a $(COMMON_LIBS) +RELEASE_LIBS := $(WHOLE_ARCHIVE) $(BUILDDIR)/flisp/libflisp.a $(WHOLE_ARCHIVE) $(BUILDDIR)/support/libsupport.a $(COMMON_LIBS) -OBJS = $(SRCS:%=$(BUILDDIR)/%.o) -DOBJS = $(SRCS:%=$(BUILDDIR)/%.dbg.obj) +OBJS := $(SRCS:%=$(BUILDDIR)/%.o) +DOBJS := $(SRCS:%=$(BUILDDIR)/%.dbg.obj) DEBUGFLAGS += $(FLAGS) SHIPFLAGS += $(FLAGS) @@ -54,20 +60,12 @@ SHIPFLAGS += $(FLAGS) SHIPFLAGS += "-DJL_SYSTEM_IMAGE_PATH=\"$(build_private_libdir_rel)/sys.$(SHLIB_EXT)\"" DEBUGFLAGS += "-DJL_SYSTEM_IMAGE_PATH=\"$(build_private_libdir_rel)/sys-debug.$(SHLIB_EXT)\"" -ifeq ($(JULIAGC),MARKSWEEP) -SRCS += gc -endif - -ifeq ($(USE_COPY_STACKS),1) -JCFLAGS += -DCOPY_STACKS -endif - -FLISP_EXECUTABLE_debug = $(BUILDDIR)/flisp/flisp-debug -FLISP_EXECUTABLE_release = $(BUILDDIR)/flisp/flisp +FLISP_EXECUTABLE_debug := $(BUILDDIR)/flisp/flisp-debug +FLISP_EXECUTABLE_release := $(BUILDDIR)/flisp/flisp ifeq ($(OS),WINNT) -FLISP_EXECUTABLE = $(FLISP_EXECUTABLE_release) +FLISP_EXECUTABLE := $(FLISP_EXECUTABLE_release) else -FLISP_EXECUTABLE = $(FLISP_EXECUTABLE_$(JULIA_BUILD_MODE)) +FLISP_EXECUTABLE := $(FLISP_EXECUTABLE_$(JULIA_BUILD_MODE)) endif default: $(JULIA_BUILD_MODE) # contains either "debug" or "release" @@ -78,13 +76,15 @@ release debug: %: libjulia-% $(BUILDDIR): mkdir $(BUILDDIR) +LLVM_CONFIG_ABSOLUTE := $(shell which $(LLVM_CONFIG)) + $(BUILDDIR)/%.o: $(SRCDIR)/%.c $(HEADERS) | $(BUILDDIR) @$(call PRINT_CC, $(CC) $(CPPFLAGS) $(CFLAGS) $(SHIPFLAGS) $(DISABLE_ASSERTIONS) -c $< -o $@) $(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.c $(HEADERS) | $(BUILDDIR) @$(call PRINT_CC, $(CC) $(CPPFLAGS) $(CFLAGS) $(DEBUGFLAGS) -c $< -o $@) -$(BUILDDIR)/%.o: $(SRCDIR)/%.cpp $(HEADERS) $(shell which $(LLVM_CONFIG)) | $(BUILDDIR) +$(BUILDDIR)/%.o: $(SRCDIR)/%.cpp $(HEADERS) $(LLVM_CONFIG_ABSOLUTE) | $(BUILDDIR) @$(call PRINT_CC, $(CXX) $(shell $(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(SHIPFLAGS) -c $< -o $@) -$(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.cpp $(HEADERS) $(shell which $(LLVM_CONFIG)) | $(BUILDDIR) +$(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.cpp $(HEADERS) $(LLVM_CONFIG_ABSOLUTE) | $(BUILDDIR) @$(call PRINT_CC, $(CXX) $(shell $(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(DEBUGFLAGS) -c $< -o $@) $(BUILDDIR)/julia_flisp.boot.inc: $(BUILDDIR)/julia_flisp.boot $(FLISP_EXECUTABLE) @@ -130,12 +130,12 @@ $(BUILDDIR)/julia_version.h: $(JULIAHOME)/VERSION mv $@.$(JULIA_BUILD_MODE).tmp $@ ifneq ($(USEMSVC), 1) -CXXLD = $(CXX) -shared +CXXLD := $(CXX) -shared ifeq ($(OS),WINNT) CXXLD += -Wl,--out-implib,$(build_libdir)/$(notdir $@).a endif else -CXXLD = $(LD) -dll -export:jl_setjmp -export:jl_longjmp +CXXLD := $(LD) -dll -export:jl_setjmp -export:jl_longjmp endif $(build_shlibdir)/libjulia-debug.$(SHLIB_EXT): $(SRCDIR)/julia.expmap $(DOBJS) $(BUILDDIR)/flisp/libflisp-debug.a $(BUILDDIR)/support/libsupport-debug.a $(LIBUV) @@ -148,9 +148,9 @@ $(BUILDDIR)/libjulia-debug.a: $(SRCDIR)/julia.expmap $(DOBJS) $(BUILDDIR)/flisp/ libjulia-debug: $(build_shlibdir)/libjulia-debug.$(SHLIB_EXT) ifeq ($(SHLIB_EXT), so) - SONAME = -Wl,-soname=libjulia.so + SONAME := -Wl,-soname=libjulia.so else - SONAME = + SONAME := endif $(build_shlibdir)/libjulia.$(SHLIB_EXT): $(SRCDIR)/julia.expmap $(OBJS) $(BUILDDIR)/flisp/libflisp.a $(BUILDDIR)/support/libsupport.a $(LIBUV) diff --git a/src/flisp/Makefile b/src/flisp/Makefile index 1dfc7a52ed318..cbd250f594ea1 100644 --- a/src/flisp/Makefile +++ b/src/flisp/Makefile @@ -6,31 +6,31 @@ override CFLAGS += $(JCFLAGS) override CXXFLAGS += $(JCXXFLAGS) override CPPFLAGS += $(JCPPFLAGS) -NAME = flisp -EXENAME = $(NAME) -LIBTARGET = lib$(NAME) +NAME := flisp +EXENAME := $(NAME) +LIBTARGET := lib$(NAME) -SRCS = flisp.c builtins.c string.c equalhash.c table.c iostream.c \ +SRCS := flisp.c builtins.c string.c equalhash.c table.c iostream.c \ julia_extensions.c ifeq ($(USEMSVC), 1) SRCS += dirname.c endif -HEADERS = $(wildcard *.h) $(LIBUV_INC)/uv.h +HEADERS := $(wildcard *.h) $(LIBUV_INC)/uv.h -OBJS = $(SRCS:%.c=$(BUILDDIR)/%.o) -DOBJS = $(SRCS:%.c=$(BUILDDIR)/%.dbg.obj) -LLTDIR = ../support -LLT_release = $(BUILDDIR)/$(LLTDIR)/libsupport.a -LLT_debug = $(BUILDDIR)/$(LLTDIR)/libsupport-debug.a -LIBFILES_release = $(LLT_release) $(LIBUV) $(LIBUTF8PROC) -LIBFILES_debug = $(LLT_debug) $(LIBUV) $(LIBUTF8PROC) -LIBS = +OBJS := $(SRCS:%.c=$(BUILDDIR)/%.o) +DOBJS := $(SRCS:%.c=$(BUILDDIR)/%.dbg.obj) +LLTDIR := ../support +LLT_release := $(BUILDDIR)/$(LLTDIR)/libsupport.a +LLT_debug := $(BUILDDIR)/$(LLTDIR)/libsupport-debug.a +LIBFILES_release := $(LLT_release) $(LIBUV) $(LIBUTF8PROC) +LIBFILES_debug := $(LLT_debug) $(LIBUV) $(LIBUTF8PROC) +LIBS := ifneq ($(OS),WINNT) LIBS += -lpthread endif -FLAGS = -I$(LLTDIR) $(CFLAGS) $(HFILEDIRS:%=-I%) \ +FLAGS := -I$(LLTDIR) $(CFLAGS) $(HFILEDIRS:%=-I%) \ -I$(LIBUV_INC) -I$(build_includedir) $(LIBDIRS:%=-L%) \ -DLIBRARY_EXPORTS -DUTF8PROC_EXPORTS ifneq ($(USEMSVC), 1) @@ -74,9 +74,9 @@ $(BUILDDIR)/$(LIBTARGET).a: $(OBJS) | $(BUILDDIR) @$(call PRINT_LINK, $(AR) -rcs $@ $(OBJS)) ifneq ($(USEMSVC), 1) -CCLD = $(CC) +CCLD := $(CC) else -CCLD = $(LD) +CCLD := $(LD) endif $(BUILDDIR)/$(EXENAME)-debug: $(DOBJS) $(LIBFILES_debug) $(BUILDDIR)/$(LIBTARGET)-debug.a $(BUILDDIR)/flmain.dbg.obj diff --git a/src/support/Makefile b/src/support/Makefile index c03767c2dbafe..cf67fccd9270e 100644 --- a/src/support/Makefile +++ b/src/support/Makefile @@ -6,7 +6,7 @@ override CFLAGS += $(JCFLAGS) override CXXFLAGS += $(JCXXFLAGS) override CPPFLAGS += $(JCPPFLAGS) -SRCS = hashing timefuncs ptrhash operators \ +SRCS := hashing timefuncs ptrhash operators \ utf8 ios htable bitvector \ int2str libsupportinit arraylist strtod ifeq ($(OS),WINNT) @@ -20,12 +20,12 @@ SRCS += _setjmp.win64 _longjmp.win64 endif endif -HEADERS = $(wildcard *.h) $(LIBUV_INC)/uv.h +HEADERS := $(wildcard *.h) $(LIBUV_INC)/uv.h -OBJS = $(SRCS:%=$(BUILDDIR)/%.o) -DOBJS = $(SRCS:%=$(BUILDDIR)/%.dbg.obj) +OBJS := $(SRCS:%=$(BUILDDIR)/%.o) +DOBJS := $(SRCS:%=$(BUILDDIR)/%.dbg.obj) -FLAGS = $(CFLAGS) $(HFILEDIRS:%=-I%) -I$(LIBUV_INC) -I$(UTF8PROC_INC) -DLIBRARY_EXPORTS -DUTF8PROC_EXPORTS +FLAGS := $(CFLAGS) $(HFILEDIRS:%=-I%) -I$(LIBUV_INC) -I$(UTF8PROC_INC) -DLIBRARY_EXPORTS -DUTF8PROC_EXPORTS ifneq ($(USEMSVC), 1) FLAGS += -Wall -Wno-strict-aliasing -fvisibility=hidden endif diff --git a/ui/Makefile b/ui/Makefile index aaa6c904bbb97..4879ec14a6639 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -8,28 +8,28 @@ override CFLAGS += $(JCFLAGS) override CXXFLAGS += $(JCXXFLAGS) override CPPFLAGS += $(JCPPFLAGS) -SRCS = repl +SRCS := repl ifeq ($(USEMSVC), 1) SRCS += getopt endif -FLAGS = -I$(JULIAHOME)/src -I$(JULIAHOME)/src/support -I$(build_includedir) +FLAGS := -I$(JULIAHOME)/src -I$(JULIAHOME)/src/support -I$(build_includedir) ifneq ($(USEMSVC), 1) FLAGS += -Wall -Wno-strict-aliasing -fno-omit-frame-pointer endif -OBJS = $(SRCS:%=$(BUILDDIR)/%.o) -DOBJS = $(SRCS:%=$(BUILDDIR)/%.dbg.obj) +OBJS := $(SRCS:%=$(BUILDDIR)/%.o) +DOBJS := $(SRCS:%=$(BUILDDIR)/%.dbg.obj) DEBUGFLAGS += $(FLAGS) SHIPFLAGS += $(FLAGS) ifeq ($(USE_LLVM_SHLIB),1) ifeq ($(LLVM_USE_CMAKE),1) -LLVMLINK = $(call exec,$(LLVM_CONFIG) --ldflags) -lLLVM +LLVMLINK := $(call exec,$(LLVM_CONFIG) --ldflags) -lLLVM else -LLVMLINK = $(call exec,$(LLVM_CONFIG) --ldflags) -lLLVM-$(call exec,$(LLVM_CONFIG) --version) +LLVMLINK := $(call exec,$(LLVM_CONFIG) --ldflags) -lLLVM-$(call exec,$(LLVM_CONFIG) --version) endif else -LLVMLINK = +LLVMLINK := endif JLDFLAGS += $(LDFLAGS) $(NO_WHOLE_ARCHIVE) $(OSLIBS) $(LLVMLINK) $(RPATH) @@ -69,9 +69,9 @@ julia-release: $(build_bindir)/julia$(EXE) julia-debug: $(build_bindir)/julia-debug$(EXE) ifneq ($(USEMSVC), 1) -CXXLD = $(CXX) +CXXLD := $(CXX) else -CXXLD = $(LD) +CXXLD := $(LD) endif $(build_bindir)/julia$(EXE): $(OBJS) From ffae404e46787d0afd813c64e4f09fbc85898dcb Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 8 Aug 2015 16:34:15 -0400 Subject: [PATCH 0055/1938] fix error in expansion order of $(HOSTCC) = $(CC) --- Make.inc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Make.inc b/Make.inc index f2b62b7be76c6..e3adbcdf70f6f 100644 --- a/Make.inc +++ b/Make.inc @@ -189,7 +189,8 @@ endif ifeq ($(XC_HOST),) CROSS_COMPILE:= -HOSTCC := $(CC) +# delayed expansion of $(CC), since it won't be computed until later +HOSTCC = $(CC) else HOSTCC := gcc override OPENBLAS_DYNAMIC_ARCH := 1 From 05c573eef036908810de3c9c1e4441806d15873b Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 8 Aug 2015 15:05:39 -0400 Subject: [PATCH 0056/1938] add make configure target to setup out-of-tree build directory, and add mention in README --- Makefile | 37 +++++++++++++++++++++++++++---------- README.md | 4 +++- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 3be22db4b1244..4982eed8f3100 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ all: debug release # sort is used to remove potential duplicates DIRS := $(sort $(build_bindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_man1dir)) ifneq ($(BUILDROOT),$(JULIAHOME)) -BUILDDIRS := $(addprefix $(BUILDROOT)/,. base src ui doc deps test test/perf) +BUILDDIRS := $(abspath $(addprefix $(BUILDROOT)/,. base src ui doc deps test test/perf)) BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS)) DIRS := $(DIRS) $(BUILDDIRS) $(BUILDDIRMAKE): | $(BUILDDIRS) @@ -30,6 +30,22 @@ $(BUILDDIRMAKE): | $(BUILDDIRS) @echo 'BUILDROOT=$(BUILDROOT)' >> $@ @echo 'include $(JULIAHOME)/$(patsubst $(BUILDROOT)/%,%,$(@:/Makefile=))/Makefile' >> $@ julia-deps: | $(BUILDDIRMAKE) +configure-y: | $(BUILDDIRMAKE) +configure: +ifeq ("$(origin O)", "command line") + @if [ "$$(ls '$(BUILDROOT)' 2> /dev/null)" ]; then \ + echo 'WARNING: configure called on non-empty directory $(BUILDROOT)'; \ + read -p "Proceed [y/n]? " answer; \ + else \ + answer=y;\ + fi; \ + [ $$answer = 'y' ] && $(MAKE) configure-$$answer +else + $(error "cannot rerun configure from within a build directory") +endif +else +configure: + $(error "must specify O=builddir to run the Julia `make configure` target") endif $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) @@ -124,15 +140,16 @@ release-candidate: release testall @echo @echo 1. Remove deprecations in base/deprecated.jl - @echo 2. Bump VERSION - @echo 3. Create tag, push to github "\(git tag v\`cat VERSION\` && git push --tags\)" #"` # These comments deal with incompetent syntax highlighting rules - @echo 4. Clean out old .tar.gz files living in deps/, "\`git clean -fdx\`" seems to work #"` - @echo 5. Replace github release tarball with tarballs created from make light-source-dist and make full-source-dist - @echo 6. Follow packaging instructions in DISTRIBUTING.md to create binary packages for all platforms - @echo 7. Upload to AWS, update http://julialang.org/downloads and http://status.julialang.org/stable links - @echo 8. Update checksums on AWS for tarball and packaged binaries - @echo 9. Announce on mailing lists - @echo 10. Change master to release-0.X in base/version.jl and base/version_git.sh as in 4cb1e20 + @echo 2. Update references to the julia version in the source directories, such as in README.md + @echo 3. Bump VERSION + @echo 4. Create tag, push to github "\(git tag v\`cat VERSION\` && git push --tags\)" #"` # These comments deal with incompetent syntax highlighting rules + @echo 5. Clean out old .tar.gz files living in deps/, "\`git clean -fdx\`" seems to work #"` + @echo 6. Replace github release tarball with tarballs created from make light-source-dist and make full-source-dist + @echo 7. Follow packaging instructions in DISTRIBUTING.md to create binary packages for all platforms + @echo 8. Upload to AWS, update http://julialang.org/downloads and http://status.julialang.org/stable links + @echo 9. Update checksums on AWS for tarball and packaged binaries + @echo 10. Announce on mailing lists + @echo 11. Change master to release-0.X in base/version.jl and base/version_git.sh as in 4cb1e20 @echo $(build_man1dir)/julia.1: $(JULIAHOME)/doc/man/julia.1 | $(build_man1dir) diff --git a/README.md b/README.md index 908bddff2b048..fd7469d95d446 100644 --- a/README.md +++ b/README.md @@ -77,9 +77,11 @@ When compiled the first time, it will automatically download and build its [exte This takes a while, but only has to be done once. If the defaults in the build do not work for you, and you need to set specific make parameters, you can save them in `Make.user`. The build will automatically check for the existence of `Make.user` and use it if it exists. Building Julia requires 1.5GiB of disk space and approximately 700MiB of virtual memory. +For release-0.4 and newer builds of Julia, you can create out-of-tree builds of Julia by specifying `make O= configure` on the command line. This will create a directory mirror, with all of the necessary Makefiles to build Julia, in the specified directory. These builds will share the source files in Julia and `deps/srccache`. Each out-of-tree build directory can have its own `Make.user` file to override the global `Make.user` file in the toplevel folder. + If you need to build Julia in an environment that does not allow access to the outside world, use `make -C deps getall` to download all the necessary files. Then, copy the `julia` directory over to the target environment and build with `make`. -**Note:** the build process will not work if any of the build directory's parent directories have spaces in their names (this is due to a limitation in GNU make). +**Note:** the build process fail badly if any of the build directory's parent directories have spaces or other shell meta-characters such as `$` or `:` in their names (this is due to a limitation in GNU make). Once it is built, you can run the `julia` executable using its full path in the directory created above (the `julia` directory), or, to run it from anywhere, either From 72d3f2299e852b9dd05dbcfa2e052ce2f2dd97ea Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 8 Aug 2015 15:59:36 -0400 Subject: [PATCH 0057/1938] use an emulated llvm-config in 3.3 and before llvm-config-host describes the build target, not the host target on llvm <= 3.3, so we must use the (emulated) llvm-config binary. on newer versions of llvm, however, we can avoid the need to call target code and use llvm-config-host directly --- Make.inc | 31 ++++++++++++++++++++++--------- Makefile | 8 ++++---- src/Makefile | 12 ++++++------ ui/Makefile | 4 ++-- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/Make.inc b/Make.inc index e3adbcdf70f6f..22877ab97df01 100644 --- a/Make.inc +++ b/Make.inc @@ -229,11 +229,11 @@ endif ifeq ($(BUILD_OS), WINNT) PATH := $(PATH):$(build_libdir):$(build_private_libdir):/c/Program Files/7-zip:$(JULIAHOME)/dist-extras -HOST_EXE := .exe +BUILD_EXE := .exe else ifneq (,$(findstring CYGWIN,$(BUILD_OS))) -HOST_EXE := .exe +BUILD_EXE := .exe else -HOST_EXE := +BUILD_EXE := endif ifeq ($(OS), WINNT) fPIC := @@ -607,16 +607,29 @@ endif endif ifeq ($(USE_SYSTEM_LLVM), 1) -LLVM_CONFIG ?= llvm-config$(HOST_EXE) JCPPFLAGS+=-DSYSTEM_LLVM -LLVM_VER := $(shell $(LLVM_CONFIG) --version) -else -ifeq ($(BUILD_OS),$(OS)) -LLVM_CONFIG := $(build_bindir)/llvm-config$(HOST_EXE) +endif + +ifeq ($(origin LLVM_CONFIG), undefined) +ifeq ($(USE_SYSTEM_LLVM), 1) +LLVM_CONFIG := llvm-config$(EXE) else -LLVM_CONFIG := $(build_bindir)/llvm-config-host$(HOST_EXE) +LLVM_CONFIG := $(build_bindir)/llvm-config$(EXE) +endif +endif # LLVM_CONFIG undefined + +ifneq ($(BUILD_OS),$(OS)) +LLVM_CONFIG_HOST := $(basename $(LLVM_CONFIG))-host$(BUILD_EXE) +ifeq (exists, $$(shell [ -d '$(LLVM_CONFIG_HOST)' ] && echo exists )) +ifeq ($(shell $(LLVM_CONFIG_HOST) --version),3.3) +# llvm-config-host <= 3.3 is broken, use llvm-config instead (in an emulator) +# use delayed expansion (= not :=) because spawn isn't defined until later +LLVM_CONFIG_HOST = $(call spawn,$(LLVM_CONFIG)) endif endif +else +LLVM_CONFIG_HOST := $(LLVM_CONFIG) +endif # SYSTEM_LLVM ifeq ($(USE_SYSTEM_PCRE), 1) PCRE_CONFIG := pcre2-config diff --git a/Makefile b/Makefile index 4982eed8f3100..154f42acf705b 100644 --- a/Makefile +++ b/Makefile @@ -21,14 +21,14 @@ all: debug release # sort is used to remove potential duplicates DIRS := $(sort $(build_bindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_man1dir)) ifneq ($(BUILDROOT),$(JULIAHOME)) -BUILDDIRS := $(abspath $(addprefix $(BUILDROOT)/,. base src ui doc deps test test/perf)) +BUILDDIRS := $(BUILDROOT) $(addprefix $(BUILDROOT)/,base src ui doc deps test test/perf) BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS)) DIRS := $(DIRS) $(BUILDDIRS) $(BUILDDIRMAKE): | $(BUILDDIRS) @# add Makefiles to the build directories for convenience (pointing back to the source location of each) @echo '# -- This file is automatically generated in julia/Makefile -- #' > $@ @echo 'BUILDROOT=$(BUILDROOT)' >> $@ - @echo 'include $(JULIAHOME)/$(patsubst $(BUILDROOT)/%,%,$(@:/Makefile=))/Makefile' >> $@ + @echo 'include $(JULIAHOME)$(patsubst $(BUILDROOT)%,%,$@)' >> $@ julia-deps: | $(BUILDDIRMAKE) configure-y: | $(BUILDDIRMAKE) configure: @@ -62,8 +62,8 @@ clean-docdir: $(subst $(abspath $(BUILDROOT))/,,$(abspath $(build_docdir))): $(build_docdir) $(build_docdir): @mkdir -p $@/examples - @cp -R examples/*.jl $@/examples/ - @cp -R examples/clustermanager $@/examples/ + @cp -R $(JULIAHOME)/examples/*.jl $@/examples/ + @cp -R $(JULIAHOME)/examples/clustermanager $@/examples/ julia-symlink: julia-ui-$(JULIA_BUILD_MODE) ifneq ($(OS),WINNT) diff --git a/src/Makefile b/src/Makefile index d326e9e230624..b92500096a226 100644 --- a/src/Makefile +++ b/src/Makefile @@ -28,7 +28,7 @@ HEADERS := $(addprefix $(SRCDIR)/,julia.h julia_internal.h options.h) $(BUILDDIR FLAGS := \ -D_GNU_SOURCE -I$(BUILDDIR) -I$(SRCDIR) \ -I$(SRCDIR)/flisp -I$(SRCDIR)/support \ - -I$(shell $(LLVM_CONFIG) --includedir) \ + -I$(shell $(LLVM_CONFIG_HOST) --includedir) \ -I$(LIBUV_INC) -I$(build_includedir) -DLIBRARY_EXPORTS \ -I$(JULIAHOME)/deps/valgrind ifneq ($(USEMSVC), 1) @@ -37,12 +37,12 @@ endif # In LLVM < 3.4, --ldflags includes both options and libraries, so use it both before and after --libs # In LLVM >= 3.4, --ldflags has only options, and --system-libs has the libraries. -LLVMLINK := $(shell $(LLVM_CONFIG) --ldflags) $(shell $(LLVM_CONFIG) --libs) $(shell $(LLVM_CONFIG) --ldflags) $(shell $(LLVM_CONFIG) --system-libs 2> /dev/null) +LLVMLINK := $(shell $(LLVM_CONFIG_HOST) --ldflags) $(shell $(LLVM_CONFIG_HOST) --libs) $(shell $(LLVM_CONFIG_HOST) --ldflags) $(shell $(LLVM_CONFIG_HOST) --system-libs 2> /dev/null) ifeq ($(USE_LLVM_SHLIB),1) ifeq ($(LLVM_USE_CMAKE),1) -LLVMLINK := $(shell $(LLVM_CONFIG) --ldflags) -lLLVM +LLVMLINK := $(shell $(LLVM_CONFIG_HOST) --ldflags) -lLLVM else -LLVMLINK := $(shell $(LLVM_CONFIG) --ldflags) -lLLVM-$(shell $(LLVM_CONFIG) --version) +LLVMLINK := $(shell $(LLVM_CONFIG_HOST) --ldflags) -lLLVM-$(shell $(LLVM_CONFIG_HOST) --version) endif FLAGS += -DLLVM_SHLIB endif @@ -83,9 +83,9 @@ $(BUILDDIR)/%.o: $(SRCDIR)/%.c $(HEADERS) | $(BUILDDIR) $(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.c $(HEADERS) | $(BUILDDIR) @$(call PRINT_CC, $(CC) $(CPPFLAGS) $(CFLAGS) $(DEBUGFLAGS) -c $< -o $@) $(BUILDDIR)/%.o: $(SRCDIR)/%.cpp $(HEADERS) $(LLVM_CONFIG_ABSOLUTE) | $(BUILDDIR) - @$(call PRINT_CC, $(CXX) $(shell $(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(SHIPFLAGS) -c $< -o $@) + @$(call PRINT_CC, $(CXX) $(shell $(LLVM_CONFIG_HOST) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(SHIPFLAGS) -c $< -o $@) $(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.cpp $(HEADERS) $(LLVM_CONFIG_ABSOLUTE) | $(BUILDDIR) - @$(call PRINT_CC, $(CXX) $(shell $(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(DEBUGFLAGS) -c $< -o $@) + @$(call PRINT_CC, $(CXX) $(shell $(LLVM_CONFIG_HOST) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(DEBUGFLAGS) -c $< -o $@) $(BUILDDIR)/julia_flisp.boot.inc: $(BUILDDIR)/julia_flisp.boot $(FLISP_EXECUTABLE) @$(call PRINT_FLISP, $(call spawn,$(FLISP_EXECUTABLE)) $(call cygpath_w,$(SRCDIR)/bin2hex.scm) < $< > $@) diff --git a/ui/Makefile b/ui/Makefile index 4879ec14a6639..3d3c207b83481 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -24,9 +24,9 @@ DEBUGFLAGS += $(FLAGS) SHIPFLAGS += $(FLAGS) ifeq ($(USE_LLVM_SHLIB),1) ifeq ($(LLVM_USE_CMAKE),1) -LLVMLINK := $(call exec,$(LLVM_CONFIG) --ldflags) -lLLVM +LLVMLINK := $(shell $(LLVM_CONFIG_HOST) --ldflags) -lLLVM else -LLVMLINK := $(call exec,$(LLVM_CONFIG) --ldflags) -lLLVM-$(call exec,$(LLVM_CONFIG) --version) +LLVMLINK := $(shell $(LLVM_CONFIG_HOST) --ldflags) -lLLVM-$(shell $(LLVM_CONFIG_HOST) --version) endif else LLVMLINK := From 57505511393bc3e49bbc917105c74e1cea298e31 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 9 Aug 2015 16:52:59 -0400 Subject: [PATCH 0058/1938] always segregate deps build files from the source files and the legacy builds --- deps/.gitignore | 4 +++- deps/Makefile | 4 ++++ doc/Makefile | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/deps/.gitignore b/deps/.gitignore index c4a02377c0308..89ab4cc8ca36d 100644 --- a/deps/.gitignore +++ b/deps/.gitignore @@ -1,3 +1,6 @@ +/srccache +/build + /arpack-* /cfe-* /clang-* @@ -31,7 +34,6 @@ /Rmath-julia* # git-externals: -/srccache /libgit2 /libgit2-* /libuv diff --git a/deps/Makefile b/deps/Makefile index 0d8c66a846087..8b83f26caf9b3 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -1,7 +1,11 @@ ## high-level setup ## SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) JULIAHOME := $(abspath $(SRCDIR)/..) +ifeq ($(abspath .),$(abspath $(SRCDIR))) +BUILDDIR := build +else BUILDDIR := . +endif include $(SRCDIR)/Versions.make include $(JULIAHOME)/Make.inc diff --git a/doc/Makefile b/doc/Makefile index e7e83b23f1bd4..df4fd75fccfb9 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -17,7 +17,11 @@ ALLSPHINXOPTS := -d $(BUILDDIR)/_build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINX # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS := $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +ifeq ($(abspath .),$(abspath $(SRCDIR))) +JULIA_ENV := $(BUILDDIR)/../deps/build/julia-env +else JULIA_ENV := $(BUILDDIR)/../deps/julia-env +endif ACTIVATE := $(JULIA_ENV)/bin/activate SPHINX_BUILD := $(JULIA_ENV)/bin/sphinx-build From ffdedd34e3df977aee978154a1979b61c9f877c4 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 9 Aug 2015 21:30:49 -0400 Subject: [PATCH 0059/1938] revert libgit2 build to using $(MAKE) $(CMAKE) with recursive makefiles would require a + prefix to pass the right environment, but that's too cmake-target specific to be useful for generalization of the cmake usage --- deps/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index 8b83f26caf9b3..588e3b183c51d 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -2017,11 +2017,11 @@ $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/C $(CMAKE) $(dir $<) $(LIBGIT2_OPTS) touch -c $@ $(LIBGIT2_OBJ_SOURCE): $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile - $(CMAKE) --build $(dir $<) + $(MAKE) -C $(dir $<) touch -c $@ -$(LIBGIT2_SRC_DIR)/checked: $(LIBGIT2_OBJ_SOURCE) +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/checked: $(LIBGIT2_OBJ_SOURCE) ifeq ($(OS),$(BUILD_OS)) - $(CMAKE) --build $(dir $@) --target test + $(MAKE) -C $(dir $@) test endif echo 1 > $@ $(LIBGIT2_OBJ_TARGET): $(LIBGIT2_OBJ_SOURCE) | $(build_shlibdir) From f7f119e6e59f67df16dcd856a23eb70c5af89eb5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 9 Aug 2015 21:56:49 -0400 Subject: [PATCH 0060/1938] also delete build files as part of the deps distclean targets --- deps/Makefile | 65 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index 588e3b183c51d..f966c7ce4ac14 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -310,7 +310,7 @@ $5/$$($2_SRC_DIR)/$3: $$($2_SRC_FILE) endif # NO_GIT distclean-$1: - -rm -rf $5/$$($2_SRC_DIR) $$($2_SRC_FILE) + -rm -rf $5/$$($2_SRC_DIR) $$($2_SRC_FILE) $$(BUILDDIR)/$1 endef @@ -744,7 +744,7 @@ clean-llvm: distclean-llvm: -rm -rf $(LLVM_TAR) $(LLVM_CLANG_TAR) \ $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR) \ - $(LLVM_SRC_DIR) + $(LLVM_SRC_DIR) $(LLVM_BUILDDIR_withtype) ifneq ($(LLVM_VER),svn) get-llvm: $(LLVM_TAR) $(LLVM_CLANG_TAR) $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR) @@ -866,8 +866,8 @@ $(PCRE_OBJ_TARGET): $(PCRE_SRC_TARGET) $(BUILDDIR)/pcre2-$(PCRE_VER)/checked clean-pcre: -$(MAKE) -C pcre2-$(PCRE_VER) clean -rm -f $(build_shlibdir)/libpcre* -distclean-pcre: clean-pcre - -rm -rf $(SRCDIR)/srccache/pcre2-$(PCRE_VER).tar.bz2 $(SRCDIR)/srccache/pcre2-$(PCRE_VER) +distclean-pcre: + -rm -rf $(SRCDIR)/srccache/pcre2-$(PCRE_VER).tar.bz2 $(SRCDIR)/srccache/pcre2-$(PCRE_VER) $(BUILDDIR)/pcre2-$(PCRE_VER) get-pcre: $(SRCDIR)/srccache/pcre2-$(PCRE_VER).tar.bz2 configure-pcre: $(BUILDDIR)/pcre2-$(PCRE_VER)/config.status @@ -985,7 +985,7 @@ $(build_shlibdir)/libdSFMT%$(SHLIB_EXT) $(build_includedir)/dSFMT%h: $(DSFMT_OBJ clean-dsfmt: -rm -f $(BUILDDIR)/dsfmt-$(DSFMT_VER)/libdSFMT.$(SHLIB_EXT) distclean-dsfmt: - -rm -rf $(SRCDIR)/srccache/dsfmt*.tar.gz $(SRCDIR)/srccache/dsfmt-$(DSFMT_VER) + -rm -rf $(SRCDIR)/srccache/dsfmt*.tar.gz $(SRCDIR)/srccache/dsfmt-$(DSFMT_VER) $(BUILDDIR)/dsfmt-$(DSFMT_VER) get-dsfmt: $(SRCDIR)/srccache/dsfmt-$(DSFMT_VER).tar.gz configure-dsfmt: $(BUILDDIR)/dsfmt-$(DSFMT_VER)/config.status @@ -1025,7 +1025,7 @@ $(RMATH_JULIA_OBJ_TARGET): $(RMATH_JULIA_OBJ_SOURCE) | $(build_shlibdir) clean-Rmath-julia: -$(MAKE) -C $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER) clean distclean-Rmath-julia: clean-Rmath-julia - -rm -rf $(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER).tar.gz $(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER) + -rm -rf $(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER).tar.gz $(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER) $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER) get-Rmath-julia: $(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER).tar.gz configure-Rmath-julia: $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER)/Makefile @@ -1202,11 +1202,12 @@ endif $(ATLAS_OBJ_TARGET): $(ATLAS_OBJ_SOURCE) cp -f $(ATLAS_OBJ_SOURCE) $@ $(INSTALL_NAME_CMD)libsatlas.$(SHLIB_EXT) $@ + touch -c $@ clean-atlas: - rm -rf atlas/build + rm -rf $(BUILDDIR)/atlas/build distclean-atlas: - rm -rf atlas + rm -rf $(BUILDDIR)/atlas get-atlas: $(SRCDIR)/srccache/atlas/configure $(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz configure-atlas: $(BUILDDIR)/atlas/Make.top @@ -1284,10 +1285,10 @@ $(LAPACK_OBJ_TARGET): $(LAPACK_OBJ_SOURCE) $(INSTALL_NAME_CMD)liblapack.$(SHLIB_EXT) $@ clean-lapack: - -$(MAKE) -C lapack-$(LAPACK_VER) clean + -$(MAKE) -C $(BUILDDIR)/lapack-$(LAPACK_VER) clean -rm -f $(LAPACK_OBJ_SOURCE) $(LAPACK_OBJ_TARGET) distclean-lapack: - -rm -rf lapack-$(LAPACK_VER).tgz lapack-$(LAPACK_VER) + -rm -rf $(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz $(BUILDDIR)/lapack-$(LAPACK_VER) get-lapack: $(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz configure-lapack: $(BUILDDIR)/lapack-$(LAPACK_VER)/make.inc @@ -1391,10 +1392,13 @@ $(ARPACK_OBJ_TARGET): $(PATCHELF) endif clean-arpack: - -$(MAKE) -C arpack-ng-$(ARPACK_VER) clean + -$(MAKE) -C $(BUILDDIR)/arpack-ng-$(ARPACK_VER) clean -rm -f $(ARPACK_OBJ_SOURCE) $(ARPACK_OBJ_TARGET) distclean-arpack: - -rm -rf arpack-ng-$(ARPACK_VER).tar.gz arpack-ng-$(ARPACK_VER) + -rm -rf $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER).tar.gz \ + $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER) \ + $(SRCDIR)/srccache/arpack-ng-3.2.0-testA.mtx \ + $(BUILDDIR)/arpack-ng-$(ARPACK_VER) get-arpack: $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER).tar.gz $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)-testA.mtx configure-arpack: $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/config.status @@ -1506,9 +1510,11 @@ clean-fftw-double: -$(MAKE) -C $(BUILDDIR)/fftw-$(FFTW_VER)-double clean -rm -f $(FFTW_DOUBLE_OBJ_TARGET) distclean-fftw: distclean-fftw-single distclean-fftw-double - -rm -rf $(SRCDIR)/srccache/fftw-$(FFTW_VER).tar.gz fftw-$(FFTW_VER)-single + -rm -rf $(SRCDIR)/srccache/fftw-$(FFTW_VER).tar.gz $(SRCDIR)/srccache/fftw-$(FFTW_VER) distclean-fftw-single: + -rm -rf $(BUILDDIR)/fftw-$(FFTW_VER)-single distclean-fftw-double: + -rm -rf $(BUILDDIR)/fftw-$(FFTW_VER)-double get-fftw: get-fftw-single get-fftw-double configure-fftw: configure-fftw-single configure-fftw-double @@ -1646,8 +1652,10 @@ $(SUITESPARSE_OBJ_TARGET): $(SUITESPARSE_OBJ_SOURCE) clean-suitesparse: -$(MAKE) -C $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER) clean -rm -fr $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/lib + -rm -f $(SUITESPARSE_OBJ_TARGET) distclean-suitesparse: - -rm -rf $(SRCDIR)/srccache/SuiteSparse-$(SUITESPARSE_VER).tar.gz $(SRCDIR)/srccache/SuiteSparse-$(SUITESPARSE_VER) + -rm -rf $(SRCDIR)/srccache/SuiteSparse-$(SUITESPARSE_VER).tar.gz \ + $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER) get-suitesparse: $(SRCDIR)/srccache/SuiteSparse-$(SUITESPARSE_VER).tar.gz configure-suitesparse: $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/Makefile @@ -1672,7 +1680,7 @@ $(build_shlibdir)/libsuitesparse_wrapper.$(SHLIB_EXT): $(SRCDIR)/SuiteSparse_wra touch -c $@ clean-suitesparse-wrapper: - -rm -f $(SUITESPARSE_OBJ_TARGET) $(build_shlibdir)/libsuitesparse_wrapper.$(SHLIB_EXT) + -rm -f $(build_shlibdir)/libsuitesparse_wrapper.$(SHLIB_EXT) distclean-suitesparse-wrapper: clean-suitesparse-wrapper get-suitesparse-wrapper: @@ -1722,7 +1730,9 @@ clean-unwind: -$(MAKE) -C $(BUILDDIR)/libunwind-$(UNWIND_VER) clean -rm -f $(LIBUNWIND_TARGET_OBJ) $(LIBUNWIND_TARGET_SOURCE) distclean-unwind: - -rm -rf $(SRCDIR)/srccache/libunwind-$(UNWIND_VER).tar.gz $(SRCDIR)/srccache/libunwind-$(UNWIND_VER) + -rm -rf $(SRCDIR)/srccache/libunwind-$(UNWIND_VER).tar.gz \ + $(SRCDIR)/srccache/libunwind-$(UNWIND_VER) \ + $(BUILDDIR)/libunwind-$(UNWIND_VER) get-unwind: $(SRCDIR)/srccache/libunwind-$(UNWIND_VER).tar.gz configure-unwind: $(BUILDDIR)/libunwind-$(UNWIND_VER)/config.status @@ -1758,7 +1768,9 @@ clean-osxunwind: -$(MAKE) -C $(BUILDDIR)/libosxunwind-$(OSXUNWIND_VER) clean $(OSXUNWIND_FLAGS) -rm $(OSXUNWIND_OBJ_TARGET) distclean-osxunwind: - -rm -rf $(SRCDIR)/srccache/libosxunwind-$(OSXUNWIND_VER).tar.gz + -rm -rf $(SRCDIR)/srccache/libosxunwind-$(OSXUNWIND_VER).tar.gz \ + $(BUILDDIR)/libosxunwind-$(OSXUNWIND_VER) + get-osxunwind: libosxunwind-$(OSXUNWIND_VER)/Makefile configure-osxunwind: get-osxunwind @@ -1807,7 +1819,9 @@ clean-gmp: -$(MAKE) -C $(BUILDDIR)/gmp-$(GMP_VER) clean -rm -f $(GMP_OBJ_TARGET) distclean-gmp: - -rm -rf $(SRCDIR)/srccache/gmp-$(GMP_VER).tar.bz2 $(SRCDIR)/srccache/gmp-$(GMP_VER) + -rm -rf $(SRCDIR)/srccache/gmp-$(GMP_VER).tar.bz2 \ + $(SRCDIR)/srccache/gmp-$(GMP_VER) \ + $(BUILDDIR)/gmp-$(GMP_VER) get-gmp: $(SRCDIR)/srccache/gmp-$(GMP_VER).tar.bz2 configure-gmp: $(BUILDDIR)/gmp-$(GMP_VER)/config.status @@ -1872,7 +1886,9 @@ clean-mpfr: -$(MAKE) -C $(BUILDDIR)/mpfr-$(MPFR_VER) clean -rm -f $(MPFR_OBJ_TARGET) distclean-mpfr: - -rm -rf $(SRCDIR)/srccache/mpfr-$(MPFR_VER).tar.bz2 $(SRCDIR)/srccache/mpfr-$(MPFR_VER) + -rm -rf $(SRCDIR)/srccache/mpfr-$(MPFR_VER).tar.bz2 \ + $(SRCDIR)/srccache/mpfr-$(MPFR_VER) \ + $(BUILDDIR)/mpfr-$(MPFR_VER) get-mpfr: $(SRCDIR)/srccache/mpfr-$(MPFR_VER).tar.bz2 configure-mpfr: $(BUILDDIR)/mpfr-$(MPFR_VER)/config.status @@ -1913,7 +1929,9 @@ clean-patchelf: -$(MAKE) -C $(BUILDDIR)/patchelf-$(PATCHELF_VER) clean -rm -f $(PATCHELF_OBJ_TARGET) distclean-patchelf: - -rm -rf $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER).tar.gz $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER) + -rm -rf $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER).tar.gz \ + $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER) \ + $(BUILDDIR)/patchelf-$(PATCHELF_VER) get-patchelf: $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER).tar.gz configure-patchelf: $(BUILDDIR)/patchelf-$(PATCHELF_VER)/config.status @@ -1950,7 +1968,8 @@ clean-git: -$(MAKE) -C $(BUILDDIR)/git-$(GIT_VER) clean -rm -f $(GIT_OBJ_TARGET) distclean-git: - -rm -rf $(SRCDIR)/srccache/git-$(GIT_VER).tar.gz $(SRCDIR)/srccache/git-$(GIT_VER) + -rm -rf $(SRCDIR)/srccache/git-$(GIT_VER).tar.gz \ + $(BUILDDIR)/git-$(GIT_VER) get-git: $(SRCDIR)/srccache/git-$(GIT_VER).tar.gz configure-git: $(BUILDDIR)/git-$(GIT_VER)/config.status @@ -1977,10 +1996,10 @@ endif touch -c $@ clean-virtualenv: - -$(MAKE) -C $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER) clean -rm -rf $(VIRTUALENV_TARGET) distclean-virtualenv: clean-virtualenv - -rm -rf $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER).tar.gz $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER) + -rm -rf $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER).tar.gz \ + $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER) get-virtualenv: $(SRCDIR)/srccache/virtualenv-$(VIRTUALENV_VER).tar.gz configure-virtualenv: get-virtualenv From 3875ec6ad9627ebe7c888f3034e9c6d7810c7f93 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 9 Aug 2015 23:02:43 -0400 Subject: [PATCH 0061/1938] perform basic validation of the user paths to ensure they don't contain shell metacharacters --- Make.inc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Make.inc b/Make.inc index 22877ab97df01..60e299292bc41 100644 --- a/Make.inc +++ b/Make.inc @@ -83,6 +83,24 @@ else endif endif export BUILDROOT +unexport O + +# Make sure the user didn't try to specify a path that will confuse the shell / make +METACHARACTERS := ][?*{}() $$%:;&|!\#,\\`\": +ifneq (,$(findstring ',$(value BUILDROOT))) +$(error cowardly refusing to build into directory with a single-quote in the path) +endif +ifneq (,$(findstring ',$(value JULIAHOME))) +$(error cowardly refusing to build from source directory with a single-quote in the path) +endif +ifneq (,$(shell echo '$(value BUILDROOT)' | grep '[$(METACHARACTERS)]')) +$(error cowardly refusing to build into directory with a shell-metacharacter in the path\ + (got: $(value BUILDROOT))) +endif +ifneq (,$(shell echo '$(value JULIAHOME)' | grep '[$(METACHARACTERS)]')) +$(error cowardly refusing to build from source directory with a shell-metacharacter in the path\ + (got: $(value JULIAHOME))) +endif # we include twice to pickup user definitions better # include from JULIAHOME first so that BUILDROOT can override From 5e530b495f2d7268443531cba426f3136fd595d6 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 9 Aug 2015 23:03:26 -0400 Subject: [PATCH 0062/1938] add a makefile trick for printing any makefile variable via `make print-VAR` --- Make.inc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Make.inc b/Make.inc index 60e299292bc41..29bba25cc58e9 100644 --- a/Make.inc +++ b/Make.inc @@ -1019,3 +1019,8 @@ PRINT_FLISP = echo '$(subst ','\'',$(1))'; $(1) PRINT_JULIA = echo '$(subst ','\'',$(1))'; $(1) endif + +# Makefile debugging trick: +# call print-VARIABLE to see the runtime value of any variable +print-%: + @echo '$*=$($*)' From 7545aa9d07b678351de81cf297f9c325087923f7 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 9 Aug 2015 23:39:16 -0400 Subject: [PATCH 0063/1938] fix repackage_system_suitesparse4.make file to accommodate verification that no shell metacharacters are present in the definition of JULIAHOME --- contrib/repackage_system_suitesparse4.make | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/repackage_system_suitesparse4.make b/contrib/repackage_system_suitesparse4.make index 08f80f164dd84..cb39df7a46060 100755 --- a/contrib/repackage_system_suitesparse4.make +++ b/contrib/repackage_system_suitesparse4.make @@ -1,6 +1,6 @@ #!/usr/bin/make -f -JULIAHOME = $(abspath ..) +JULIAHOME := $(abspath ..) include $(JULIAHOME)/Make.inc all: default @@ -22,8 +22,8 @@ endif default: mkdir -p $(build_libdir) - mkdir -p $(JULIAHOME)/deps/SuiteSparse-SYSTEM/lib - cd $(JULIAHOME)/deps/SuiteSparse-SYSTEM/lib && \ + mkdir -p $(JULIAHOME)/deps/build/SuiteSparse-SYSTEM/lib + cd $(JULIAHOME)/deps/build/SuiteSparse-SYSTEM/lib && \ rm -f $(build_libdir)/lib{amd,cholmod,colamd,spqr,umfpack}.$(SHLIB_EXT) $(CC) -shared $(WHOLE_ARCHIVE) $(SS_LIB)/libsuitesparseconfig.a $(NO_WHOLE_ARCHIVE) -o $(build_libdir)/libsuitesparseconfig.$(SHLIB_EXT) $(INSTALL_NAME_CMD)libsuitesparseconfig.$(SHLIB_EXT) $(build_libdir)/libsuitesparseconfig.$(SHLIB_EXT) From 0c491e58cb9469941e1e2353e895b903abc786d4 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 10 Aug 2015 02:01:11 -0400 Subject: [PATCH 0064/1938] fixup arpack tests to support 64-bit blas --- deps/Makefile | 15 ++- deps/arpack-tests-blasint.patch | 158 ++++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 deps/arpack-tests-blasint.patch diff --git a/deps/Makefile b/deps/Makefile index f966c7ce4ac14..d177d7c00775c 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -1302,6 +1302,7 @@ ARPACK_FFLAGS := $(GFORTBLAS_FFLAGS) ARPACK_CFLAGS := ifeq ($(USE_BLAS64), 1) +ARPACK_CFLAGS += -DBLASINT=int64_t ifeq ($(USEIFC),1) ARPACK_FFLAGS += -i8 else @@ -1309,10 +1310,10 @@ ARPACK_FFLAGS += -fdefault-integer-8 ifeq ($(USE_SYSTEM_BLAS), 0) ifeq ($(OPENBLAS_SYMBOLSUFFIX), 64_) ARPACK_FFLAGS += -cpp -ffixed-line-length-none -ARPACK_OPENBLASFCNS1 := axpy copy gemv geqr2 lacpy lahqr lanhs larnv lartg lascl laset scal trevc trmm trsen +ARPACK_OPENBLASFCNS1 := axpy copy gemv geqr2 lacpy lahqr lanhs larnv lartg lascl laset scal trevc trmm trsen gbmv gbtrf gbtrs gttrf gttrs pttrf pttrs ARPACK_OPENBLASFCNS2 := dot ger labad laev2 lamch lanst lanv2 lapy2 larf larfg lasr nrm2 orm2r rot steqr swap ARPACK_OPENBLASFCNS3 := dotc geru unm2r -ARPACK_OPENBLASFCNS4 := COPY LABAD LAMCH LANHS LANV2 LARFG ROT +ARPACK_OPENBLASFCNS4 := COPY LABAD LAMCH LANHS LANV2 LARFG ROT GEMV ARPACK_FFLAGS += $(foreach fcn, $(ARPACK_OPENBLASFCNS1) $(ARPACK_OPENBLASFCNS2), -Ds$(fcn)=s$(fcn)_64 -Dd$(fcn)=d$(fcn)_64) ARPACK_FFLAGS += $(foreach fcn, $(ARPACK_OPENBLASFCNS1) $(ARPACK_OPENBLASFCNS3), -Dc$(fcn)=c$(fcn)_64 -Dz$(fcn)=z$(fcn)_64) ARPACK_FFLAGS += $(foreach fcn, $(ARPACK_OPENBLASFCNS4), -DS$(fcn)=S$(fcn)_64 -DD$(fcn)=D$(fcn)_64) @@ -1360,7 +1361,10 @@ else ifeq ($(USE_SYSTEM_LAPACK), 0) $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/config.status: | $(LAPACK_OBJ_TARGET) endif -$(BUILDDIR)/arpack-ng-$(ARPACK_VER)/config.status: $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)/configure +$(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)/arpack-tests-blasint.patch-applied: $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)/configure + cd $(dir $@) && patch -p1 < $(SRCDIR)/arpack-tests-blasint.patch + echo 1 > $@ +$(BUILDDIR)/arpack-ng-$(ARPACK_VER)/config.status: $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)/configure $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER)/arpack-tests-blasint.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ $< $(CONFIGURE_COMMON) $(ARPACK_FLAGS) @@ -1375,7 +1379,10 @@ ifeq ($(OS),$(BUILD_OS)) $(MAKE) -C $(dir $@) check $(ARPACK_MFLAGS) endif echo 1 > $@ -$(ARPACK_OBJ_TARGET): $(ARPACK_OBJ_SOURCE) $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/checked | $(build_shlibdir) +ifneq ($(USE_BLAS64), 1) # XXX: one of the ARPACK tests fail with BLAS64 +$(ARPACK_OBJ_TARGET): $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/checked +endif +$(ARPACK_OBJ_TARGET): $(ARPACK_OBJ_SOURCE) | $(build_shlibdir) $(call make-install,arpack-ng-$(ARPACK_VER),$(ARPACK_MFLAGS)) ifeq ($(OS), WINNT) mv $(build_shlibdir)/libarpack-2.dll $@ diff --git a/deps/arpack-tests-blasint.patch b/deps/arpack-tests-blasint.patch new file mode 100644 index 0000000000000..915b03034798a --- /dev/null +++ b/deps/arpack-tests-blasint.patch @@ -0,0 +1,158 @@ +diff -urN arpack-ng-3.2.0/TESTS/bug_1315_double.c arpack-ng-3.2.0-BLASINT/TESTS/bug_1315_double.c +--- arpack-ng-3.2.0/TESTS/bug_1315_double.c 2014-11-14 09:47:06.000000000 -0500 ++++ arpack-ng-3.2.0-BLASINT/TESTS/bug_1315_double.c 2015-08-10 01:08:29.000000000 -0400 +@@ -9,15 +9,18 @@ + * This is not efficient since the problem is + * symmetric but is done to exhibit the bug. + * */ +- +-extern void dnaupd_(int *, char *, int *, char *, int *, +- double *, double *, int *, double *, +- int *, int *, int *, double *, +- double *, int *, int *); +- +-extern void dneupd_( int*, char*, int *, double *, double *, double *, int*, double *, +- double *, double *, char *, int *, char *, int *, double *, double *, int *, +- double *, int *, int *, int *, double *, double *, int *, int * ); ++#ifndef BLASINT ++#define BLASINT int ++#endif ++ ++extern void dnaupd_(BLASINT *, char *, BLASINT *, char *, BLASINT *, ++ double *, double *, BLASINT *, double *, ++ BLASINT *, BLASINT *, BLASINT *, double *, ++ double *, BLASINT *, BLASINT *); ++ ++extern void dneupd_( BLASINT*, char*, BLASINT *, double *, double *, double *, BLASINT*, double *, ++ double *, double *, char *, BLASINT *, char *, BLASINT *, double *, double *, BLASINT *, ++ double *, BLASINT *, BLASINT *, BLASINT *, double *, double *, BLASINT *, BLASINT * ); + + void matVec(double * x, double * y) { + int i; +@@ -26,26 +29,26 @@ + }; + + int main() { +- int ido = 0; ++ BLASINT ido = 0; + char bmat[] = "I"; +- int N = 1000; ++ BLASINT N = 1000; + char which[] = "LM"; +- int nev = 9; ++ BLASINT nev = 9; + double tol = 0; + double resid[N]; +- int ncv = 2*nev+1; ++ BLASINT ncv = 2*nev+1; + double V[ncv*N]; +- int ldv = N; +- int iparam[11]; +- int ipntr[14]; ++ BLASINT ldv = N; ++ BLASINT iparam[11]; ++ BLASINT ipntr[14]; + double workd[3*N]; +- int rvec = 1; ++ BLASINT rvec = 1; + char howmny[] = "A"; + double* dr = (double*) malloc((nev+1)*sizeof(double)); + double* di = (double*) malloc((nev+1)*sizeof(double)); +- int select[3*ncv]; ++ BLASINT select[3*ncv]; + double z[(N+1)*(nev+1)]; +- int ldz = N+1; ++ BLASINT ldz = N+1; + double sigmar=0; + double sigmai=0; + double workev[3*ncv]; +@@ -55,8 +58,8 @@ + double workl[3*(ncv*ncv) + 6*ncv]; + for (k=0; k < 3*(ncv*ncv) + 6*ncv; ++k ) + workl[k] = 0; +- int lworkl = 3*(ncv*ncv) + 6*ncv; +- int info = 0; ++ BLASINT lworkl = 3*(ncv*ncv) + 6*ncv; ++ BLASINT info = 0; + + iparam[0] = 1; + iparam[2] = 10*N; +diff -urN arpack-ng-3.2.0/TESTS/bug_1315_single.c arpack-ng-3.2.0-BLASINT/TESTS/bug_1315_single.c +--- arpack-ng-3.2.0/TESTS/bug_1315_single.c 2014-11-14 09:47:06.000000000 -0500 ++++ arpack-ng-3.2.0-BLASINT/TESTS/bug_1315_single.c 2015-08-10 01:29:58.000000000 -0400 +@@ -9,15 +9,18 @@ + * This is not efficient since the problem is + * symmetric but is done to exhibit the bug. + */ +- +-extern void snaupd_(int *, char *, int *, char *, int *, +- float *, float *, int *, float *, +- int *, int *, int *, float *, +- float *, int *, int *); +- +-extern void sneupd_( int*, char*, int *, float *, float *, float *, int*, float *, +- float *, float *, char *, int *, char *, int *, float *, float *, int *, +- float *, int *, int *, int *, float *, float *, int *, int * ); ++#ifndef BLASINT ++#define BLASINT int ++#endif ++ ++extern void snaupd_(BLASINT *, char *, BLASINT *, char *, BLASINT *, ++ float *, float *, BLASINT *, float *, ++ BLASINT *, BLASINT *, BLASINT *, float *, ++ float *, BLASINT *, BLASINT *); ++ ++extern void sneupd_( BLASINT*, char*, BLASINT *, float *, float *, float *, BLASINT*, float *, ++ float *, float *, char *, BLASINT *, char *, BLASINT *, float *, float *, BLASINT *, ++ float *, BLASINT *, BLASINT *, BLASINT *, float *, float *, BLASINT *, BLASINT * ); + + + void matVec(float * x, float * y) { +@@ -27,26 +30,26 @@ + }; + + int main() { +- int ido = 0; ++ BLASINT ido = 0; + char bmat[] = "I"; +- int N = 1000; ++ BLASINT N = 1000; + char which[] = "LM"; +- int nev = 9; ++ BLASINT nev = 9; + float tol = 0; + float resid[N]; +- int ncv = 2*nev+1; ++ BLASINT ncv = 2*nev+1; + float V[ncv*N]; +- int ldv = N; +- int iparam[11]; +- int ipntr[14]; ++ BLASINT ldv = N; ++ BLASINT iparam[11]; ++ BLASINT ipntr[14]; + float workd[3*N]; +- int rvec = 1; ++ BLASINT rvec = 1; + char howmny[] = "A"; + float* dr = (float*) malloc((nev+1)*sizeof(float)); + float* di = (float*) malloc((nev+1)*sizeof(float)); +- int select[3*ncv]; ++ BLASINT select[3*ncv]; + float z[(N+1)*(nev+1)]; +- int ldz = N+1; ++ BLASINT ldz = N+1; + float sigmar=0; + float sigmai=0; + float workev[3*ncv]; +@@ -56,8 +59,8 @@ + float workl[3*(ncv*ncv) + 6*ncv]; + for (k=0; k < 3*(ncv*ncv) + 6*ncv; ++k ) + workl[k] = 0; +- int lworkl = 3*(ncv*ncv) + 6*ncv; +- int info = 0; ++ BLASINT lworkl = 3*(ncv*ncv) + 6*ncv; ++ BLASINT info = 0; + + iparam[0] = 1; + iparam[2] = 10*N; From 1fbf962f9409a677113c92322ed170dec78c35a4 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 10 Aug 2015 15:54:35 -0400 Subject: [PATCH 0065/1938] disable arpack test entirely since it fails stochastically --- deps/Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index d177d7c00775c..7925dd0ac4841 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -1379,9 +1379,8 @@ ifeq ($(OS),$(BUILD_OS)) $(MAKE) -C $(dir $@) check $(ARPACK_MFLAGS) endif echo 1 > $@ -ifneq ($(USE_BLAS64), 1) # XXX: one of the ARPACK tests fail with BLAS64 -$(ARPACK_OBJ_TARGET): $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/checked -endif +# XXX: bug_1315 ARPACK tests fail stochastically +#$(ARPACK_OBJ_TARGET): $(BUILDDIR)/arpack-ng-$(ARPACK_VER)/checked $(ARPACK_OBJ_TARGET): $(ARPACK_OBJ_SOURCE) | $(build_shlibdir) $(call make-install,arpack-ng-$(ARPACK_VER),$(ARPACK_MFLAGS)) ifeq ($(OS), WINNT) From 15129c292c6e63dfd8ad7f71a1c51106bbc5e4b7 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 21 Jul 2015 04:33:13 -0400 Subject: [PATCH 0066/1938] fix makefile rule for copying examples to dest ref #8428 (comments). fix #12539 --- Makefile | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 154f42acf705b..45524e9355026 100644 --- a/Makefile +++ b/Makefile @@ -59,11 +59,13 @@ $(BUILDROOT)/doc/_build/html: CLEAN_TARGETS += clean-docdir clean-docdir: @-rm -fr $(abspath $(build_docdir)) -$(subst $(abspath $(BUILDROOT))/,,$(abspath $(build_docdir))): $(build_docdir) -$(build_docdir): - @mkdir -p $@/examples - @cp -R $(JULIAHOME)/examples/*.jl $@/examples/ - @cp -R $(JULIAHOME)/examples/clustermanager $@/examples/ +$(build_prefix)/.examples: $(wildcard $(JULIAHOME)/examples/*.jl) $(shell find $(JULIAHOME)/examples/clustermanager) + @echo Copying in usr/share/doc/julia/examples + @-rm -fr $(build_docdir)/examples + @mkdir -p $(build_docdir)/examples + @cp -R $(JULIAHOME)/examples/*.jl $(build_docdir)/examples/ + @cp -R $(JULIAHOME)/examples/clustermanager $(build_docdir)/examples/ + @echo 1 > $@ julia-symlink: julia-ui-$(JULIA_BUILD_MODE) ifneq ($(OS),WINNT) @@ -72,10 +74,10 @@ ifndef JULIA_VAGRANT_BUILD endif endif -julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test $(build_docdir) $(build_sysconfdir)/julia/juliarc.jl $(build_man1dir)/julia.1 +julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/deps -julia-base: julia-deps +julia-base: julia-deps $(build_sysconfdir)/julia/juliarc.jl $(build_man1dir)/julia.1 @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/base julia-libccalltest: @@ -87,7 +89,7 @@ julia-src-release julia-src-debug : julia-src-% : julia-deps julia-ui-release julia-ui-debug : julia-ui-% : julia-src-% @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/ui julia-$* -julia-inference : julia-base julia-ui-$(JULIA_BUILD_MODE) +julia-inference : julia-base julia-ui-$(JULIA_BUILD_MODE) $(build_prefix)/.examples @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) $(build_private_libdir)/inference.ji JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) julia-sysimg-release : julia-inference julia-ui-release @@ -153,10 +155,12 @@ release-candidate: release testall @echo $(build_man1dir)/julia.1: $(JULIAHOME)/doc/man/julia.1 | $(build_man1dir) + @echo Copying in usr/share/man/man1/julia.1 @mkdir -p $(build_man1dir) @cp $< $@ $(build_sysconfdir)/julia/juliarc.jl: $(JULIAHOME)/etc/juliarc.jl | $(build_sysconfdir)/julia + @echo Creating usr/etc/julia/juliarc.jl @cp $< $@ ifeq ($(OS), WINNT) @cat $(JULIAHOME)/contrib/windows/juliarc.jl >> $(build_sysconfdir)/julia/juliarc.jl From d7a0950cc4f27ac7bb88e0c48d4345a65a2938c0 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 10 Aug 2015 15:09:05 -0400 Subject: [PATCH 0067/1938] point ui build at build output directory first, to pickup and build configuration files --- ui/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/Makefile b/ui/Makefile index 3d3c207b83481..15aaca3d367b9 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -13,7 +13,7 @@ ifeq ($(USEMSVC), 1) SRCS += getopt endif -FLAGS := -I$(JULIAHOME)/src -I$(JULIAHOME)/src/support -I$(build_includedir) +FLAGS := -I$(BUILDROOT)/src -I$(JULIAHOME)/src -I$(JULIAHOME)/src/support -I$(build_includedir) ifneq ($(USEMSVC), 1) FLAGS += -Wall -Wno-strict-aliasing -fno-omit-frame-pointer endif From eb23feda0879e73a20bdd27aa0b1fa7a173a3215 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 14 Aug 2015 01:18:53 -0400 Subject: [PATCH 0068/1938] fix test wildcard rule --- test/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Makefile b/test/Makefile index 1da9f0242edfb..1290ab5c934f4 100644 --- a/test/Makefile +++ b/test/Makefile @@ -4,7 +4,7 @@ BUILDDIR := . include $(JULIAHOME)/Make.inc # TODO: this Makefile ignores BUILDDIR, except for computing JULIA_EXECUTABLE -TESTS = all linalg $(filter-out TestHelpers runtests testdefs,$(subst .jl,,$(wildcard *.jl linalg/*.jl))) +TESTS = all linalg $(filter-out TestHelpers runtests testdefs,$(patsubst $(SRCDIR)/%.jl,%,$(wildcard $(SRCDIR)/*.jl $(SRCDIR)/linalg/*.jl))) default: all From 67b050649e9547227ff8754aa8f18219b8db346f Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 16 Aug 2015 16:30:36 -0400 Subject: [PATCH 0069/1938] cleaup ARCH/MARCH/XC_HOST/BUILD_MACHINE detection logic so it works better & cross-compile works on Darwin --- Make.inc | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/Make.inc b/Make.inc index 29bba25cc58e9..ca86dd8643663 100644 --- a/Make.inc +++ b/Make.inc @@ -484,9 +484,18 @@ endif # if MARCH is set newer than the native processor, be forewarned that the compile might fail # JULIA_CPU_TARGET is the JIT-only complement to MARCH. Setting it explicitly is not generally necessary, # since it is set equal to MARCH by default + BUILD_MACHINE := $(shell $(HOSTCC) -dumpmachine) + ifeq ($(ARCH),) override ARCH := $(shell $(CC) -dumpmachine | sed "s/\([^-]*\).*$$/\1/") +else +ifneq ($(XC_HOST),) +XC_HOST := $(ARCH)$(shell echo $(XC_HOST) | sed "s/[^-]*\(.*\)$$/\1/") +MARCH := $(subst _,-,$(ARCH)) +endif +endif + ifeq ($(ARCH),mingw32) $(error "the mingw32 compiler you are using fails the openblas testsuite. please see the README.windows document for a replacement") else ifeq (cygwin, $(shell $(CC) -dumpmachine | cut -d\- -f3)) @@ -494,24 +503,25 @@ $(error "cannot build julia with cygwin-target compilers. set XC_HOST to i686-w6 else ifeq (msys, $(shell $(CC) -dumpmachine | cut -d\- -f3)) $(error "cannot build julia with msys-target compilers. please see the README.windows document for instructions on setting up mingw-w64 compilers") endif + ifeq ($(BUILD_OS),Darwin) -## Mac is a rather amazing 64-bit user-space on 32-bit kernel architecture, so to determine arch we detect +## Mac is a rather cool 64-bit user-space on 32-bit kernel architecture, so to determine arch we detect ## the capabilities of the hardware, rather than the compiler or kernel, and make a substitution -ifeq ($(ARCH),x86_64) -override ARCH := i686 -else ifeq ($(ARCH),i386) -override ARCH := i686 +BUILD_ARCH := $(shell echo $(BUILD_MACHINE) | sed "s/\([^-]*\).*$$/\1/") +ifeq ($(BUILD_ARCH),x86_64) +BUILD_ARCH := i686 +else ifeq ($(BUILD_ARCH),i386) +BUILD_ARCH := i686 endif -ifeq ($(ARCH),i686) +ifeq ($(BUILD_ARCH),i686) ifeq ($(shell sysctl -n hw.cpu64bit_capable),1) -override ARCH := x86_64 +BUILD_ARCH := x86_64 endif -BUILD_MACHINE := $(ARCH)$(shell echo $(BUILD_MACHINE) | sed "s/[^-]*\(.*\)$$/\1/") +BUILD_MACHINE := $(BUILD_ARCH)$(shell echo $(BUILD_MACHINE) | sed "s/[^-]*\(.*\)$$/\1/") endif +ifeq ($(BUILD_OS),$(OS)) +ARCH := $(BUILD_OS) endif -else -XC_HOST := $(ARCH)$(shell echo $(BUILD_MACHINE) | sed "s/[^-]*\(.*\)$$/\1/") -MARCH := $(ARCH) endif ifneq ($(MARCH),) From 171d7fba1ab1de2a1f59c44d9b91f90fd605c854 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 16 Aug 2015 22:25:38 -0400 Subject: [PATCH 0070/1938] fix llvm-config.exe cross-compile rule detection --- Make.inc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Make.inc b/Make.inc index ca86dd8643663..2621d0d1c6796 100644 --- a/Make.inc +++ b/Make.inc @@ -646,17 +646,17 @@ LLVM_CONFIG := $(build_bindir)/llvm-config$(EXE) endif endif # LLVM_CONFIG undefined -ifneq ($(BUILD_OS),$(OS)) +ifeq ($(BUILD_OS),$(OS)) +LLVM_CONFIG_HOST := $(LLVM_CONFIG) +else LLVM_CONFIG_HOST := $(basename $(LLVM_CONFIG))-host$(BUILD_EXE) -ifeq (exists, $$(shell [ -d '$(LLVM_CONFIG_HOST)' ] && echo exists )) +ifeq (exists, $(shell [ -f '$(LLVM_CONFIG_HOST)' ] && echo exists )) ifeq ($(shell $(LLVM_CONFIG_HOST) --version),3.3) # llvm-config-host <= 3.3 is broken, use llvm-config instead (in an emulator) # use delayed expansion (= not :=) because spawn isn't defined until later LLVM_CONFIG_HOST = $(call spawn,$(LLVM_CONFIG)) endif endif -else -LLVM_CONFIG_HOST := $(LLVM_CONFIG) endif # SYSTEM_LLVM ifeq ($(USE_SYSTEM_PCRE), 1) From 285539887db1771cf3e21a59202f619baa98b842 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 16 Aug 2015 22:36:17 -0400 Subject: [PATCH 0071/1938] add missing updates to := for perf/*/Makefile --- test/perf/arrayalloc/Makefile | 2 +- test/perf/micro/Makefile | 2 +- test/perf/shootout/Makefile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/perf/arrayalloc/Makefile b/test/perf/arrayalloc/Makefile index ace591be023d2..5ee74d8bbf34b 100644 --- a/test/perf/arrayalloc/Makefile +++ b/test/perf/arrayalloc/Makefile @@ -1,4 +1,4 @@ -JULIAHOME = $(abspath ../../..) +JULIAHOME := $(abspath ../../..) include $(JULIAHOME)/Make.inc ifndef PYTHONBIN diff --git a/test/perf/micro/Makefile b/test/perf/micro/Makefile index ef7bbe80472d1..ee9db271a0153 100644 --- a/test/perf/micro/Makefile +++ b/test/perf/micro/Makefile @@ -1,4 +1,4 @@ -JULIAHOME = $(abspath ../../..) +JULIAHOME := $(abspath ../../..) include $(JULIAHOME)/Make.inc include $(JULIAHOME)/deps/Versions.make diff --git a/test/perf/shootout/Makefile b/test/perf/shootout/Makefile index 622e92708e0b9..cbdc6cd30374e 100644 --- a/test/perf/shootout/Makefile +++ b/test/perf/shootout/Makefile @@ -1,4 +1,4 @@ -JULIAHOME = $(abspath ../../..) +JULIAHOME := $(abspath ../../..) include ../../../Make.inc SHOOTOUTFILES = knucleotide-input.txt regexdna-input.txt revcomp-input.txt fasta-output.txt mandelbrot-output.txt nbody-output.txt fannkuchredux-output.txt pidigits-output.txt knucleotide-output.txt revcomp-output.txt spectralnorm-output.txt regexdna-output.txt meteor-output.txt From c7253e6f4360dcd9cf8edae89a36c9c37a3a4643 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 11 Sep 2015 16:13:05 -0400 Subject: [PATCH 0072/1938] fix include location for looking up julia_flisp.boot.inc --- Makefile | 6 +++--- src/ast.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 45524e9355026..d796917868cc2 100644 --- a/Makefile +++ b/Makefile @@ -214,10 +214,10 @@ $(build_private_libdir)/inference.ji: $(build_private_libdir)/inference0.ji RELBUILDROOT := $(call cygpath_w,$(shell $(JULIAHOME)/contrib/relative_path.sh "$(JULIAHOME)/base" "$(BUILDROOT)/base/")) COMMA:=, define sysimg_builder -$$(build_private_libdir)/sys$1.o: $$(build_private_libdir)/inference.ji $(JULIAHOME)/VERSION $$(BASE_SRCS) - @$$(call PRINT_JULIA, cd $(JULIAHOME)/base && \ +$$(build_private_libdir)/sys$1.o: $$(build_private_libdir)/inference.ji $$(JULIAHOME)/VERSION $$(BASE_SRCS) + @$$(call PRINT_JULIA, cd $$(JULIAHOME)/base && \ $$(call spawn,$2) -C $$(JULIA_CPU_TARGET) --output-o $$(call cygpath_w,$$@) $$(JULIA_SYSIMG_BUILD_FLAGS) -f \ - -J $$(call cygpath_w,$$<) sysimg.jl $(RELBUILDROOT) \ + -J $$(call cygpath_w,$$<) sysimg.jl $$(RELBUILDROOT) \ || { echo '*** This error is usually fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***' && false; } ) .SECONDARY: $(build_private_libdir)/sys$1.o endef diff --git a/src/ast.c b/src/ast.c index 0f30f19ddca78..e9454c8f6ecdb 100644 --- a/src/ast.c +++ b/src/ast.c @@ -26,7 +26,7 @@ extern "C" { #endif static uint8_t flisp_system_image[] = { -#include "julia_flisp.boot.inc" +#include }; static fltype_t *jvtype=NULL; From c129d1e32709ca8cf91b75229eeca77e4e59a68b Mon Sep 17 00:00:00 2001 From: Kiran Pamnany Date: Fri, 11 Sep 2015 16:07:09 -0700 Subject: [PATCH 0073/1938] Atomics. --- Make.user | 4 +-- base/atomics.jl | 85 +++++++++++++++++++++++++++++++++++++++++++++ base/sysimg.jl | 3 +- base/threading.jl | 25 +++++++++++++ test/atomics.jl | 12 +++++++ test/choosetests.jl | 2 +- test/threads.jl | 2 -- 7 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 base/atomics.jl create mode 100644 test/atomics.jl diff --git a/Make.user b/Make.user index 1fd6a61d95bd9..861c40993919d 100644 --- a/Make.user +++ b/Make.user @@ -1,6 +1,6 @@ JCPPFLAGS = -ftls-model=global-dynamic LLVM_VER = svn -LLVM_GIT_URL_LLVM = https://github.com/JuliaLang/llvm.git -b kf/tlsrebase5 +LLVM_GIT_URL_LLVM = https://github.com/JuliaLang/llvm.git -b jn/tlsrebaseagainagain -LLVM_DEBUG = 1 +LLVM_DEBUG = 0 LLVM_ASSERTIONS = 1 diff --git a/base/atomics.jl b/base/atomics.jl new file mode 100644 index 0000000000000..f0b4752846a24 --- /dev/null +++ b/base/atomics.jl @@ -0,0 +1,85 @@ +# Copyright (c) 2015, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Atomics + +using Base.Intrinsics: llvmcall + +import Base: setindex!, getindex + +export + Atomic, + atomic_cas!, + atomic_xchg!, + atomic_add!, atomic_sub!, + atomic_and!, atomic_nand!, atomic_or!, atomic_xor!, + atomic_max!, atomic_min!, atomic_umax!, atomic_umin! + +type Atomic{T<:Integer} + value::T + Atomic() = new(zero(T)) + Atomic(value) = new(value) +end + +Atomic() = Atomic{Int}() + +atomicintsmap = Dict(Int8 => "i8", Uint8 => "i8", + Int16 => "i16", Uint16 => "i16", + Int32 => "i32", Uint32 => "i32", + Int64 => "i64", Uint64 => "i64", + Int128 => "i128", Uint128 => "i128") + +unsafe_convert{T}(::Type{Ptr{T}}, x::Atomic{T}) = convert(Ptr{T}, pointer_from_objref(x)) +setindex!{T}(x::Atomic{T}, v) = setindex!(x, convert(T, v)) + +for (typ, lt) in atomicintsmap + @eval getindex(x::Atomic{$typ}) = + llvmcall($""" + %rv = load atomic volatile $lt, $lt* %0 monotonic, align $WORD_SIZE + ret $lt %rv + """, $typ, Tuple{Ptr{$typ}}, unsafe_convert(Ptr{$typ}, x)) + @eval setindex!(x::Atomic{$typ}, v::$typ) = + llvmcall($""" + store atomic volatile $lt %1, $lt* %0 monotonic, align $WORD_SIZE + ret void + """, Void, Tuple{Ptr{$typ},$typ}, unsafe_convert(Ptr{$typ}, x), v) + @eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) = + llvmcall($""" + %rv = cmpxchg $lt* %0, $lt %1, $lt %2 acq_rel monotonic + %bv = extractvalue { $lt, i1 } %rv, 1 + ret i1 %bv + """, Bool, Tuple{Ptr{$typ},$typ,$typ}, unsafe_convert(Ptr{$typ}, x), cmp, new) + for rmwop in [:xchg, :add, :sub, :and, :nand, :or, :xor, :max, :min, :umax, :umin] + rmw = string(rmwop) + fn = symbol("atomic_", rmw, "!") + @eval $fn(x::Atomic{$typ}, v::$typ) = + llvmcall($""" + %rv = atomicrmw volatile $rmw $lt* %0, $lt %1 acquire + ret $lt %rv + """, $typ, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v) + end +end + +end diff --git a/base/sysimg.jl b/base/sysimg.jl index 672033d50c4c1..50954e94e66cf 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -299,8 +299,9 @@ include("deprecated.jl") include("docs/helpdb.jl") include("docs/basedocs.jl") -# threading +# threading stuff include("threading.jl") +include("atomics.jl") function __init__() # Base library init diff --git a/base/threading.jl b/base/threading.jl index 117e108056b52..b2e4496a3838d 100644 --- a/base/threading.jl +++ b/base/threading.jl @@ -1,3 +1,28 @@ +# Copyright (c) 2015, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + module Threading export threadid, maxthreads, nthreads, @threads diff --git a/test/atomics.jl b/test/atomics.jl new file mode 100644 index 0000000000000..9c7d4d511d25e --- /dev/null +++ b/test/atomics.jl @@ -0,0 +1,12 @@ +using Base.Test +using Base.Threading +using Base.Atomics + +x = Atomic() + +@threads all for i = 1:10000 + atomic_add!(x, 1) +end + +@test x[] == 10000 + diff --git a/test/choosetests.jl b/test/choosetests.jl index 8316cf016a547..1ffda104bd22a 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -30,7 +30,7 @@ function choosetests(choices = []) "euler", "show", "lineedit", "replcompletions", "repl", "replutil", "sets", "test", "goto", "llvmcall", "grisu", "nullable", "meta", "profile", "libgit2", "docs", "markdown", - "threads", "base64", "serialize", "functors", "misc", + "threads", "atomics", "base64", "serialize", "functors", "misc", "enums", "cmdlineargs", "i18n", "workspace", "libdl", "int", "intset", "floatfuncs", "compile" ] diff --git a/test/threads.jl b/test/threads.jl index e05145c261f32..c6969474b7e5d 100644 --- a/test/threads.jl +++ b/test/threads.jl @@ -1,8 +1,6 @@ using Base.Test using Base.Threading -println("Threading tests") - expected = [1:nthreads();] # test 1 From dea2fa65cfeae50280ad99e1e058a69550be3553 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 12 Sep 2015 19:58:19 -0400 Subject: [PATCH 0074/1938] also fix contrib/mac/app/Makefile for := change we have too many Makefiles --- contrib/mac/app/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/mac/app/Makefile b/contrib/mac/app/Makefile index 90ae14e829222..f3cee4649b337 100644 --- a/contrib/mac/app/Makefile +++ b/contrib/mac/app/Makefile @@ -2,15 +2,15 @@ # You may have to wipe your openblas build to ensure that it is built # with support for all architectures, or else performance may suffer. -JULIAHOME = $(abspath ../../..) +JULIAHOME := $(abspath ../../..) include ../../../Make.inc -JULIA_PKGDIR=$(shell echo $$JULIA_PKGDIR) +JULIA_PKGDIR:=$(shell echo $$JULIA_PKGDIR) ifeq ($(JULIA_PKGDIR),) - JULIA_PKGDIR=$(shell echo ~)/.julia + JULIA_PKGDIR:=$(shell echo ~)/.julia endif -VERSION_SUFFIX=$(shell [ $$(git describe --tags --exact-match 2>/dev/null) ] && echo $(JULIA_VERSION) || echo $(JULIA_VERSION)-$(JULIA_COMMIT)) +VERSION_SUFFIX:=$(shell [ $$(git describe --tags --exact-match 2>/dev/null) ] && echo $(JULIA_VERSION) || echo $(JULIA_VERSION)-$(JULIA_COMMIT)) all: clean dmg From 1565377be0fde1186a12a60909821b161d5e7501 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 12 Sep 2015 20:36:33 -0400 Subject: [PATCH 0075/1938] remove OPENLIBM_HOME override that was breaking the openlibm makefile configuration on cygwin --- deps/Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index 7925dd0ac4841..c681a0925edfc 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -884,9 +884,6 @@ $(eval $(call git-external,openlibm,OPENLIBM,Makefile,libopenlibm.$(SHLIB_EXT),$ OPENLIBM_OBJ_TARGET := $(build_shlibdir)/libopenlibm.$(SHLIB_EXT) $(build_libdir)/libopenlibm.a OPENLIBM_OBJ_SOURCE := $(BUILDDIR)/$(OPENLIBM_SRC_DIR)/libopenlibm.$(SHLIB_EXT) OPENLIBM_FLAGS := ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) -ifeq (CYGWIN,$(findstring CYGWIN,$(BUILD_OS))) -OPENLIBM_FLAGS += OPENLIBM_HOME="$(call cygpath_w,$(abspath $(OPENLIBM_SRC_DIR)))" -endif $(OPENLIBM_OBJ_SOURCE): $(BUILDDIR)/$(OPENLIBM_SRC_DIR)/Makefile $(MAKE) -C $(dir $<) $(OPENLIBM_FLAGS) $(MAKE_COMMON) From cc0d9fe47ebeca296b1b446a9981088bf4810487 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 12 Sep 2015 20:39:34 -0400 Subject: [PATCH 0076/1938] use spaces not tabs in docs/makefile to lineup := --- doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Makefile b/doc/Makefile index df4fd75fccfb9..c5091f3764c04 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -5,7 +5,7 @@ default: html # You can set these variables from the command line. SPHINXOPTS := PAPER := -SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) BUILDDIR := . JULIAHOME := $(abspath $(SRCDIR)/..) JULIA_EXECUTABLE := $(JULIAHOME)/usr/bin/julia From 130083ff0fc6b3290e0bf0868272ccd89df4d725 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 12 Sep 2015 21:14:18 -0400 Subject: [PATCH 0077/1938] make the srccache a full git repo this should make it more practical to develop and hop branches without network access to cut back down on network traffic, we could consider also switching to NO_GIT tarballs as a default, but that would also prevent incremental recompilation for those dependencies (which could be good or bad depending on your viewpoint), so that change will require more thought --- deps/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index c681a0925edfc..a7a66437b612a 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -264,11 +264,11 @@ ifneq ($(NO_GIT),1) $2_SRC_DIR := $1 $2_SRC_FILE := $$(SRCDIR)/srccache/$1.git $$($2_SRC_FILE): | $$(SRCDIR)/srccache - git clone -q --mirror --depth=10 --branch $$($2_BRANCH) $$($2_GIT_URL) $$@ + git clone -q --mirror --branch $$($2_BRANCH) $$($2_GIT_URL) $$@ $5/$1: | $$($2_SRC_FILE) # try to update the cache, if that fails, attempt to continue anyways (the ref might already be local) -cd $$(SRCDIR)/srccache/$1.git && git fetch -q $$($2_GIT_URL) $$($2_BRANCH):remotes/origin/$$($2_BRANCH) - git clone -q --branch $$($2_BRANCH) $$(SRCDIR)/srccache/$1.git $$@ + git clone -q --depth=10 --branch $$($2_BRANCH) $$(SRCDIR)/srccache/$1.git $$@ cd $5/$1 && git remote set-url origin $$($2_GIT_URL) touch -c $5/$1/$3 ifneq ($5,$$(BUILDDIR)) From 43e283d6535efa793e968964c75e6d60c43b0120 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 28 Aug 2015 00:41:09 -0400 Subject: [PATCH 0078/1938] refactor IO types to be less brittle, more flat, and to fix #12829 and #12050 --- base/REPL.jl | 5 +- base/fs.jl | 15 +- base/io.jl | 45 ++++- base/iobuffer.jl | 1 + base/iostream.jl | 13 +- base/multi.jl | 8 +- base/process.jl | 49 +++--- base/socket.jl | 24 +-- base/stream.jl | 405 ++++++++++++++++++++++----------------------- base/strings/io.jl | 21 ++- test/spawn.jl | 46 ++++- 11 files changed, 348 insertions(+), 284 deletions(-) diff --git a/base/REPL.jl b/base/REPL.jl index 267ba1d4cbd2d..04faf504cf529 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -13,7 +13,6 @@ export StreamREPL import Base: - AsyncStream, Display, display, writemime, @@ -884,7 +883,7 @@ end outstream(s::StreamREPL) = s.stream -StreamREPL(stream::AsyncStream) = StreamREPL(stream, julia_green, Base.text_colors[:white], Base.answer_color()) +StreamREPL(stream::IO) = StreamREPL(stream, julia_green, Base.text_colors[:white], Base.answer_color()) answer_color(r::LineEditREPL) = r.envcolors ? Base.answer_color() : r.answer_color answer_color(r::StreamREPL) = r.answer_color @@ -892,7 +891,7 @@ input_color(r::LineEditREPL) = r.envcolors ? Base.input_color() : r.input_color input_color(r::StreamREPL) = r.input_color -function run_repl(stream::AsyncStream) +function run_repl(stream::IO) repl = @async begin repl_channel = Channel(1) diff --git a/base/fs.jl b/base/fs.jl index db991a47a9206..c890d16402213 100644 --- a/base/fs.jl +++ b/base/fs.jl @@ -231,19 +231,14 @@ function read(f::File, ::Type{UInt8}) return ret%UInt8 end -function read!{T}(f::File, a::Array{T}, nel=length(a)) +function read!(f::File, a::Vector{UInt8}, nel=length(a)) if nel < 0 || nel > length(a) throw(BoundsError()) end - if isbits(T) - nb = nel*sizeof(T) - ret = ccall(:jl_fs_read, Int32, (Int32, Ptr{Void}, Csize_t), - f.handle, a, nb) - uv_error("read",ret) - else - invoke(read, Tuple{IO, Array}, s, a) - end - a + ret = ccall(:jl_fs_read, Int32, (Int32, Ptr{Void}, Csize_t), + f.handle, a, nel) + uv_error("read",ret) + return a end nb_available(f::File) = filesize(f) - position(f) diff --git a/base/io.jl b/base/io.jl index 1203f1244e6dd..ca653d8970dd9 100644 --- a/base/io.jl +++ b/base/io.jl @@ -28,9 +28,6 @@ isreadonly(s) = isreadable(s) && !iswritable(s) ## binary I/O ## -# all subtypes should implement this -write(s::IO, x::UInt8) = error(typeof(s)," does not support byte I/O") - write(io::IO, x) = throw(MethodError(write, (io, x))) function write(io::IO, xs...) local written::Int = 0 @@ -107,9 +104,6 @@ function write(io::IO, s::Symbol) return write(io, pname, Int(ccall(:strlen, Csize_t, (Ptr{UInt8},), pname))) end -# all subtypes should implement this -read(s::IO, ::Type{UInt8}) = error(typeof(s)," does not support byte I/O") - read(s::IO, ::Type{Int8}) = reinterpret(Int8, read(s,UInt8)) function read{T <: Integer}(s::IO, ::Type{T}) @@ -131,9 +125,20 @@ read{T}(s::IO, t::Type{T}, d1::Integer, dims::Integer...) = read{T}(s::IO, ::Type{T}, dims::Dims) = read!(s, Array(T, dims)) +function read!(s::IO, a::Vector{UInt8}) + for i in 1:length(a) + a[i] = read(s, UInt8) + end +end + function read!{T}(s::IO, a::Array{T}) - for i in eachindex(a) - a[i] = read(s, T) + if isbits(T) + nb::Int = length(a) * sizeof(T) + read!(s, reinterpret(UInt8, a, (nb,))) + else + for i in eachindex(a) + a[i] = read(s, T) + end end return a end @@ -219,7 +224,7 @@ function readuntil(s::IO, t::AbstractString) return takebuf_string(out) end - +readline() = readline(STDIN) readline(s::IO) = readuntil(s, '\n') readchomp(x) = chomp!(readall(x)) @@ -308,3 +313,25 @@ ismarked(io::IO) = io.mark >= 0 lock(::IO) = nothing unlock(::IO) = nothing +reseteof(x::IO) = nothing + +const SZ_UNBUFFERED_IO = 65536 +buffer_writes(x::IO, bufsize=SZ_UNBUFFERED_IO) = nothing + +function isopen end +function close end +function flush end +function wait_connected end +function wait_readnb end +function wait_readbyte end +function wait_close end +function nb_available end +function readavailable end +function isreadable end +function iswritable end +function copy end +function eof end + +# all subtypes should implement this +read(s::IO, ::Type{UInt8}) = error(typeof(s)," does not support byte I/O") +write(s::IO, x::UInt8) = error(typeof(s)," does not support byte I/O") diff --git a/base/iobuffer.jl b/base/iobuffer.jl index e6f4914741b1e..0b3c177b02b6e 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -52,6 +52,7 @@ show(io::IO, b::AbstractIOBuffer) = print(io, "IOBuffer(data=UInt8[...], ", "ptr=", b.ptr, ", ", "mark=", b.mark, ")") +read!(from::AbstractIOBuffer, a::Vector{UInt8}) = read_sub(from, a, 1, length(a)) read!(from::AbstractIOBuffer, a::Array) = read_sub(from, a, 1, length(a)) function read_sub{T}(from::AbstractIOBuffer, a::AbstractArray{T}, offs, nel) diff --git a/base/iostream.jl b/base/iostream.jl index b1c08cfb0be60..e3bdd07329baf 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -170,15 +170,10 @@ function read{T<:Union{UInt16, Int16, UInt32, Int32, UInt64, Int64}}(s::IOStream ccall(:jl_ios_get_nbyte_int, UInt64, (Ptr{Void}, Csize_t), s.ios, sizeof(T)) % T end -function read!{T}(s::IOStream, a::Array{T}) - if isbits(T) - nb = length(a)*sizeof(T) - if ccall(:ios_readall, UInt, - (Ptr{Void}, Ptr{Void}, UInt), s.ios, a, nb) < nb - throw(EOFError()) - end - else - invoke(read!, Tuple{IO, Array}, s, a) +function read!(s::IOStream, a::Vector{UInt8}) + if ccall(:ios_readall, UInt, + (Ptr{Void}, Ptr{Void}, UInt), s.ios, a, sizeof(a)) < sizeof(a) + throw(EOFError()) end a end diff --git a/base/multi.jl b/base/multi.jl index cff41f09549aa..3378654457d59 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -134,8 +134,8 @@ type Worker c_state::Condition # wait for state changes ct_time::Float64 # creation time - r_stream::AsyncStream - w_stream::AsyncStream + r_stream::IO + w_stream::IO manager::ClusterManager config::WorkerConfig @@ -836,9 +836,9 @@ function process_tcp_streams(r_stream::TCPSocket, w_stream::TCPSocket) message_handler_loop(r_stream, w_stream) end -process_messages(r_stream::AsyncStream, w_stream::AsyncStream) = @schedule message_handler_loop(r_stream, w_stream) +process_messages(r_stream::IO, w_stream::IO) = @schedule message_handler_loop(r_stream, w_stream) -function message_handler_loop(r_stream::AsyncStream, w_stream::AsyncStream) +function message_handler_loop(r_stream::IO, w_stream::IO) global PGRP global cluster_manager diff --git a/base/process.jl b/base/process.jl index 879e4288bf356..c9af9053f9d7d 100644 --- a/base/process.jl +++ b/base/process.jl @@ -86,7 +86,7 @@ immutable FileRedirect end end -immutable DevNullStream <: AsyncStream end +immutable DevNullStream <: IO end const DevNull = DevNullStream() isreadable(::DevNullStream) = false iswritable(::DevNullStream) = true @@ -96,17 +96,24 @@ write{T<:DevNullStream}(::T, args...) = 0 close(::DevNullStream) = nothing flush(::DevNullStream) = nothing copy(::DevNullStream) = DevNull +wait_connected(::DevNullStream) = nothing +wait_readnb(::DevNullStream) = wait() +wait_readbyte(::DevNullStream) = wait() +wait_close(::DevNullStream) = wait() +eof(::DevNullStream) = true uvhandle(::DevNullStream) = C_NULL +uvtype(::DevNullStream) = UV_STREAM + uvhandle(x::Ptr) = x uvtype(::Ptr) = UV_STREAM -uvtype(::DevNullStream) = UV_STREAM # Not actually a pointer, but that's how we pass it through the C API so it's fine uvhandle(x::RawFD) = convert(Ptr{Void}, x.fd % UInt) uvtype(x::RawFD) = UV_RAW_FD -typealias Redirectable Union{AsyncStream, FS.File, FileRedirect, DevNullStream, IOStream, RawFD} +typealias Redirectable Union{IO, FileRedirect, RawFD} +typealias StdIOSet NTuple{3, Union{Redirectable, Ptr{Void}}} # XXX: remove Ptr{Void} once libuv is refactored to use upstream release immutable CmdRedirect <: AbstractCmd cmd::AbstractCmd @@ -197,30 +204,30 @@ pipeline(src::Union{Redirectable,AbstractString}, cmd::AbstractCmd) = pipeline(c pipeline(a, b, c, d...) = pipeline(pipeline(a,b), c, d...) -typealias RawOrBoxedHandle Union{UVHandle,AsyncStream,Redirectable,IOStream} -typealias StdIOSet NTuple{3,RawOrBoxedHandle} - type Process <: AbstractPipe cmd::Cmd handle::Ptr{Void} - in::AsyncStream - out::AsyncStream - err::AsyncStream + in::IO + out::IO + err::IO exitcode::Int64 termsignal::Int32 exitcb::Callback exitnotify::Condition closecb::Callback closenotify::Condition - function Process(cmd::Cmd, handle::Ptr{Void}, in::RawOrBoxedHandle, out::RawOrBoxedHandle, err::RawOrBoxedHandle) - if !isa(in, AsyncStream) || in === DevNull - in=DevNull + function Process(cmd::Cmd, handle::Ptr{Void}, + in::Union{Redirectable, Ptr{Void}}, + out::Union{Redirectable, Ptr{Void}}, + err::Union{Redirectable, Ptr{Void}}) + if !isa(in, IO) + in = DevNull end - if !isa(out, AsyncStream) || out === DevNull - out=DevNull + if !isa(out, IO) + out = DevNull end - if !isa(err, AsyncStream) || err === DevNull - err=DevNull + if !isa(err, IO) + err = DevNull end this = new(cmd, handle, in, out, err, typemin(fieldtype(Process, :exitcode)), @@ -431,7 +438,7 @@ end # | | \ The function to be called once the uv handle is closed # | \ The function to be called once the process exits # \ A set of up to 256 stdio instructions, where each entry can be either: -# | - An AsyncStream to be passed to the child +# | - An IO to be passed to the child # | - DevNull to pass /dev/null # | - An FS.File object to redirect the output to # \ - An ASCIIString specifying a filename to be opened @@ -464,7 +471,7 @@ end eachline(cmd::AbstractCmd) = eachline(cmd, DevNull) # return a Process object to read-to/write-from the pipeline -function open(cmds::AbstractCmd, mode::AbstractString="r", other::AsyncStream=DevNull) +function open(cmds::AbstractCmd, mode::AbstractString="r", other::Redirectable=DevNull) if mode == "r" in = other out = io = Pipe() @@ -502,18 +509,18 @@ function readandwrite(cmds::AbstractCmd) (out, in, processes) end -function readbytes(cmd::AbstractCmd, stdin::AsyncStream=DevNull) +function readbytes(cmd::AbstractCmd, stdin::Redirectable=DevNull) out, procs = open(cmd, "r", stdin) bytes = readbytes(out) !success(procs) && pipeline_error(procs) return bytes end -function readall(cmd::AbstractCmd, stdin::AsyncStream=DevNull) +function readall(cmd::AbstractCmd, stdin::Redirectable=DevNull) return bytestring(readbytes(cmd, stdin)) end -function writeall(cmd::AbstractCmd, stdin::AbstractString, stdout::AsyncStream=DevNull) +function writeall(cmd::AbstractCmd, stdin::AbstractString, stdout::Redirectable=DevNull) open(cmd, "w", stdout) do io write(io, stdin) end diff --git a/base/socket.jl b/base/socket.jl index 01cfa0756d31d..8c5491fe86a66 100644 --- a/base/socket.jl +++ b/base/socket.jl @@ -248,9 +248,7 @@ end ## SOCKETS ## -abstract Socket <: AsyncStream - -type TCPSocket <: Socket +type TCPSocket <: LibuvStream handle::Ptr{Void} status::Int line_buffered::Bool @@ -294,10 +292,7 @@ function TCPSocket() this end -lock(s::TCPSocket) = lock(s.lock) -unlock(s::TCPSocket) = unlock(s.lock) - -type TCPServer <: UVServer +type TCPServer <: LibuvServer handle::Ptr{Void} status::Int ccb::Callback @@ -328,13 +323,8 @@ function TCPServer() this end -isreadable(io::TCPSocket) = true -iswritable(io::TCPSocket) = true - -show(io::IO,sock::TCPSocket) = print(io,"TCPSocket(",uv_status_string(sock),", ", - nb_available(sock.buffer)," bytes waiting)") - -show(io::IO,sock::TCPServer) = print(io,"TCPServer(",uv_status_string(sock),")") +isreadable(io::TCPSocket) = isopen(io) || nb_available(io) > 0 +iswritable(io::TCPSocket) = isopen(io) && io.status != StatusClosing ## VARIOUS METHODS TO BE MOVED TO BETTER LOCATION @@ -365,7 +355,7 @@ _bind(sock::TCPServer, host::IPv6, port::UInt16) = ccall(:jl_tcp_bind6, Int32, ( # UDP -type UDPSocket <: Socket +type UDPSocket <: LibuvStream handle::Ptr{Void} status::Int recvnotify::Condition @@ -694,7 +684,7 @@ end ## -listen(sock::UVServer; backlog::Integer=BACKLOG_DEFAULT) = (uv_error("listen",_listen(sock;backlog=backlog)); sock) +listen(sock::LibuvServer; backlog::Integer=BACKLOG_DEFAULT) = (uv_error("listen",_listen(sock;backlog=backlog)); sock) function listen(addr; backlog::Integer=BACKLOG_DEFAULT) sock = TCPServer() @@ -706,7 +696,7 @@ listen(port::Integer; backlog::Integer=BACKLOG_DEFAULT) = listen(IPv4(UInt32(0)) listen(host::IPAddr, port::Integer; backlog::Integer=BACKLOG_DEFAULT) = listen(InetAddr(host,port);backlog=backlog) listen(cb::Callback,args...; backlog::Integer=BACKLOG_DEFAULT) = (sock=listen(args...;backlog=backlog);sock.ccb=cb;sock) -listen(cb::Callback,sock::Socket; backlog::Integer=BACKLOG_DEFAULT) = (sock.ccb=cb;listen(sock;backlog=backlog)) +listen(cb::Callback,sock::Union{TCPSocket,UDPSocket}; backlog::Integer=BACKLOG_DEFAULT) = (sock.ccb=cb;listen(sock;backlog=backlog)) ## diff --git a/base/stream.jl b/base/stream.jl index b254a1f5e8e76..8027d464f4c21 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -9,10 +9,35 @@ import .Libc: RawFD, dup ## types ## typealias Callback Union{Function,Bool} -abstract AsyncStream <: IO -abstract UVServer - -typealias UVHandle Ptr{Void} +abstract IOServer +abstract LibuvServer <: IOServer +abstract LibuvStream <: IO + +# IO +# +- AbstractIOBuffer{T<:AbstractArray{UInt8,1}} (not exported) +# +- AbstractPipe (not exported) +# . +- Pipe +# . +- Process (not exported) +# . +- ProcessChain (not exported) +# +- Base64DecodePipe +# +- Base64EncodePipe +# +- BufferStream +# +- DevNullStream (not exported) +# +- Filesystem.File +# +- LibuvStream (not exported) +# . +- PipeEndpoint (not exported) +# . +- TCPSocket +# . +- TTY (not exported) +# . +- UDPSocket +# +- IOBuffer = Base.AbstractIOBuffer{Array{UInt8,1}} +# +- IOStream + +# IOServer +# +- LibuvServer +# . +- PipeServer +# . +- TCPServer + +# Redirectable = Union{IO, FileRedirect, Libc.RawFD} (not exported) # convert UV handle data to julia object, checking for null macro handle_as(hand, typ) @@ -26,10 +51,19 @@ end # A dict of all libuv handles that are being waited on somewhere in the system # and should thus not be garbage collected const uvhandles = ObjectIdDict() - preserve_handle(x) = uvhandles[x] = get(uvhandles,x,0)+1 unpreserve_handle(x) = (v = uvhandles[x]; v == 1 ? pop!(uvhandles,x) : (uvhandles[x] = v-1); nothing) +function stream_wait(x, c...) # for x::LibuvObject + preserve_handle(x) + try + return wait(c...) + finally + unpreserve_handle(x) + end +end + + function uv_sizeof_handle(handle) if !(UV_UNKNOWN_HANDLE < handle < UV_HANDLE_TYPE_MAX) throw(DomainError()) @@ -51,9 +85,9 @@ for r in uv_req_types @eval const $(symbol("_sizeof_"*lowercase(string(r)))) = uv_sizeof_req($r) end -nb_available(s::AsyncStream) = nb_available(s.buffer) +nb_available(s::LibuvStream) = nb_available(s.buffer) -function eof(s::AsyncStream) +function eof(s::LibuvStream) wait_readnb(s,1) !isopen(s) && nb_available(s)<=0 end @@ -102,7 +136,7 @@ uv_req_data(handle) = ccall(:jl_uv_req_data,Ptr{Void},(Ptr{Void},),handle) uv_req_set_data(req,data) = ccall(:jl_uv_req_set_data,Void,(Ptr{Void},Any),req,data) uv_req_set_data(req,data::Ptr{Void}) = ccall(:jl_uv_req_set_data,Void,(Ptr{Void},Ptr{Void}),req,data) -type PipeEndpoint <: AsyncStream +type PipeEndpoint <: LibuvStream handle::Ptr{Void} status::Int buffer::IOBuffer @@ -129,10 +163,7 @@ type PipeEndpoint <: AsyncStream DEFAULT_READ_BUFFER_SZ) end -lock(p::PipeEndpoint) = lock(p.lock) -unlock(p::PipeEndpoint) = unlock(p.lock) - -type PipeServer <: UVServer +type PipeServer <: LibuvServer handle::Ptr{Void} status::Int ccb::Callback @@ -146,20 +177,7 @@ type PipeServer <: UVServer false,Condition()) end -function init_pipe!(pipe::Union{PipeEndpoint,PipeServer}; - readable::Bool = false, - writable::Bool = false, - julia_only::Bool = true) - if pipe.status != StatusUninit - error("pipe is already initialized") - end - if pipe.handle == C_NULL - malloc_julia_pipe!(pipe) - end - uv_error("init_pipe",ccall(:jl_init_pipe, Cint, (Ptr{Void},Int32,Int32,Int32), pipe.handle, writable,readable,julia_only)) - pipe.status = StatusInit - pipe -end +typealias LibuvPipe Union{PipeEndpoint, PipeServer} function PipeServer() handle = Libc.malloc(_sizeof_uv_named_pipe) @@ -174,11 +192,7 @@ function PipeServer() end end -show(io::IO,stream::PipeEndpoint) = print(io,"PipeEndpoint(",uv_status_string(stream),", ", - nb_available(stream.buffer)," bytes waiting)") -show(io::IO,stream::PipeServer) = print(io,"PipeServer(",uv_status_string(stream),")") - -type TTY <: AsyncStream +type TTY <: LibuvStream handle::Ptr{Void} status::Int line_buffered::Bool @@ -219,55 +233,44 @@ function TTY(fd::RawFD; readable::Bool = false) ret end -lock(t::TTY) = lock(t.lock) -unlock(t::TTY) = unlock(t.lock) +show(io::IO,stream::LibuvServer) = print(io, typeof(stream), "(", uv_status_string(stream), ")") +show(io::IO, stream::LibuvStream) = print(io, typeof(stream), "(", uv_status_string(stream), ", ", + nb_available(stream.buffer)," bytes waiting)") -# note that uv_is_readable/writable work for any subtype of -# uv_stream_t, including uv_tty_t and uv_pipe_t -function isreadable(io::Union{PipeEndpoint,TTY}) +# Shared LibuvStream object interface + +function isreadable(io::LibuvStream) nb_available(io) > 0 && return true isopen(io) || return false return ccall(:uv_is_readable, Cint, (Ptr{Void},), io.handle) != 0 end -function iswritable(io::Union{PipeEndpoint,TTY}) + +function iswritable(io::LibuvStream) isopen(io) || return false io.status == StatusClosing && return false return ccall(:uv_is_writable, Cint, (Ptr{Void},), io.handle) != 0 end -nb_available(stream::AsyncStream) = nb_available(stream.buffer) - -show(io::IO,stream::TTY) = print(io,"TTY(",uv_status_string(stream),", ", - nb_available(stream.buffer)," bytes waiting)") - -function println(io::AsyncStream, xs...) - lock(io) - try - invoke(println, Tuple{IO, map(typeof,xs)...}, io, xs...) - finally - unlock(io) - end -end - +nb_available(stream::LibuvStream) = nb_available(stream.buffer) -uvtype(::AsyncStream) = UV_STREAM -uvhandle(stream::AsyncStream) = stream.handle +lock(s::LibuvStream) = lock(s.lock) +unlock(s::LibuvStream) = unlock(s.lock) -convert(T::Type{Ptr{Void}}, s::AsyncStream) = convert(T, s.handle) -handle(s::AsyncStream) = s.handle -handle(s::Ptr{Void}) = s +uvtype(::LibuvStream) = UV_STREAM +uvhandle(stream::LibuvStream) = stream.handle +unsafe_convert(::Type{Ptr{Void}}, s::Union{LibuvStream, LibuvServer}) = s.handle -associate_julia_struct(handle::Ptr{Void},jlobj::ANY) = - ccall(:jl_uv_associate_julia_struct,Void,(Ptr{Void},Any),handle,jlobj) +associate_julia_struct(handle::Ptr{Void}, jlobj::ANY) = + ccall(:jl_uv_associate_julia_struct, Void, (Ptr{Void}, Any), handle, jlobj) disassociate_julia_struct(uv) = disassociate_julia_struct(uv.handle) disassociate_julia_struct(handle::Ptr{Void}) = - handle != C_NULL && ccall(:jl_uv_disassociate_julia_struct,Void,(Ptr{Void},),handle) + handle != C_NULL && ccall(:jl_uv_disassociate_julia_struct, Void, (Ptr{Void},), handle) -function init_stdio(handle) - t = ccall(:jl_uv_handle_type,Int32,(Ptr{Void},),handle) +function init_stdio(handle::Ptr{Void}) + t = ccall(:jl_uv_handle_type, Int32, (Ptr{Void},), handle) if t == UV_FILE - return fdio(ccall(:jl_uv_file_handle,Int32,(Ptr{Void},),handle)) -# Replace ios.c filw with libuv file? + return fdio(ccall(:jl_uv_file_handle, Int32, (Ptr{Void},), handle)) +# Replace ios.c file with libuv file? # return File(RawFD(ccall(:jl_uv_file_handle,Int32,(Ptr{Void},),handle))) else if t == UV_TTY @@ -281,21 +284,12 @@ function init_stdio(handle) end ret.status = StatusOpen ret.line_buffered = false - associate_julia_struct(ret.handle,ret) - finalizer(ret,uvfinalize) + associate_julia_struct(ret.handle, ret) + finalizer(ret, uvfinalize) return ret end end -function stream_wait(x, c...) # for x::LibuvObject - preserve_handle(x) - try - return wait(c...) - finally - unpreserve_handle(x) - end -end - function reinit_stdio() global uv_jl_asynccb = cfunction(uv_asynccb, Void, (Ptr{Void},)) global uv_jl_timercb = cfunction(uv_timercb, Void, (Ptr{Void},)) @@ -318,20 +312,20 @@ function reinit_stdio() global STDERR = init_stdio(ccall(:jl_stderr_stream,Ptr{Void},())) end -function isopen(x::Union{AsyncStream,UVServer}) +function isopen(x::Union{LibuvStream, LibuvServer}) if x.status == StatusUninit || x.status == StatusInit throw(ArgumentError("$x is not initialized")) end x.status != StatusClosed && x.status != StatusEOF end -function check_open(x::Union{AsyncStream,UVServer}) +function check_open(x::Union{LibuvStream, LibuvServer}) if !isopen(x) || x.status == StatusClosing throw(ArgumentError("stream is closed or unusable")) end end -function wait_connected(x) +function wait_connected(x::Union{LibuvStream, LibuvServer}) check_open(x) while x.status == StatusConnecting stream_wait(x, x.connectnotify) @@ -339,10 +333,10 @@ function wait_connected(x) end end -function wait_readbyte(x::AsyncStream, c::UInt8) +function wait_readbyte(x::LibuvStream, c::UInt8) preserve_handle(x) try - while isopen(x) && search(x.buffer,c) <= 0 + while isopen(x) && search(x.buffer, c) <= 0 start_reading(x) # ensure we are reading wait(x.readnotify) end @@ -354,7 +348,7 @@ function wait_readbyte(x::AsyncStream, c::UInt8) end end -function wait_readnb(x::AsyncStream, nb::Int) +function wait_readnb(x::LibuvStream, nb::Int) oldthrottle = x.throttle preserve_handle(x) try @@ -374,16 +368,26 @@ function wait_readnb(x::AsyncStream, nb::Int) end end -function wait_close(x::AsyncStream) +function wait_close(x::Union{LibuvStream, LibuvServer}) if isopen(x) stream_wait(x, x.closenotify) end end +function close(stream::Union{LibuvStream, LibuvServer}) + if isopen(stream) && stream.status != StatusClosing + ccall(:jl_close_uv,Void, (Ptr{Void},), stream.handle) + stream.status = StatusClosing + end + nothing +end + +### Libuv callbacks ### + #from `connect` function uv_connectcb(conn::Ptr{Void}, status::Cint) hand = ccall(:jl_uv_connect_handle, Ptr{Void}, (Ptr{Void},), conn) - sock = @handle_as hand AsyncStream + sock = @handle_as hand LibuvStream @assert sock.status == StatusConnecting if status >= 0 sock.status = StatusOpen @@ -402,16 +406,16 @@ end # from `listen` function uv_connectioncb(stream::Ptr{Void}, status::Cint) - sock = @handle_as stream UVServer + sock = @handle_as stream LibuvServer if status >= 0 err = nothing else err = UVError("connection",status) end - if isa(sock.ccb,Function) - sock.ccb(sock,status) + if isa(sock.ccb, Function) + sock.ccb(sock, status) end - err===nothing ? notify(sock.connectnotify) : notify_error(sock.connectnotify, err) + err === nothing ? notify(sock.connectnotify) : notify_error(sock.connectnotify, err) end ## BUFFER ## @@ -419,7 +423,7 @@ end function alloc_request(buffer::IOBuffer, recommended_size::UInt) ensureroom(buffer, Int(recommended_size)) ptr = buffer.append ? buffer.size + 1 : buffer.ptr - return (pointer(buffer.data, ptr), length(buffer.data)-ptr+1) + return (pointer(buffer.data, ptr), length(buffer.data) - ptr + 1) end function uv_alloc_buf(handle::Ptr{Void}, size::Csize_t, buf::Ptr{Void}) @@ -428,9 +432,9 @@ function uv_alloc_buf(handle::Ptr{Void}, size::Csize_t, buf::Ptr{Void}) ccall(:jl_uv_buf_set_len, Void, (Ptr{Void}, Csize_t), buf, 0) return nothing end - stream = unsafe_pointer_to_objref(hd)::AsyncStream + stream = unsafe_pointer_to_objref(hd)::LibuvStream - (data,newsize) = alloc_buf_hook(stream, UInt(size)) + (data, newsize) = alloc_buf_hook(stream, UInt(size)) ccall(:jl_uv_buf_set_base, Void, (Ptr{Void}, Ptr{Void}), buf, data) ccall(:jl_uv_buf_set_len, Void, (Ptr{Void}, Csize_t), buf, newsize) @@ -438,7 +442,7 @@ function uv_alloc_buf(handle::Ptr{Void}, size::Csize_t, buf::Ptr{Void}) nothing end -alloc_buf_hook(stream::AsyncStream, size::UInt) = alloc_request(stream.buffer, UInt(size)) +alloc_buf_hook(stream::LibuvStream, size::UInt) = alloc_request(stream.buffer, UInt(size)) function notify_filled(buffer::IOBuffer, nread::Int, base::Ptr{Void}, len::UInt) if buffer.append @@ -447,7 +451,8 @@ function notify_filled(buffer::IOBuffer, nread::Int, base::Ptr{Void}, len::UInt) buffer.ptr += nread end end -function notify_filled(stream::AsyncStream, nread::Int) + +function notify_filled(stream::LibuvStream, nread::Int) more = true while more if isa(stream.readcb,Function) @@ -464,7 +469,7 @@ function notify_filled(stream::AsyncStream, nread::Int) end function uv_readcb(handle::Ptr{Void}, nread::Cssize_t, buf::Ptr{Void}) - stream = @handle_as handle AsyncStream + stream = @handle_as handle LibuvStream nread = Int(nread) base = ccall(:jl_uv_buf_base, Ptr{Void}, (Ptr{Void},), buf) len = UInt(ccall(:jl_uv_buf_len, Csize_t, (Ptr{Void},), buf)) @@ -474,7 +479,7 @@ function uv_readcb(handle::Ptr{Void}, nread::Cssize_t, buf::Ptr{Void}) # remind the client that stream.buffer is full notify(stream.readnotify) elseif nread == UV_EOF - if isa(stream,TTY) + if isa(stream, TTY) stream.status = StatusEOF # libuv called stop_reading already notify(stream.readnotify) notify(stream.closenotify) @@ -484,7 +489,7 @@ function uv_readcb(handle::Ptr{Void}, nread::Cssize_t, buf::Ptr{Void}) else # This is a fatal connection error. Shutdown requests as per the usual # close function won't work and libuv will fail with an assertion failure - ccall(:jl_forceclose_uv,Void,(Ptr{Void},),stream.handle) + ccall(:jl_forceclose_uv, Void, (Ptr{Void},), stream) notify_error(stream.readnotify, UVError("readcb",nread)) end else @@ -503,7 +508,6 @@ function uv_readcb(handle::Ptr{Void}, nread::Cssize_t, buf::Ptr{Void}) nothing end -reseteof(x::IO) = nothing function reseteof(x::TTY) if x.status == StatusEOF x.status = StatusOpen @@ -511,7 +515,7 @@ function reseteof(x::TTY) nothing end -function _uv_hook_close(uv::Union{AsyncStream,UVServer}) +function _uv_hook_close(uv::Union{LibuvStream, LibuvServer}) uv.handle = C_NULL uv.status = StatusClosed if isa(uv.closecb, Function) @@ -526,10 +530,10 @@ end ########################################## # Pipe Abstraction -# (composed of two half-pipes) +# (composed of two half-pipes: .in and .out) ########################################## -abstract AbstractPipe <: AsyncStream +abstract AbstractPipe <: IO # allows sharing implementation with Process and ProcessChain type Pipe <: AbstractPipe @@ -549,6 +553,7 @@ show(io::IO,stream::Pipe) = print(io, uv_status_string(stream.in), " => ", uv_status_string(stream.out), ", ", nb_available(stream), " bytes waiting)") + write(io::AbstractPipe, byte::UInt8) = write(io.in, byte) write(io::AbstractPipe, bytes::Vector{UInt8}) = write(io.in, bytes) write{T<:AbstractPipe}(io::T, args...) = write(io.in, args...) @@ -711,8 +716,26 @@ function process_events(block::Bool) end end -## pipe functions ## -function malloc_julia_pipe!(x) +## Functions for PipeEndpoint and PipeServer ## + +function init_pipe!(pipe::LibuvPipe; + readable::Bool = false, + writable::Bool = false, + julia_only::Bool = true) + if pipe.status != StatusUninit + error("pipe is already initialized") + end + if pipe.handle == C_NULL + malloc_julia_pipe!(pipe) + end + uv_error("init_pipe",ccall(:jl_init_pipe, Cint, + (Ptr{Void}, Int32, Int32, Int32), + pipe.handle, writable, readable, julia_only)) + pipe.status = StatusInit + pipe +end + +function malloc_julia_pipe!(x::LibuvPipe) assert(x.handle == C_NULL) x.handle = Libc.malloc(_sizeof_uv_named_pipe) associate_julia_struct(x.handle, x) @@ -726,7 +749,7 @@ end function link_pipe(read_end::Ptr{Void}, readable_julia_only::Bool, write_end::Ptr{Void}, writable_julia_only::Bool, - readpipe::AsyncStream, writepipe::AsyncStream) + readpipe::PipeEndpoint, writepipe::PipeEndpoint) #make the pipe an unbuffered stream for now #TODO: this is probably not freeing memory properly after errors uv_error("init_pipe(read)", @@ -794,26 +817,20 @@ function close_pipe_sync(p::PipeEndpoint) p.status = StatusClosed nothing end -function close_pipe_sync(handle::UVHandle) - ccall(:uv_pipe_close_sync, Void, (UVHandle,), handle) -end -function close(stream::Union{AsyncStream, UVServer}) - if isopen(stream) && stream.status != StatusClosing - ccall(:jl_close_uv,Void, (Ptr{Void},), stream.handle) - stream.status = StatusClosing - end - nothing +function close_pipe_sync(handle::Ptr{Void}) + ccall(:uv_pipe_close_sync, Void, (Ptr{Void},), handle) end -## stream functions ## -function start_reading(stream::AsyncStream) +## Functions for any LibuvStream ## + +function start_reading(stream::LibuvStream) if stream.status == StatusOpen if !isreadable(stream) error("tried to read a stream that is not readable") end - ret = ccall(:uv_read_start,Cint,(Ptr{Void},Ptr{Void},Ptr{Void}), - handle(stream),uv_jl_alloc_buf::Ptr{Void},uv_jl_readcb::Ptr{Void}) + ret = ccall(:uv_read_start, Cint, (Ptr{Void}, Ptr{Void}, Ptr{Void}), + stream, uv_jl_alloc_buf::Ptr{Void}, uv_jl_readcb::Ptr{Void}) stream.status = StatusActive ret elseif stream.status == StatusActive @@ -822,7 +839,8 @@ function start_reading(stream::AsyncStream) Int32(-1) end end -function start_reading(stream::AsyncStream, cb::Function) + +function start_reading(stream::LibuvStream, cb::Function) failure = start_reading(stream) stream.readcb = cb nread = nb_available(stream.buffer) @@ -831,15 +849,16 @@ function start_reading(stream::AsyncStream, cb::Function) end return failure_code end -function start_reading(stream::AsyncStream, cb::Bool) + +function start_reading(stream::LibuvStream, cb::Bool) failure_code = start_reading(stream) stream.readcb = cb return failure_code end -function stop_reading(stream::AsyncStream) +function stop_reading(stream::LibuvStream) if stream.status == StatusActive - ret = ccall(:uv_read_stop,Cint,(Ptr{Void},),stream.handle) + ret = ccall(:uv_read_stop, Cint, (Ptr{Void},), stream) stream.status = StatusOpen ret elseif stream.status == StatusOpen @@ -849,20 +868,12 @@ function stop_reading(stream::AsyncStream) end end -function readbytes(stream::AsyncStream) +function readbytes(stream::LibuvStream) wait_readnb(stream, typemax(Int)) return takebuf_array(stream.buffer) end -function read!{T}(s::AsyncStream, a::Array{T}) - isbits(T) || throw(ArgumentError("read from AsyncStream only supports bits types or arrays of bits types")) - nb = length(a) * sizeof(T) - read!(s, reshape(reinterpret(UInt8, a), nb)) - return a -end - -const SZ_UNBUFFERED_IO=65536 -function read!(s::AsyncStream, a::Vector{UInt8}) +function read!(s::LibuvStream, a::Array{UInt8, 1}) nb = length(a) sbuf = s.buffer @assert sbuf.seekable == false @@ -893,66 +904,44 @@ function read!(s::AsyncStream, a::Vector{UInt8}) return a end -function read{T}(s::AsyncStream, ::Type{T}, dims::Dims) - isbits(T) || throw(ArgumentError("read from AsyncStream only supports bits types or arrays of bits types")) - nb = prod(dims)*sizeof(T) - a = read!(s, Array(UInt8, nb)) - reshape(reinterpret(T, a), dims) -end - -function read(this::AsyncStream,::Type{UInt8}) +function read(this::LibuvStream, ::Type{UInt8}) + wait_readnb(this, 1) buf = this.buffer @assert buf.seekable == false - wait_readnb(this, 1) read(buf, UInt8) end -readline(this::AsyncStream) = readuntil(this, '\n') - -readline() = readline(STDIN) - -function readavailable(this::AsyncStream) +function readavailable(this::LibuvStream) + wait_readnb(this, 1) buf = this.buffer @assert buf.seekable == false - wait_readnb(this, 1) takebuf_array(buf) end -function readuntil(this::AsyncStream, c::UInt8) +function readuntil(this::LibuvStream, c::UInt8) + wait_readbyte(this, c) buf = this.buffer @assert buf.seekable == false - wait_readbyte(this, c) readuntil(buf, c) end -#function finish_read(pipe::PipeEndpoint) -# close(pipe) #handles to UV and ios will be invalid after this point -#end -# -#function finish_read(state::(PipeEndpoint,ByteString)) -# finish_read(state...) -#end - -function uv_write(s::AsyncStream, p, n::Integer) +uv_write(s::LibuvStream, p::Vector{UInt8}) = uv_write(s, pointer(p), UInt(length(p))) +function uv_write(s::LibuvStream, p::Ptr, n::UInt) check_open(s) uvw = Libc.malloc(_sizeof_uv_write) - try - uv_req_set_data(uvw,C_NULL) - err = ccall(:jl_uv_write, - Int32, - (Ptr{Void}, Ptr{Void}, UInt, Ptr{Void}, Ptr{Void}), - handle(s), p, n, uvw, - uv_jl_writecb_task::Ptr{Void}) - if err < 0 - uv_error("write", err) - end - ct = current_task() - uv_req_set_data(uvw,ct) - ct.state = :waiting - stream_wait(ct) - finally - Libc.free(uvw) + uv_req_set_data(uvw,C_NULL) + err = ccall(:jl_uv_write, + Int32, + (Ptr{Void}, Ptr{Void}, UInt, Ptr{Void}, Ptr{Void}), + s, p, n, uvw, + uv_jl_writecb_task::Ptr{Void}) + if err < 0 + uv_error("write", err) end + ct = current_task() + uv_req_set_data(uvw,ct) + ct.state = :waiting + stream_wait(ct) return Int(n) end @@ -960,13 +949,12 @@ end # - smaller writes are buffered, final uv write on flush or when buffer full # - large isbits arrays are unbuffered and written directly -function buffer_or_write(s::AsyncStream, p::Ptr, n::Integer) +function buffer_or_write(s::LibuvStream, p::Ptr, n::Integer) if isnull(s.sendbuf) - return uv_write(s, p, n) - else - buf = get(s.sendbuf) + return uv_write(s, p, UInt(n)) end + buf = get(s.sendbuf) totb = nb_available(buf) + n if totb < buf.maxsize nb = write(buf, p, n) @@ -981,39 +969,37 @@ function buffer_or_write(s::AsyncStream, p::Ptr, n::Integer) return nb end -function flush(s::AsyncStream) +function flush(s::LibuvStream) if isnull(s.sendbuf) return s end buf = get(s.sendbuf) if nb_available(buf) > 0 arr = takebuf_array(buf) # Array of UInt8s - uv_write(s, arr, length(arr)) + uv_write(s, arr) end s end -buffer_writes(s::AsyncStream, bufsize=SZ_UNBUFFERED_IO) = (s.sendbuf=PipeBuffer(bufsize); s) +buffer_writes(s::LibuvStream, bufsize) = (s.sendbuf=PipeBuffer(bufsize); s) -## low-level calls ## +## low-level calls to libuv ## -write(s::AsyncStream, b::UInt8) = write(s, [b]) -write(s::AsyncStream, c::Char) = write(s, string(c)) -function write{T}(s::AsyncStream, a::Array{T}) +write(s::LibuvStream, b::UInt8) = write(s, [b]) +write(s::LibuvStream, c::Char) = write(s, string(c)) +function write{T}(s::LibuvStream, a::Array{T}) if isbits(T) - n = UInt(length(a)*sizeof(T)) - return buffer_or_write(s, pointer(a), n); + n = UInt(length(a) * sizeof(T)) + return buffer_or_write(s, pointer(a), n) else check_open(s) - invoke(write, Tuple{IO, Array},s,a) + invoke(write, Tuple{IO, typeof(a)}, s, a) end end -write(s::AsyncStream, p::Ptr, n::Integer) = buffer_or_write(s, p, n) +write(s::LibuvStream, p::Ptr, n::Integer) = buffer_or_write(s, p, n) function uv_writecb_task(req::Ptr{Void}, status::Cint) - #handle = ccall(:jl_uv_write_handle, Ptr{Void}, (Ptr{Void},), req) - #s = @handle_as handle AsyncStream d = uv_req_data(req) @assert d != C_NULL if status < 0 @@ -1022,10 +1008,12 @@ function uv_writecb_task(req::Ptr{Void}, status::Cint) else schedule(unsafe_pointer_to_objref(d)::Task) end + Libc.free(req) nothing end ## Libuv error handling ## + type UVError <: Exception prefix::AbstractString code::Int32 @@ -1059,7 +1047,7 @@ function accept_nonblock(server::PipeServer) client end -function accept(server::UVServer, client::AsyncStream) +function accept(server::LibuvServer, client::LibuvStream) if server.status != StatusActive throw(ArgumentError("server not connected, make sure \"listen\" has been called")) end @@ -1077,10 +1065,10 @@ end const BACKLOG_DEFAULT = 511 -function _listen(sock::UVServer; backlog::Integer=BACKLOG_DEFAULT) +function _listen(sock::LibuvServer; backlog::Integer=BACKLOG_DEFAULT) check_open(sock) err = ccall(:uv_listen, Cint, (Ptr{Void}, Cint, Ptr{Void}), - sock.handle, backlog, uv_jl_connectioncb::Ptr{Void}) + sock, backlog, uv_jl_connectioncb::Ptr{Void}) sock.status = StatusActive err end @@ -1088,7 +1076,7 @@ end function bind(server::PipeServer, name::AbstractString) @assert server.status == StatusInit err = ccall(:uv_pipe_bind, Int32, (Ptr{Void}, Cstring), - server.handle, name) + server, name) if err != 0 if err != UV_EADDRINUSE && err != UV_EACCES #TODO: this codepath is currently not tested @@ -1118,8 +1106,8 @@ function connect!(sock::PipeEndpoint, path::AbstractString) sock end -function connect(sock::AsyncStream, args...) - connect!(sock,args...) +function connect(sock::LibuvStream, args...) + connect!(sock, args...) wait_connected(sock) sock end @@ -1129,8 +1117,8 @@ end connect(path::AbstractString) = connect(init_pipe!(PipeEndpoint(); readable=false, writable=false, julia_only=true),path) _fd(x::IOStream) = RawFD(fd(x)) -@unix_only _fd(x::AsyncStream) = RawFD(ccall(:jl_uv_handle,Int32,(Ptr{Void},),x.handle)) -@windows_only _fd(x::AsyncStream) = WindowsRawSocket( +@unix_only _fd(x::LibuvStream) = RawFD(ccall(:jl_uv_handle,Int32,(Ptr{Void},),x.handle)) +@windows_only _fd(x::LibuvStream) = WindowsRawSocket( ccall(:jl_uv_handle,Ptr{Void},(Ptr{Void},),x.handle)) for (x,writable,unix_fd,c_symbol) in ((:STDIN,false,0,:jl_uv_stdin),(:STDOUT,true,1,:jl_uv_stdout),(:STDERR,true,2,:jl_uv_stderr)) @@ -1141,11 +1129,11 @@ for (x,writable,unix_fd,c_symbol) in ((:STDIN,false,0,:jl_uv_stdin),(:STDOUT,tru global $x @windows? ( ccall(:SetStdHandle,stdcall,Int32,(Int32,Ptr{Void}), - $(-10-unix_fd), Libc._get_osfhandle(_fd(stream)).handle) : + $(-10-unix_fd), Libc._get_osfhandle(_fd(stream)).handle) ) : ( dup(_fd(stream), RawFD($unix_fd)) ) $x = stream end - function ($f)(handle::Union{AsyncStream,IOStream}) + function ($f)(handle::Union{LibuvStream,IOStream}) $(_f)(handle) unsafe_store!(cglobal($(Expr(:quote,c_symbol)),Ptr{Void}), handle.handle) @@ -1160,13 +1148,13 @@ for (x,writable,unix_fd,c_symbol) in ((:STDIN,false,0,:jl_uv_stdin),(:STDOUT,tru end end -mark(x::AsyncStream) = mark(x.buffer) -unmark(x::AsyncStream) = unmark(x.buffer) -reset(x::AsyncStream) = reset(x.buffer) -ismarked(x::AsyncStream) = ismarked(x.buffer) +mark(x::LibuvStream) = mark(x.buffer) +unmark(x::LibuvStream) = unmark(x.buffer) +reset(x::LibuvStream) = reset(x.buffer) +ismarked(x::LibuvStream) = ismarked(x.buffer) # BufferStream's are non-OS streams, backed by a regular IOBuffer -type BufferStream <: AsyncStream +type BufferStream <: LibuvStream buffer::IOBuffer r_c::Condition close_c::Condition @@ -1177,10 +1165,14 @@ type BufferStream <: AsyncStream BufferStream() = new(PipeBuffer(), Condition(), Condition(), true, false, ReentrantLock()) end -lock(s::BufferStream) = lock(s.lock) -unlock(s::BufferStream) = unlock(s.unlock) isopen(s::BufferStream) = s.is_open close(s::BufferStream) = (s.is_open = false; notify(s.r_c; all=true); notify(s.close_c; all=true); nothing) +read(s::BufferStream, ::Type{UInt8}) = (wait_readnb(s, 1); read(s.buffer, UInt8)) +read!(s::BufferStream, a::Vector{UInt8}) = (wait_readnb(s, length(a)); read!(s.buffer, a)) +nb_available(s::BufferStream) = nb_available(s.buffer) + +isreadable(s::BufferStream) = s.buffer.readable +iswritable(s::BufferStream) = s.buffer.writable function wait_readnb(s::BufferStream, nb::Int) while isopen(s) && nb_available(s.buffer) < nb @@ -1213,6 +1205,11 @@ function write(s::BufferStream, p::Ptr, nb::Integer) rv end +function eof(s::LibuvStream) + wait_readnb(s,1) + !isopen(s) && nb_available(s)<=0 +end + # If buffer_writes is called, it will delay notifying waiters till a flush is called. buffer_writes(s::BufferStream, bufsize=0) = (s.buffer_writes=true; s) flush(s::BufferStream) = (notify(s.r_c; all=true); s) diff --git a/base/strings/io.jl b/base/strings/io.jl index 6bef21cbaa6db..e78cb5ddcd6ce 100644 --- a/base/strings/io.jl +++ b/base/strings/io.jl @@ -2,8 +2,25 @@ ## core text I/O ## -print(io::IO, x) = show(io, x) -print(io::IO, xs...) = for x in xs print(io, x) end +function print(io::IO, x) + lock(io) + try + show(io, x) + finally + unlock(io) + end +end + +function print(io::IO, xs...) + lock(io) + try + for x in xs + print(io, x) + end + finally + unlock(io) + end +end println(io::IO, xs...) = print(io, xs..., '\n') diff --git a/test/spawn.jl b/test/spawn.jl index 1856df40147d1..a0a4083bda9ae 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -249,13 +249,49 @@ let bad = "bad\0name" end # issue #12829 -let out = Pipe() +let out = Pipe(), echo = `$exename -f -e 'print(STDOUT, " 1\t", readall(STDIN))'`, ready = Condition() @test_throws ArgumentError write(out, "not open error") - open(`cat`, "w", out) do io - println(io, 1) + @async begin # spawn writer task + open(echo, "w", out) do in1 + open(echo, "w", out) do in2 + notify(ready) + write(in1, 'h') + write(in2, UInt8['w']) + println(in1, "ello") + write(in2, "orld\n") + end + end + show(out, out) + notify(ready) + @test isreadable(out) + @test iswritable(out) + close(out.in) + @test_throws ArgumentError write(out, "now closed error") + @test isreadable(out) + @test !iswritable(out) + @test isopen(out) end - close(out.in) - @test readline(out) == "1\n" + wait(ready) # wait for writer task to be ready before using `out` + @test nb_available(out) == 0 + @test endswith(readuntil(out, '1'), '1') + @test read(out, UInt8) == '\t' + c = UInt8[0] + @test c == read!(out, c) + Base.wait_readnb(out, 1) + @test nb_available(out) > 0 + ln1 = readline(out) + ln2 = readline(out) + desc = readall(out) + @test !isreadable(out) + @test !iswritable(out) + @test !isopen(out) + @test nb_available(out) == 0 + @test c == ['w'] + @test lstrip(ln2) == "1\thello\n" + @test ln1 == "orld\n" + @test isempty(readbytes(out)) + @test eof(out) + @test desc == "Pipe(open => active, 0 bytes waiting)" end # issue #8529 From 780d9afac6c2575192e85fdca9994d6ec2c80a78 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 12 Sep 2015 19:16:18 -0400 Subject: [PATCH 0079/1938] add deprecation for AsyncStream --- base/deprecated.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/base/deprecated.jl b/base/deprecated.jl index 156b86e5981a1..79dc092d81762 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -787,3 +787,6 @@ export FloatingPoint end @deprecate cartesianmap(f, dims) for idx in CartesianRange(dims); f(idx.I...); end + +# 12839 +const AsyncStream = IO From 382548f015b4be4abb39c8a1c17ca17ebba2d06d Mon Sep 17 00:00:00 2001 From: "David P. Sanders" Date: Sun, 13 Sep 2015 17:15:45 -0500 Subject: [PATCH 0080/1938] Inline Latex for A_mul_Bt etc. --- base/docs/helpdb.jl | 40 ++++++++++++++++++++-------------------- doc/stdlib/math.rst | 38 +++++++++++++++++++------------------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 74cc08427d8fb..51c4edcbe3d6f 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -1041,7 +1041,7 @@ Base.(:(//)) doc""" At_mul_B(A, B) -For matrices or vectors `A` and `B`, calculates `Aᵀ B` +For matrices or vectors $A$ and $B$, calculates $Aᵀ⋅B$ """ At_mul_B @@ -1360,7 +1360,7 @@ airybiprime doc""" Ac_rdiv_B(A, B) -For matrices or vectors `A` and `B`, calculates `Aᴴ / B` +For matrices or vectors $A$ and $B$, calculates $Aᴴ / B$ """ Ac_rdiv_B @@ -2528,7 +2528,7 @@ redisplay doc""" A_mul_Bc(A, B) -For matrices or vectors `A` and `B`, calculates `A Bᴴ` +For matrices or vectors $A$ and $B$, calculates $A⋅Bᴴ$ """ A_mul_Bc @@ -2662,7 +2662,7 @@ watch_file doc""" At_rdiv_Bt(A, B) -For matrices or vectors `A` and `B`, calculates `Aᵀ / Bᵀ` +For matrices or vectors $A$ and $B$, calculates $Aᵀ / Bᵀ$ """ At_rdiv_Bt @@ -2676,7 +2676,7 @@ isinteractive doc""" At_mul_Bt(A, B) -For matrices or vectors `A` and `B`, calculates `Aᵀ Bᵀ` +For matrices or vectors $A$ and $B$, calculates $Aᵀ⋅Bᵀ$ """ At_mul_Bt @@ -3429,7 +3429,7 @@ Mmap.Anonymous doc""" A_rdiv_Bc(A, B) -For matrices or vectors `A` and `B`, calculates `A / Bᴴ` +For matrices or vectors $A$ and $B$, calculates $A / Bᴴ$ """ A_rdiv_Bc @@ -6457,8 +6457,8 @@ doc""" A_mul_B!(Y, A, B) -> Y -Calculates the matrix-matrix or matrix-vector product `A * B` and stores the -result in `Y`, overwriting the existing value of `Y`. +Calculates the matrix-matrix or matrix-vector product $A⋅B$ and stores the +result in $Y$, overwriting the existing value of $Y$. ```jldoctest julia> A=[1.0 2.0; 3.0 4.0]; B=[1.0 1.0; 1.0 1.0]; A_mul_B!(B, A, B); @@ -6490,7 +6490,7 @@ idct! doc""" Ac_rdiv_Bc(A, B) -For matrices or vectors `A` and `B`, calculates `Aᴴ \ Bᴴ` +For matrices or vectors $A$ and $B$, calculates $Aᴴ / Bᴴ$ """ Ac_rdiv_Bc @@ -8789,7 +8789,7 @@ plan_fft doc""" A_rdiv_Bt(A, B) -For matrices or vectors `A` and `B`, calculates `A / Bᵀ` +For matrices or vectors $A$ and $B$, calculates $A / Bᵀ$ """ A_rdiv_Bt @@ -9549,7 +9549,7 @@ issubnormal doc""" Ac_ldiv_B(A, B) -For matrices or vectors `A` and `B`, calculates `Aᴴ \ B` +For matrices or vectors $A$ and $B$, calculates $Aᴴ$ \ $B$ """ Ac_ldiv_B @@ -9807,7 +9807,7 @@ nprocs doc""" Ac_mul_B(A, B) -For matrices or vectors `A` and `B`, calculates `Aᴴ B` +For matrices or vectors $A$ and $B$, calculates $Aᴴ⋅B$ """ Ac_mul_B @@ -9823,7 +9823,7 @@ qrfact! doc""" At_rdiv_B(A, B) -For matrices or vectors `A` and `B`, calculates `Aᵀ / B` +For matrices or vectors $A$ and $B$, calculates $Aᵀ / B$ """ At_rdiv_B @@ -10072,7 +10072,7 @@ convert doc""" A_ldiv_Bt(A, B) -For matrices or vectors `A` and `B`, calculates `A \ Bᵀ` +For matrices or vectors $A$ and $B$, calculates $A$ \ $Bᵀ$ """ A_ldiv_Bt @@ -10154,7 +10154,7 @@ eigvals doc""" A_ldiv_Bc(A, B) -For matrices or vectors `A` and `B`, calculates `A \ Bᴴ` +For matrices or vectors $A$ and $B$, calculates $A$ \ $Bᴴ$ """ A_ldiv_Bc @@ -10786,14 +10786,14 @@ getkey doc""" At_ldiv_Bt(A, B) -For matrices or vectors `A` and `B`, calculates `Aᵀ \ Bᵀ` +For matrices or vectors $A$ and $B$, calculates $Aᵀ$ \ $Bᵀ$ """ At_ldiv_Bt doc""" Ac_mul_Bc(A, B) -For matrices or vectors `A` and `B`, calculates `Aᴴ Bᴴ` +For matrices or vectors $A$ and $B$, calculates $Aᴴ Bᴴ$ """ Ac_mul_Bc @@ -10852,7 +10852,7 @@ sprand doc""" A_mul_Bt(A, B) -For matrices or vectors `A` and `B`, calculates `A Bᵀ` +For matrices or vectors $A$ and $B$, calculates $A⋅Bᵀ$ """ A_mul_Bt @@ -11388,7 +11388,7 @@ union! doc""" At_ldiv_B(A, B) -For matrices or vectors `A` and `B`, calculates `Aᵀ \ B` +For matrices or vectors $A$ and $B$, calculates $Aᵀ$ \ $B$ """ At_ldiv_B @@ -11649,7 +11649,7 @@ ror doc""" Ac_ldiv_Bc(A, B) -For matrices or vectors `A` and `B`, calculates `Aᴴ \ Bᴴ` +For matrices or vectors $A$ and $B$, calculates $Aᴴ$ \ $Bᴴ$ """ Ac_ldiv_Bc diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 52390aca95797..b17c8b2d6d85a 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -413,19 +413,19 @@ Mathematical Operators .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``A \ Bᴴ`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A` \\ :math:`Bᴴ` .. function:: A_ldiv_Bt(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``A \ Bᵀ`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A` \\ :math:`Bᵀ` .. function:: A_mul_B!(Y, A, B) -> Y .. Docstring generated from Julia source - Calculates the matrix-matrix or matrix-vector product ``A * B`` and stores the result in ``Y``\ , overwriting the existing value of ``Y``\ . + Calculates the matrix-matrix or matrix-vector product :math:`A⋅B` and stores the result in :math:`Y`\ , overwriting the existing value of :math:`Y`\ . .. doctest:: @@ -440,97 +440,97 @@ Mathematical Operators .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``A Bᴴ`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A⋅Bᴴ` .. function:: A_mul_Bt(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``A Bᵀ`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A⋅Bᵀ` .. function:: A_rdiv_Bc(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``A / Bᴴ`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A / Bᴴ` .. function:: A_rdiv_Bt(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``A / Bᵀ`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A / Bᵀ` .. function:: Ac_ldiv_B(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``Aᴴ \ B`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ` \\ :math:`B` .. function:: Ac_ldiv_Bc(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``Aᴴ \ Bᴴ`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ` \\ :math:`Bᴴ` .. function:: Ac_mul_B(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``Aᴴ B`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ⋅B` .. function:: Ac_mul_Bc(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``Aᴴ Bᴴ`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ Bᴴ` .. function:: Ac_rdiv_B(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``Aᴴ / B`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ / B` .. function:: Ac_rdiv_Bc(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``Aᴴ \ Bᴴ`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ / Bᴴ` .. function:: At_ldiv_B(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``Aᵀ \ B`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ` \\ :math:`B` .. function:: At_ldiv_Bt(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``Aᵀ \ Bᵀ`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ` \\ :math:`Bᵀ` .. function:: At_mul_B(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``Aᵀ B`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ⋅B` .. function:: At_mul_Bt(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``Aᵀ Bᵀ`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ⋅Bᵀ` .. function:: At_rdiv_B(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``Aᵀ / B`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ / B` .. function:: At_rdiv_Bt(A, B) .. Docstring generated from Julia source - For matrices or vectors ``A`` and ``B``\ , calculates ``Aᵀ / Bᵀ`` + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ / Bᵀ` Mathematical Functions ---------------------- From c4407394c84fd2030120d156d42235fd4caef56f Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Fri, 11 Sep 2015 13:34:52 -0700 Subject: [PATCH 0081/1938] Move ccalltest.c to src, install into private_libdir, ensure symbols are public --- Makefile | 4 +-- src/Makefile | 10 ++++++- {test => src}/ccalltest.c | 57 ++++++++++++++++++++------------------- test/Makefile | 10 +------ test/ccall.jl | 2 +- test/libdl.jl | 28 ++++++++++--------- 6 files changed, 58 insertions(+), 53 deletions(-) rename {test => src}/ccalltest.c (84%) diff --git a/Makefile b/Makefile index 5d8ba3a7f3b3f..026f09e346009 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ julia-base: julia-deps $(build_sysconfdir)/julia/juliarc.jl $(build_man1dir)/jul @$(MAKE) $(QUIET_MAKE) -C base julia-libccalltest: - @$(MAKE) $(QUIET_MAKE) -C test libccalltest + @$(MAKE) $(QUIET_MAKE) -C src libccalltest julia-src-release julia-src-debug : julia-src-% : julia-deps @$(MAKE) $(QUIET_MAKE) -C src libjulia-$* @@ -202,7 +202,7 @@ $(build_bindir)/stringreplace: contrib/stringreplace.c | $(build_bindir) JL_LIBS = julia julia-debug # private libraries, that are installed in $(prefix)/lib/julia -JL_PRIVATE_LIBS = suitesparse_wrapper Rmath +JL_PRIVATE_LIBS = suitesparse_wrapper Rmath ccalltest ifeq ($(USE_SYSTEM_FFTW),0) JL_PRIVATE_LIBS += fftw3 fftw3f fftw3_threads fftw3f_threads endif diff --git a/src/Makefile b/src/Makefile index f33e562c58d40..101a664388f98 100644 --- a/src/Makefile +++ b/src/Makefile @@ -82,6 +82,14 @@ $(BUILDDIR)/%.o: %.cpp $(HEADERS) $(shell which $(LLVM_CONFIG)) | $(BUILDDIR) $(BUILDDIR)/%.dbg.obj: %.cpp $(HEADERS) $(shell which $(LLVM_CONFIG)) | $(BUILDDIR) @$(call PRINT_CC, $(CXX) $(call exec,$(LLVM_CONFIG) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(DEBUGFLAGS) -c $< -o $@) +libccalltest: $(build_shlibdir)/libccalltest.$(SHLIB_EXT) +$(build_shlibdir)/libccalltest.$(SHLIB_EXT): ccalltest.c + @$(call PRINT_CC, $(CC) $(CFLAGS) $(CPPFLAGS) $(DEBUGFLAGS) -O3 $< $(fPIC) -shared -o $@ $(LDFLAGS) -DCC="$(CC)") +$(BUILDDIR)/ccalltest: ccalltest.c $(BUILDDIR)/libccalltest.$(SHLIB_EXT) + @$(call PRINT_CC, $(CC) $(CFLAGS) $(DEBUGFLAGS) -O3 $< -o $@ $(LDFLAGS) -DCC="$(CC)") + + + $(BUILDDIR)/julia_flisp.boot.inc: $(BUILDDIR)/julia_flisp.boot $(FLISP_EXECUTABLE) @$(call PRINT_FLISP, $(call spawn,$(FLISP_EXECUTABLE)) ./bin2hex.scm < $< > $@) @@ -157,7 +165,7 @@ $(BUILDDIR)/libjulia.a: julia.expmap $(OBJS) $(BUILDDIR)/flisp/libflisp.a $(BUIL libjulia-release: $(build_shlibdir)/libjulia.$(SHLIB_EXT) clean: - -rm -f $(build_shlibdir)/libjulia* + -rm -f $(build_shlibdir)/libjulia* $(build_shlibdir)/libccalltest* -rm -f $(BUILDDIR)/julia_flisp.boot $(BUILDDIR)/julia_flisp.boot.inc -rm -f $(BUILDDIR)/*.dbg.obj $(BUILDDIR)/*.o *~ $(BUILDDIR)/*.$(SHLIB_EXT) $(BUILDDIR)/*.a *# -rm -f $(BUILDDIR)/julia_version.h diff --git a/test/ccalltest.c b/src/ccalltest.c similarity index 84% rename from test/ccalltest.c rename to src/ccalltest.c index ef16718ef3ca4..049eeb105f099 100644 --- a/test/ccalltest.c +++ b/src/ccalltest.c @@ -8,6 +8,7 @@ int verbose = 1; #include "../src/support/platform.h" +#include "../src/support/dtypes.h" #ifdef _P64 #define jint int64_t @@ -26,7 +27,7 @@ int __declspec(noinline) #else int __attribute__((noinline)) #endif -testUcharX(unsigned char x) { +DLLEXPORT testUcharX(unsigned char x) { return xs[x]; } @@ -45,41 +46,41 @@ typedef struct { jint imag; } complex_t; -complex_t ctest(complex_t a) { +DLLEXPORT complex_t ctest(complex_t a) { a.real += 1; a.imag -= 2; return a; } -complex double cgtest(complex double a) { +DLLEXPORT complex double cgtest(complex double a) { //Unpack a ComplexPair{Float64} struct if (verbose) fprintf(stderr,"%g + %g i\n", creal(a), cimag(a)); a += 1 - (2.0*I); return a; } -complex double* cgptest(complex double *a) { +DLLEXPORT complex double* cgptest(complex double *a) { //Unpack a ComplexPair{Float64} struct if (verbose) fprintf(stderr,"%g + %g i\n", creal(*a), cimag(*a)); *a += 1 - (2.0*I); return a; } -complex float cftest(complex float a) { +DLLEXPORT complex float cftest(complex float a) { //Unpack a ComplexPair{Float32} struct if (verbose) fprintf(stderr,"%g + %g i\n", creal(a), cimag(a)); a += 1 - (2.0*I); return a; } -complex float* cfptest(complex float *a) { +DLLEXPORT complex float* cfptest(complex float *a) { //Unpack a ComplexPair{Float64} struct if (verbose) fprintf(stderr,"%g + %g i\n", creal(*a), cimag(*a)); *a += 1 - (2.0*I); return a; } -complex_t* cptest(complex_t *a) { +DLLEXPORT complex_t* cptest(complex_t *a) { //Unpack a ComplexPair{Int} struct pointer if (verbose) fprintf(stderr,"%lld + %lld i\n", (long long)a->real, (long long)a->imag); a->real += 1; @@ -87,7 +88,7 @@ complex_t* cptest(complex_t *a) { return a; } -complex_t* cptest_static(complex_t *a) { +DLLEXPORT complex_t* cptest_static(complex_t *a) { complex_t *b = (complex_t*)malloc(sizeof(complex_t)); b->real = a->real; b->imag = a->imag; @@ -95,7 +96,7 @@ complex_t* cptest_static(complex_t *a) { } // Native-like data types -char* stest(char *x) { +DLLEXPORT char* stest(char *x) { //Print a character Array if (verbose) fprintf(stderr,"%s\n", x); return x; @@ -203,7 +204,7 @@ typedef struct { char z; } struct_big; -struct1 test_1(struct1 a) { +DLLEXPORT struct1 test_1(struct1 a) { //Unpack a "small" struct { float, double } if (verbose) fprintf(stderr,"%g + %g i\n", a.x, a.y); a.x += 1; @@ -211,7 +212,7 @@ struct1 test_1(struct1 a) { return a; } -struct1 add_1(struct1 a, struct1 b) { +DLLEXPORT struct1 add_1(struct1 a, struct1 b) { // Two small structs struct1 c; c.x = a.x + b.x; @@ -219,7 +220,7 @@ struct1 add_1(struct1 a, struct1 b) { return c; } -struct2a test_2a(struct2a a) { +DLLEXPORT struct2a test_2a(struct2a a) { //Unpack a ComplexPair{Int32} struct if (verbose) fprintf(stderr,"%" PRId32 " + %" PRId32 " i\n", a.x.x, a.y.y); a.x.x += 1; @@ -227,7 +228,7 @@ struct2a test_2a(struct2a a) { return a; } -struct2b test_2b(struct2b a) { +DLLEXPORT struct2b test_2b(struct2b a) { //Unpack a ComplexPair{Int32} struct if (verbose) fprintf(stderr,"%" PRId32 " + %" PRId32 " i\n", a.x, a.y); a.x += 1; @@ -235,7 +236,7 @@ struct2b test_2b(struct2b a) { return a; } -struct3a test_3a(struct3a a) { +DLLEXPORT struct3a test_3a(struct3a a) { //Unpack a ComplexPair{Int64} struct if (verbose) fprintf(stderr,"%" PRId64 " + %" PRId64 " i\n", a.x.x, a.y.y); a.x.x += 1; @@ -243,7 +244,7 @@ struct3a test_3a(struct3a a) { return a; } -struct3b test_3b(struct3b a) { +DLLEXPORT struct3b test_3b(struct3b a) { //Unpack a ComplexPair{Int64} struct if (verbose) fprintf(stderr,"%" PRId64 " + %" PRId64 " i\n", a.x, a.y); a.x += 1; @@ -251,7 +252,7 @@ struct3b test_3b(struct3b a) { return a; } -struct4 test_4(struct4 a) +DLLEXPORT struct4 test_4(struct4 a) { if (verbose) fprintf(stderr,"(%" PRId32 ",%" PRId32 ",%" PRId32 ")\n", a.x, a.y, a.z); a.x += 1; @@ -261,7 +262,7 @@ struct4 test_4(struct4 a) } -struct5 test_5(struct5 a) +DLLEXPORT struct5 test_5(struct5 a) { if (verbose) fprintf(stderr,"(%" PRId32 ",%" PRId32 ",%" PRId32 ",%" PRId32 ")\n", a.x, a.y, a.z, a.a); a.x += 1; @@ -273,7 +274,7 @@ struct5 test_5(struct5 a) } -struct6 test_6(struct6 a) +DLLEXPORT struct6 test_6(struct6 a) { if (verbose) fprintf(stderr,"(%" PRId64 ",%" PRId64 ",%" PRId64 ")\n", a.x, a.y, a.z); a.x += 1; @@ -282,7 +283,7 @@ struct6 test_6(struct6 a) return a; } -struct7 test_7(struct7 a) +DLLEXPORT struct7 test_7(struct7 a) { if (verbose) fprintf(stderr,"(%" PRId64 ",%" PRId8 ")\n", a.x, a.y); a.x += 1; @@ -290,7 +291,7 @@ struct7 test_7(struct7 a) return a; } -struct8 test_8(struct8 a) +DLLEXPORT struct8 test_8(struct8 a) { if (verbose) fprintf(stderr,"(%" PRId32 ",%" PRId8 ")\n", a.x, a.y); a.x += 1; @@ -298,7 +299,7 @@ struct8 test_8(struct8 a) return a; } -struct9 test_9(struct9 a) +DLLEXPORT struct9 test_9(struct9 a) { if (verbose) fprintf(stderr,"(%" PRId32 ",%" PRId16 ")\n", a.x, a.y); a.x += 1; @@ -306,7 +307,7 @@ struct9 test_9(struct9 a) return a; } -struct10 test_10(struct10 a) +DLLEXPORT struct10 test_10(struct10 a) { if (verbose) fprintf(stderr,"(%" PRId8 ",%" PRId8 ",%" PRId8 ",%" PRId8 ")\n", a.x, a.y, a.z, a.a); a.x += 1; @@ -317,7 +318,7 @@ struct10 test_10(struct10 a) return a; } -struct14 test_14(struct14 a) { +DLLEXPORT struct14 test_14(struct14 a) { //The C equivalent of a ComplexPair{Float32} struct (but without special complex ABI) if (verbose) fprintf(stderr,"%g + %g i\n", a.x, a.y); a.x += 1; @@ -325,7 +326,7 @@ struct14 test_14(struct14 a) { return a; } -struct15 test_15(struct15 a) { +DLLEXPORT struct15 test_15(struct15 a) { //The C equivalent of a ComplexPair{Float32} struct (but without special complex ABI) if (verbose) fprintf(stderr,"%g + %g i\n", a.x, a.y); a.x += 1; @@ -334,7 +335,7 @@ struct15 test_15(struct15 a) { } #define int128_t struct3b -int128_t test_128(int128_t a) { +DLLEXPORT int128_t test_128(int128_t a) { //Unpack a Int128 if (verbose) fprintf(stderr,"0x%016" PRIx64 "%016" PRIx64 "\n", a.y, a.x); a.x += 1; @@ -343,7 +344,7 @@ int128_t test_128(int128_t a) { return a; } -struct_big test_big(struct_big a) { +DLLEXPORT struct_big test_big(struct_big a) { //Unpack a "big" struct { int, int, char } if (verbose) fprintf(stderr,"%lld %lld %c\n", (long long)a.x, (long long)a.y, a.z); a.x += 1; @@ -367,10 +368,10 @@ int main() { ////////////////////////////////// // Turn off verbose for automated tests, leave on for debugging -void set_verbose(int level) { +DLLEXPORT void set_verbose(int level) { verbose = level; } -void *test_echo_p(void *p) { +DLLEXPORT void *test_echo_p(void *p) { return p; } diff --git a/test/Makefile b/test/Makefile index c7848e0644c45..0f4ac1c905695 100644 --- a/test/Makefile +++ b/test/Makefile @@ -13,13 +13,5 @@ perf: clean: @$(MAKE) -C perf $@ - -rm -f libccalltest.$(SHLIB_EXT) ccalltest -.PHONY: $(TESTS) perf clean libccalltest - -all ccall libccalltest: libccalltest.$(SHLIB_EXT) -libccalltest.$(SHLIB_EXT): ccalltest.c - @$(call PRINT_CC, $(CC) $(CFLAGS) $(DEBUGFLAGS) -O3 $< $(fPIC) -shared -o $@ $(LDFLAGS) -DCC="$(CC)") - -ccalltest: ccalltest.c libccalltest.$(SHLIB_EXT) - @$(call PRINT_CC, $(CC) $(CFLAGS) $(DEBUGFLAGS) -O3 $< -o $@ $(LDFLAGS) -DCC="$(CC)") +.PHONY: $(TESTS) perf clean \ No newline at end of file diff --git a/test/ccall.jl b/test/ccall.jl index 4cdc59b1ad999..63452320c26af 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -2,7 +2,7 @@ import Base.copy, Base.== const verbose = false -const libccalltest = joinpath(dirname(@__FILE__), "libccalltest") +const libccalltest = "libccalltest" ccall((:set_verbose, libccalltest), Void, (Int32,), verbose) # Test for proper argument register truncation diff --git a/test/libdl.jl b/test/libdl.jl index 5a1cdfb761b06..4e8716e300ad0 100644 --- a/test/libdl.jl +++ b/test/libdl.jl @@ -24,10 +24,17 @@ cd(dirname(@__FILE__)) do # @test !isempty(Libdl.find_library(["libccalltest"], [dirname(@__FILE__)])) +# Find the private library directory by finding the path of libjulia (or libjulia-debug, as the case may be) +if ccall(:jl_is_debugbuild, Cint, ()) != 0 + private_libdir = dirname(abspath(Libdl.dlpath("libjulia-debug"))) +else + private_libdir = dirname(abspath(Libdl.dlpath("libjulia"))) +end + # dlopen should be able to handle absolute and relative paths, with and without dlext let dl = C_NULL try - dl = Libdl.dlopen_e(abspath("./libccalltest")) + dl = Libdl.dlopen_e(abspath(joinpath(private_libdir, "libccalltest"))) @test dl != C_NULL finally Libdl.dlclose(dl) @@ -36,7 +43,7 @@ end let dl = C_NULL try - dl = Libdl.dlopen_e(join((abspath("./libccalltest"), string(".", Libdl.dlext)))) + dl = Libdl.dlopen_e(abspath(joinpath(private_libdir, "libccalltest.$(Libdl.dlext)"))) @test dl != C_NULL finally Libdl.dlclose(dl) @@ -45,7 +52,7 @@ end let dl = C_NULL try - dl = Libdl.dlopen_e("./libccalltest") + dl = Libdl.dlopen_e(relpath(joinpath(private_libdir, "libccalltest"))) @test dl != C_NULL finally Libdl.dlclose(dl) @@ -54,7 +61,7 @@ end let dl = C_NULL try - dl = Libdl.dlopen_e(join(("./libccalltest", string(".", Libdl.dlext)))) + dl = Libdl.dlopen_e(relpath(joinpath(private_libdir, "libccalltest.$(Libdl.dlext)"))) @test dl != C_NULL finally Libdl.dlclose(dl) @@ -72,24 +79,20 @@ end # unqualified names present in DL_LOAD_PATH let dl = C_NULL - push!(Libdl.DL_LOAD_PATH, pwd()) try dl = Libdl.dlopen_e("libccalltest") @test dl != C_NULL finally Libdl.dlclose(dl) - pop!(Libdl.DL_LOAD_PATH) end end let dl = C_NULL - push!(Libdl.DL_LOAD_PATH, pwd()) try dl = Libdl.dlopen_e(string("libccalltest",".",Libdl.dlext)) @test dl != C_NULL finally Libdl.dlclose(dl) - pop!(Libdl.DL_LOAD_PATH) end end @@ -134,12 +137,13 @@ end # test dlpath let dl = C_NULL try - dl = Libdl.dlopen("./libccalltest") + path = abspath(joinpath(private_libdir, "libccalltest")) + dl = Libdl.dlopen(path) @test dl != C_NULL @test Base.samefile(abspath(Libdl.dlpath(dl)), - abspath(Libdl.dlpath("./libccalltest"))) + abspath(Libdl.dlpath(path))) @test Base.samefile(abspath(Libdl.dlpath(dl)), - abspath(string("./libccalltest",".", Libdl.dlext))) + string(path,".",Libdl.dlext)) finally Libdl.dlclose(dl) end @@ -151,7 +155,7 @@ end # test dlsym let dl = C_NULL try - dl = Libdl.dlopen("./libccalltest") + dl = Libdl.dlopen(abspath(joinpath(private_libdir, "libccalltest"))) fptr = Libdl.dlsym(dl, :set_verbose) @test fptr != C_NULL @test_throws ErrorException Libdl.dlsym(dl, :foo) From ffe0d53cbbc1c30d151f72e60485e7f4b67b32c8 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sat, 12 Sep 2015 11:07:33 -0700 Subject: [PATCH 0082/1938] On OSX, libraries have .dSYM folders next to them that should be cleaned as well --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 101a664388f98..6b4fc70f6ec76 100644 --- a/src/Makefile +++ b/src/Makefile @@ -165,7 +165,7 @@ $(BUILDDIR)/libjulia.a: julia.expmap $(OBJS) $(BUILDDIR)/flisp/libflisp.a $(BUIL libjulia-release: $(build_shlibdir)/libjulia.$(SHLIB_EXT) clean: - -rm -f $(build_shlibdir)/libjulia* $(build_shlibdir)/libccalltest* + -rm -fr $(build_shlibdir)/libjulia* $(build_shlibdir)/libccalltest* -rm -f $(BUILDDIR)/julia_flisp.boot $(BUILDDIR)/julia_flisp.boot.inc -rm -f $(BUILDDIR)/*.dbg.obj $(BUILDDIR)/*.o *~ $(BUILDDIR)/*.$(SHLIB_EXT) $(BUILDDIR)/*.a *# -rm -f $(BUILDDIR)/julia_version.h From 9cc67dc4b88c13528dd10be41e73ca3d1cfe9c41 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sat, 12 Sep 2015 17:26:19 -0700 Subject: [PATCH 0083/1938] Cull ccalltest executable as it has outlived its usefulness --- src/Makefile | 2 -- src/ccalltest.c | 13 ------------- 2 files changed, 15 deletions(-) diff --git a/src/Makefile b/src/Makefile index 6b4fc70f6ec76..98fa704b7b75e 100644 --- a/src/Makefile +++ b/src/Makefile @@ -85,8 +85,6 @@ $(BUILDDIR)/%.dbg.obj: %.cpp $(HEADERS) $(shell which $(LLVM_CONFIG)) | $(BUILDD libccalltest: $(build_shlibdir)/libccalltest.$(SHLIB_EXT) $(build_shlibdir)/libccalltest.$(SHLIB_EXT): ccalltest.c @$(call PRINT_CC, $(CC) $(CFLAGS) $(CPPFLAGS) $(DEBUGFLAGS) -O3 $< $(fPIC) -shared -o $@ $(LDFLAGS) -DCC="$(CC)") -$(BUILDDIR)/ccalltest: ccalltest.c $(BUILDDIR)/libccalltest.$(SHLIB_EXT) - @$(call PRINT_CC, $(CC) $(CFLAGS) $(DEBUGFLAGS) -O3 $< -o $@ $(LDFLAGS) -DCC="$(CC)") diff --git a/src/ccalltest.c b/src/ccalltest.c index 049eeb105f099..eec5dcd40624e 100644 --- a/src/ccalltest.c +++ b/src/ccalltest.c @@ -353,19 +353,6 @@ DLLEXPORT struct_big test_big(struct_big a) { return a; } -int main() { - fprintf(stderr,"all of the following should be 1 except xs[259] = 0\n"); - a = 3; - b = 259; - fptr = (int (*)(unsigned char x))&testUcharX; - if ((((size_t)fptr)&((size_t)1)) == 1) fptr = NULL; - fprintf(stderr,"compiled with: '%s'\nxs[3] = %d\nxs[259] = %d\ntestUcharX(3) = %d\ntestUcharX(%d) = %d\nfptr(3) = %d\nfptr(259) = %d\n", - xstr(CC), xs[a], xs[b], testUcharX(a), b, testUcharX((unsigned char)b), fptr(a), fptr(b)); - fprintf(stderr,"misc tests:\n"); - struct1 a = {352.39422e23, 19.287577}; - a = test_1(a); -} - ////////////////////////////////// // Turn off verbose for automated tests, leave on for debugging DLLEXPORT void set_verbose(int level) { From 0715058fe2175f36fb86fc68bd6d63f38c1b6f70 Mon Sep 17 00:00:00 2001 From: Jiahao Chen Date: Sun, 13 Sep 2015 23:47:58 -0400 Subject: [PATCH 0084/1938] README: 0.4 is the new 0.3 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 908bddff2b048..00e6aa02b0b71 100644 --- a/README.md +++ b/README.md @@ -68,9 +68,9 @@ First, acquire the source code by cloning the git repository: Be sure to also configure your system to use the appropriate proxy settings, e.g. by setting the `https_proxy` and `http_proxy` variables.) -By default you will be building the latest unstable version of Julia. However, most users should use the most recent stable version of Julia, which is currently the `0.3` series of releases. You can get this version by changing to the Julia directory and running +By default you will be building the latest unstable version of Julia. However, most users should use the most recent stable version of Julia, which is currently the `0.4` series of releases. You can get this version by changing to the Julia directory and running - git checkout release-0.3 + git checkout release-0.4 Now run `make` to build the `julia` executable. To perform a parallel build, use `make -j N` and supply the maximum number of concurrent processes. When compiled the first time, it will automatically download and build its [external dependencies](#Required-Build-Tools-External-Libraries). From 29caf271ebb002504de6cd35d0d7717b5ced367d Mon Sep 17 00:00:00 2001 From: Greg Peairs Date: Sun, 13 Sep 2015 22:15:09 -0700 Subject: [PATCH 0085/1938] Added test for spones --- test/sparsedir/sparse.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 5b00876343974..776b4b04b92a9 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -1134,3 +1134,7 @@ let @test op(A13024, B13024) == op(full(A13024), full(B13024)) end end + +let A = 2. * speye(5,5) + @test full(spones(A)) == eye(full(A)) +end From bdc430b5ba3ad624d52f1046306a90abdd472e49 Mon Sep 17 00:00:00 2001 From: nkottary Date: Mon, 14 Sep 2015 14:57:58 +0530 Subject: [PATCH 0086/1938] Update JL_PRIVATE_LIBS for USE_GPL_LIBS=1 --- Makefile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 026f09e346009..1a84526de3a3e 100644 --- a/Makefile +++ b/Makefile @@ -202,10 +202,15 @@ $(build_bindir)/stringreplace: contrib/stringreplace.c | $(build_bindir) JL_LIBS = julia julia-debug # private libraries, that are installed in $(prefix)/lib/julia -JL_PRIVATE_LIBS = suitesparse_wrapper Rmath ccalltest +JL_PRIVATE_LIBS = ccalltest +ifeq ($(USE_GPL_LIBS), 1) +JL_PRIVATE_LIBS += suitesparse_wrapper Rmath-julia +endif ifeq ($(USE_SYSTEM_FFTW),0) +ifeq ($(USE_GPL_LIBS), 1) JL_PRIVATE_LIBS += fftw3 fftw3f fftw3_threads fftw3f_threads endif +endif ifeq ($(USE_SYSTEM_PCRE),0) JL_PRIVATE_LIBS += pcre endif @@ -238,8 +243,10 @@ ifeq ($(USE_SYSTEM_ARPACK),0) JL_PRIVATE_LIBS += arpack endif ifeq ($(USE_SYSTEM_SUITESPARSE),0) +ifeq ($(USE_GPL_LIBS), 1) JL_PRIVATE_LIBS += amd camd ccolamd cholmod colamd umfpack spqr suitesparseconfig endif +endif ifeq ($(USE_SYSTEM_LLVM),0) ifeq ($(USE_LLVM_SHLIB),1) JL_PRIVATE_LIBS += LLVM From fa7bdb1ce20a58eacf1be682f3e32efb99b071be Mon Sep 17 00:00:00 2001 From: Michael Hatherly Date: Mon, 14 Sep 2015 12:42:43 +0200 Subject: [PATCH 0087/1938] Describe how to document non-toplevel code [ci skip] --- doc/manual/documentation.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/doc/manual/documentation.rst b/doc/manual/documentation.rst index dfedbaa52e002..08d9b4ecea7be 100644 --- a/doc/manual/documentation.rst +++ b/doc/manual/documentation.rst @@ -123,6 +123,27 @@ Or for use with Julia's metaprogramming functionality: @doc "`add(a,b)` adds `a` and `b` together" add @doc "`subtract(a,b)` subtracts `b` from `a`" subtract +Documentation written in non-toplevel blocks, such as ``if``, ``for``, and ``let``, are not +automatically added to the documentation system. ``@doc`` must be used in these cases. For +example: + +.. code-block:: julia + + if VERSION > v"0.4" + "..." + f(x) = x + end + +will not add any documentation to ``f`` even when the condition is ``true`` and must instead +be written as: + +.. code-block:: julia + + if VERSION > v"0.4" + @doc "..." -> + f(x) = x + end + Syntax Guide ------------ From ea5c2b9693397d08a15f19653cb6199e6f54e1eb Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 14 Sep 2015 14:16:04 -0400 Subject: [PATCH 0088/1938] remove changes to PATH variable from WINNT makefile tkelman doesn't like it --- Make.inc | 1 - 1 file changed, 1 deletion(-) diff --git a/Make.inc b/Make.inc index 2621d0d1c6796..8dfd9b762f550 100644 --- a/Make.inc +++ b/Make.inc @@ -246,7 +246,6 @@ override BUILD_OS := WINNT endif ifeq ($(BUILD_OS), WINNT) -PATH := $(PATH):$(build_libdir):$(build_private_libdir):/c/Program Files/7-zip:$(JULIAHOME)/dist-extras BUILD_EXE := .exe else ifneq (,$(findstring CYGWIN,$(BUILD_OS))) BUILD_EXE := .exe From 8f4d5868e17b4ac40e09d58f96ec91d0f8a001df Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Mon, 14 Sep 2015 11:17:20 -0700 Subject: [PATCH 0089/1938] Revert "Always run doc/genstdlib.jl after bootstrap" This reverts commit dbacfa2027f1c82c986d03d30ec55b0b827bf591. This broke on the Windows buildbots due to #11727, ref https://github.com/JuliaLang/julia/pull/13069#issuecomment-140134720 --- Makefile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 373ac70bff81f..026f09e346009 100644 --- a/Makefile +++ b/Makefile @@ -71,10 +71,7 @@ julia-sysimg-release : julia-inference julia-ui-release julia-sysimg-debug : julia-inference julia-ui-debug @$(MAKE) $(QUIET_MAKE) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) JULIA_BUILD_MODE=debug -julia-genstdlib : julia-sysimg-$(JULIA_BUILD_MODE) - @$(JULIA_EXECUTABLE) doc/genstdlib.jl - -julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest julia-genstdlib +julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest debug release : % : julia-% From bd22c1eb7ffca360350fbe7552453abc274cebdc Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 14 Sep 2015 14:18:45 -0400 Subject: [PATCH 0090/1938] makefile output directory PR did not make it into release-0.4 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fd7469d95d446..c9f87103e7928 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ When compiled the first time, it will automatically download and build its [exte This takes a while, but only has to be done once. If the defaults in the build do not work for you, and you need to set specific make parameters, you can save them in `Make.user`. The build will automatically check for the existence of `Make.user` and use it if it exists. Building Julia requires 1.5GiB of disk space and approximately 700MiB of virtual memory. -For release-0.4 and newer builds of Julia, you can create out-of-tree builds of Julia by specifying `make O= configure` on the command line. This will create a directory mirror, with all of the necessary Makefiles to build Julia, in the specified directory. These builds will share the source files in Julia and `deps/srccache`. Each out-of-tree build directory can have its own `Make.user` file to override the global `Make.user` file in the toplevel folder. +For builds of julia starting with 0.5.0-dev, you can create out-of-tree builds of Julia by specifying `make O= configure` on the command line. This will create a directory mirror, with all of the necessary Makefiles to build Julia, in the specified directory. These builds will share the source files in Julia and `deps/srccache`. Each out-of-tree build directory can have its own `Make.user` file to override the global `Make.user` file in the toplevel folder. If you need to build Julia in an environment that does not allow access to the outside world, use `make -C deps getall` to download all the necessary files. Then, copy the `julia` directory over to the target environment and build with `make`. From 3bf729141e88657bbc39ebcd99161337d9e504e5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 14 Sep 2015 14:46:02 -0400 Subject: [PATCH 0091/1938] handle TypeVar showing up in invalid places (as type-asserts) after inference passes --- src/cgutils.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index c959d433e5ea8..a1181ea144ca1 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1162,10 +1162,18 @@ static inline jl_module_t *topmod(jl_codectx_t *ctx) static jl_value_t *expr_type(jl_value_t *e, jl_codectx_t *ctx) { - if (jl_is_expr(e)) - return ((jl_expr_t*)e)->etype; - if (jl_is_symbolnode(e)) - return jl_symbolnode_type(e); + if (jl_is_expr(e)) { + jl_value_t *typ = ((jl_expr_t*)e)->etype; + if (jl_is_typevar(typ)) + typ = ((jl_tvar_t*)typ)->ub; + return typ; + } + if (jl_is_symbolnode(e)) { + jl_value_t *typ = jl_symbolnode_type(e); + if (jl_is_typevar(typ)) + typ = ((jl_tvar_t*)typ)->ub; + return typ; + } if (jl_is_gensym(e)) { int idx = ((jl_gensym_t*)e)->id; jl_value_t *gensym_types = jl_lam_gensyms(ctx->ast); From 615e746967de9a333cb5d43b4dc90abf207c55aa Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 14 Sep 2015 16:13:47 -0400 Subject: [PATCH 0092/1938] improve `require` deprecation ref #13107 --- base/deprecated.jl | 32 +++++++++++++++++++++++++++----- base/loading.jl | 4 ++-- base/require.jl | 2 +- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 156b86e5981a1..d1dc73c590da7 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -661,14 +661,36 @@ end ## require ## +function maybe_require_file(name::AbstractString) + isabspath(name) && return name + isfile(name) && return abspath(name) + if !endswith(name,".jl") + fname = string(name,".jl") + isfile(fname) && return abspath(fname) + end + return name +end + include("require.jl") @noinline function require(f::AbstractString) - if !(endswith(f,".jl") || contains(f,path_separator)) - # for require("Name") this function shouldn't be needed at all - error("use `using` or `import` to load packages") - end depwarn("`require` is deprecated, use `using` or `import` instead", :require) - OldRequire.require(f) + if endswith(f,".jl") || contains(f,path_separator) + # specifying file path + OldRequire.require(f) + else + # require("Foo") --- ambiguous. might be file or package + filename = maybe_require_file(f) + if filename == f + mod = symbol(require_modname(f)) + M = current_module() + if isdefined(M,mod) && isa(eval(M,mod),Module) + return + end + require(mod) + else + OldRequire.require(f) + end + end end @noinline function require(f::AbstractString, fs::AbstractString...) require(f) diff --git a/base/loading.jl b/base/loading.jl index 4071d136f17b7..fd6ab8755f89a 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -162,7 +162,7 @@ function __precompile__(isprecompilable::Bool=true) end end -function require_filename(name::AbstractString) +function require_modname(name::AbstractString) # This function can be deleted when the deprecation for `require` # is deleted. # While we could also strip off the absolute path, the user may be @@ -192,7 +192,7 @@ function reload(name::AbstractString) error("use `include` instead of `reload` to load source files") else # reload("Package") is ok - require(symbol(require_filename(name))) + require(symbol(require_modname(name))) end end diff --git a/base/require.jl b/base/require.jl index a1787ece7996e..50f3510961432 100644 --- a/base/require.jl +++ b/base/require.jl @@ -11,7 +11,7 @@ function find_in_path(name::AbstractString) name = string(base,".jl") isfile(name) && return abspath(name) end - for prefix in [Pkg.dir(), LOAD_PATH] + for prefix in [Pkg.dir(); LOAD_PATH] path = joinpath(prefix, name) isfile(path) && return abspath(path) path = joinpath(prefix, base, "src", name) From b4474f25310e947e8a2323f95e40b03e0e325909 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 13 Sep 2015 22:34:30 -0400 Subject: [PATCH 0093/1938] Add missing multi-argument map signature for numbers --- base/number.jl | 2 +- test/numbers.jl | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/base/number.jl b/base/number.jl index ff78f6913fb60..0e401898115cf 100644 --- a/base/number.jl +++ b/base/number.jl @@ -43,7 +43,7 @@ done(x::Number, state) = state isempty(x::Number) = false in(x::Number, y::Number) = x == y -map(f, x::Number) = f(x) +map(f, x::Number, ys::Number...) = f(x, ys...) zero(x::Number) = oftype(x,0) zero{T<:Number}(::Type{T}) = convert(T,0) diff --git a/test/numbers.jl b/test/numbers.jl index 907157acf52c4..76b10e3f0fe32 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2471,11 +2471,13 @@ end @test in(1+2im, 1+2im) == true #Imag @test in(3, 3.0) == true #mixed -#map(f::Callable, x::Number) = f(x) +#map(f::Callable, x::Number, ys::Number...) = f(x) @test map(sin, 3) == sin(3) @test map(cos, 3) == cos(3) @test map(tan, 3) == tan(3) @test map(log, 3) == log(3) +@test map(copysign, 1.0, -2.0) == -1.0 +@test map(muladd, 2, 3, 4) == 10 @test_throws InexactError convert(UInt8, big(300)) From 594c9836af3f5fe05ee4307ae28011c4151f2fe0 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 14 Sep 2015 17:04:36 -0400 Subject: [PATCH 0094/1938] Enable fast ISel for MCJIT This starts the long march back to acceptable codegen performance. By enabling fast-isel, I see about a 10-20% improvement in overall execution time when running the test suite. Also from what I've been told, generated code performance isn't significantly worse than with the regular regalloc, though this is something we should benchmark at some point. --- src/codegen.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/codegen.cpp b/src/codegen.cpp index 28fad20181102..49656c69bf19b 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5859,6 +5859,9 @@ extern "C" void jl_init_codegen(void) ); delete targetMachine; assert(jl_TargetMachine); +#ifdef USE_MCJIT + jl_TargetMachine->setFastISel(true); +#endif #if defined(LLVM38) engine_module->setDataLayout(jl_TargetMachine->createDataLayout()); #elif defined(LLVM36) && !defined(LLVM37) From f5598a84778ddadeaed272476bd60ea51620da83 Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Mon, 14 Sep 2015 18:35:47 -0400 Subject: [PATCH 0095/1938] Fix #13127, unable to show local functions --- base/show.jl | 2 +- test/show.jl | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/base/show.jl b/base/show.jl index 994bcf94b05ec..4e118d8491214 100644 --- a/base/show.jl +++ b/base/show.jl @@ -69,7 +69,7 @@ end function show(io::IO, f::Function) if isgeneric(f) - if is_exported_from_stdlib(f.env.name, f.env.module) || f.env.module === Main + if !isdefined(f.env, :module) || is_exported_from_stdlib(f.env.name, f.env.module) || f.env.module === Main print(io, f.env.name) else print(io, f.env.module, ".", f.env.name) diff --git a/test/show.jl b/test/show.jl index 18731bd42577f..0fb185acbc365 100644 --- a/test/show.jl +++ b/test/show.jl @@ -293,3 +293,12 @@ let @test sprint(show, B) == "\n\t[1, 1] = #undef\n\t[1, 2] = T12960()\n\t[2, 2] = #undef\n\t[3, 3] = #undef" @test sprint(print, B) == "\n\t[1, 1] = #undef\n\t[1, 2] = T12960()\n\t[2, 2] = #undef\n\t[3, 3] = #undef" end + +# issue #13127 +function f13127() + buf = IOBuffer() + f() = 1 + show(buf, f) + takebuf_string(buf) +end +@test f13127() == "f" From 3cb9b4879d9da4e14caa79fa6a1ed71fa93871e7 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 14 Sep 2015 22:09:09 -0400 Subject: [PATCH 0096/1938] ucontext64_t is defined under sys/ not machine/ on darwin (fix #13125) --- src/signals-apple.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/signals-apple.c b/src/signals-apple.c index ab3709bb86101..5001166311137 100644 --- a/src/signals-apple.c +++ b/src/signals-apple.c @@ -9,7 +9,7 @@ #include #else #define __need_ucontext64_t -#include +#include #endif static mach_port_t segv_port = 0; From 5b40f2bd35d87b267dc1ff1cfc6fd7e07670e89c Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 14 Sep 2015 22:09:45 -0400 Subject: [PATCH 0097/1938] clean target for .examples got lost during makefile_o rebase --- Makefile | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 3ccd56be7f86f..5fe9fa3ed1b1d 100644 --- a/Makefile +++ b/Makefile @@ -415,7 +415,7 @@ endif cp -R $(build_sysconfdir)/julia $(DESTDIR)$(sysconfdir)/ distclean dist-clean: - rm -fr $(BUILDROOT)/julia-*.tar.gz $(BUILDROOT)/julia*.exe $(BUILDROOT)/julia-*.7z $(BUILDROOT)/julia-$(JULIA_COMMIT) + -rm -fr $(BUILDROOT)/julia-*.tar.gz $(BUILDROOT)/julia*.exe $(BUILDROOT)/julia-*.7z $(BUILDROOT)/julia-$(JULIA_COMMIT) dist: @echo \'dist\' target is deprecated: use \'binary-dist\' instead. @@ -513,32 +513,33 @@ full-source-dist: light-source-dist.tmp cd ../ && tar -cz -T $$DIRNAME/full-source-dist.tmp1 --no-recursion -f $$DIRNAME/julia-$(JULIA_VERSION)_$(JULIA_COMMIT)-full.tar.gz clean: | $(CLEAN_TARGETS) - @$(MAKE) -C $(BUILDROOT)/base clean - @$(MAKE) -C $(BUILDROOT)/doc clean - @$(MAKE) -C $(BUILDROOT)/src clean - @$(MAKE) -C $(BUILDROOT)/ui clean - @$(MAKE) -C $(BUILDROOT)/test clean - @rm -f $(BUILDROOT)/julia - @rm -f $(BUILDROOT)/*.tar.gz - @rm -f $(build_bindir)/stringreplace \ - $(BUILDROOT)/light-source-dist.tmp $(BUILDROOT)/light-source-dist.tmp1 \ - $(BUILDROOT)/full-source-dist.tmp $(BUILDROOT)/full-source-dist.tmp1 - @rm -fr $(build_private_libdir) -# Temporarily add this line to the Makefile to remove extras - @rm -fr $(build_datarootdir)/julia/extras + @-$(MAKE) -C $(BUILDROOT)/base clean + @-$(MAKE) -C $(BUILDROOT)/doc clean + @-$(MAKE) -C $(BUILDROOT)/src clean + @-$(MAKE) -C $(BUILDROOT)/ui clean + @-$(MAKE) -C $(BUILDROOT)/test clean + -rm -f $(BUILDROOT)/julia + -rm -f $(BUILDROOT)/*.tar.gz + -rm -f $(build_bindir)/stringreplace \ + $(BUILDROOT)/light-source-dist.tmp $(BUILDROOT)/light-source-dist.tmp1 \ + $(BUILDROOT)/full-source-dist.tmp $(BUILDROOT)/full-source-dist.tmp1 + -rm -fr $(build_private_libdir) +# Teporarily add this line to the Makefile to remove extras + -rm -fr $(build_datarootdir)/julia/extras + -rm -f $(build_prefix)/.examples cleanall: clean - @$(MAKE) -C $(BUILDROOT)/src clean-flisp clean-support - @rm -fr $(build_shlibdir) + @-$(MAKE) -C $(BUILDROOT)/src clean-flisp clean-support + -rm -fr $(build_shlibdir) ifeq ($(OS),WINNT) - @rm -rf $(build_prefix)/lib + -rm -rf $(build_prefix)/lib endif - @$(MAKE) -C $(BUILDROOT)/deps clean-libuv + @-$(MAKE) -C $(BUILDROOT)/deps clean-libuv distcleanall: cleanall - @$(MAKE) -C $(BUILDROOT)/deps distcleanall - @$(MAKE) -C $(BUILDROOT)/doc cleanall - rm -fr $(build_prefix) $(build_staging) + @-$(MAKE) -C $(BUILDROOT)/deps distcleanall + @-$(MAKE) -C $(BUILDROOT)/doc cleanall + -rm -fr $(build_prefix) $(build_staging) .PHONY: default debug release check-whitespace release-candidate \ julia-debug julia-release julia-deps \ From c750eb2ea4deea680da4e8cfaee3e8a7cc9e1764 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Mon, 14 Sep 2015 19:10:34 -0700 Subject: [PATCH 0098/1938] RELBUILDROOT does not need to have cygpath_w called on it fixes a cygwin warning from #12463 due to `cygpath -w ` --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5fe9fa3ed1b1d..d96320c863c89 100644 --- a/Makefile +++ b/Makefile @@ -211,7 +211,7 @@ $(build_private_libdir)/inference.ji: $(build_private_libdir)/inference0.ji $(call spawn,$(JULIA_EXECUTABLE)) -C $(JULIA_CPU_TARGET) --output-ji $(call cygpath_w,$@) -f \ -J $(call cygpath_w,$<) coreimg.jl) -RELBUILDROOT := $(call cygpath_w,$(shell $(JULIAHOME)/contrib/relative_path.sh "$(JULIAHOME)/base" "$(BUILDROOT)/base/")) +RELBUILDROOT := $(shell $(JULIAHOME)/contrib/relative_path.sh "$(JULIAHOME)/base" "$(BUILDROOT)/base/") COMMA:=, define sysimg_builder $$(build_private_libdir)/sys$1.o: $$(build_private_libdir)/inference.ji $$(JULIAHOME)/VERSION $$(BASE_SRCS) From 1d16b49981428450b8ab72ed8affe43ae0161627 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Mon, 14 Sep 2015 23:06:20 -0700 Subject: [PATCH 0099/1938] Misc tests for special functions --- test/math.jl | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/test/math.jl b/test/math.jl index 8fc005a1412d6..d81bee7c8532e 100644 --- a/test/math.jl +++ b/test/math.jl @@ -181,7 +181,6 @@ for T = (Float32,Float64,Rational{Int}) @test cosd(convert(T,-90))::fT === zero(fT) @test cosd(convert(T,-270))::fT === zero(fT) - for x = -3:0.3:3 @test_approx_eq_eps sinpi(convert(T,x))::fT convert(fT,sin(pi*x)) eps(pi*convert(fT,x)) @test_approx_eq_eps cospi(convert(T,x))::fT convert(fT,cos(pi*x)) eps(pi*convert(fT,x)) @@ -193,21 +192,29 @@ for T = (Float32,Float64,Rational{Int}) T != Rational{Int} && @test sinpi(convert(T,-0.0))::fT === -zero(fT) @test sinpi(convert(T,-1.0))::fT === -zero(fT) @test sinpi(convert(T,-2.0))::fT === -zero(fT) + @test_throws DomainError sinpi(convert(T,Inf)) @test cospi(convert(T,0.5))::fT === zero(fT) @test cospi(convert(T,1.5))::fT === zero(fT) @test cospi(convert(T,-0.5))::fT === zero(fT) @test cospi(convert(T,-1.5))::fT === zero(fT) + @test_throws DomainError cospi(convert(T,Inf)) # check exact values @test sind(convert(T,30)) == 0.5 @test cosd(convert(T,60)) == 0.5 @test sind(convert(T,150)) == 0.5 @test sinpi(one(T)/convert(T,6)) == 0.5 + @test_throws DomainError sind(convert(T,Inf)) + @test_throws DomainError cosd(convert(T,Inf)) T != Float32 && @test cospi(one(T)/convert(T,3)) == 0.5 T == Rational{Int} && @test sinpi(5//6) == 0.5 end +@test sinpi(1) == 0 +@test sinpi(-1) == -0 +@test cospi(1) == -1 +@test cospi(2) == 1 # check type stability for T = (Float32,Float64,BigFloat) @@ -221,6 +228,8 @@ end @test_approx_eq erf(1) 0.84270079294971486934 @test_approx_eq erfc(1) 0.15729920705028513066 @test_approx_eq erfcx(1) 0.42758357615580700442 +@test_approx_eq erfcx(Float32(1)) 0.42758357615580700442 +@test_approx_eq erfcx(Complex64(1)) 0.42758357615580700442 @test_approx_eq erfi(1) 1.6504257587975428760 @test_approx_eq erfinv(0.84270079294971486934) 1 @test_approx_eq erfcinv(0.15729920705028513066) 1 @@ -246,7 +255,7 @@ for elty in [Float32,Float64] end @test erfinv(one(elty)) == Inf @test erfinv(-one(elty)) == -Inf - @test_throws DomainError erfinv(2.0*one(elty)) + @test_throws DomainError erfinv(convert(elty,2.0)) @test erfcinv(zero(elty)) == Inf @test_throws DomainError erfcinv(-one(elty)) @@ -265,7 +274,7 @@ end @test_throws Base.Math.AmosException airybi(200) @test_throws ArgumentError airy(5,one(Complex128)) z = 1.8 + 1.0im -for elty in [Complex64,Complex128] +for elty in [Complex64,Complex128, Complex{BigFloat}] @test_approx_eq airy(convert(elty,1.8)) 0.0470362168668458052247 z = convert(elty,z) @test_approx_eq airyx(z) airyx(0,z) @@ -336,6 +345,8 @@ j43 = besselj(4,3.) @test_approx_eq besselj(3.2, 1.3+0.6im) 0.01135309305831220201 + 0.03927719044393515275im @test_approx_eq besselj(1, 3im) 3.953370217402609396im @test_approx_eq besselj(1.0,3im) besselj(1,3im) +@test besselj(big(1.0),3im) ≈ besselj(1,3im) +@test besselj(big(0.1), complex(-0.4)) ≈ 0.820421842809028916 + 0.266571215948350899im @test_throws Base.Math.AmosException besselj(20,1000im) # besselk @@ -365,7 +376,7 @@ y33 = bessely(3,3.) @test_throws DomainError bessely(1,Float32(-1.0)) #besselhx -for elty in [Complex64,Complex128] +for elty in [Complex64,Complex128, Complex{BigFloat}] z = convert(elty, 1.0 + 1.9im) @test_approx_eq besselhx(1.0, 1, z) convert(elty,-0.5949634147786144 - 0.18451272807835967im) end @@ -404,6 +415,7 @@ end @test_approx_eq beta(5,4) beta(4,5) @test_approx_eq beta(-1/2, 3) -16/3 @test_approx_eq lbeta(-1/2, 3) log(16/3) +@test beta(Float32(5),Float32(4)) == beta(Float32(4),Float32(5)) # gamma, lgamma (complex argument) if Base.Math.libm == "libopenlibm" @@ -477,6 +489,9 @@ end @test_approx_eq zeta(2) pi^2/6 @test_approx_eq zeta(4) pi^4/90 @test_approx_eq zeta(one(Float32)) Float32(zeta(one(Float64))) +@test isnan(zeta(NaN)) +@test isnan(zeta(complex(0,Inf))) +@test isnan(zeta(complex(-Inf,0))) # quadgk @test_approx_eq quadgk(cos, 0,0.7,1)[1] sin(1) From 7c117aabb0661d3b467bb1a6febd4f3ac31f72be Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Tue, 15 Sep 2015 10:07:19 -0400 Subject: [PATCH 0100/1938] Always use emit_allocobj to emit heap allocation --- src/intrinsics.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index d3a75ddb391f7..4f82c6437687d 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -654,9 +654,7 @@ static jl_cgval_t emit_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ct } assert(jl_is_datatype(ety)); uint64_t size = jl_datatype_size(ety); - Value *strct = - builder.CreateCall(prepare_call(jlallocobj_func), - ConstantInt::get(T_size, size)); + Value *strct = emit_allocobj(size); builder.CreateStore(literal_pointer_val((jl_value_t*)ety), emit_typeptr_addr(strct)); im1 = builder.CreateMul(im1, ConstantInt::get(T_size, From 02b7ab967fe3952caf610e58ab008147059b463c Mon Sep 17 00:00:00 2001 From: Micky Latowicki Date: Tue, 15 Sep 2015 18:05:27 +0300 Subject: [PATCH 0101/1938] test that max() can take 4 or more array arguments (bug #13144) --- test/operators.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/operators.jl b/test/operators.jl index 88116b88decf9..6106afa1bd330 100644 --- a/test/operators.jl +++ b/test/operators.jl @@ -47,3 +47,10 @@ p = 1=>:foo @test 1 .== 1 @test 1 .< 2 @test 1 .<= 2 + +# issue #13144: max() with 4 or more array arguments +let xs = [[i:i+4;] for i in 1:10] + for n in 2:10 + @test max(xs[1:n]...) == [n:n+4;] + end +end From bcc3736d958324e26465fdd2e025e3808b21c8e6 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 15 Sep 2015 12:30:12 -0400 Subject: [PATCH 0102/1938] Adjust to LLVM 3.8 API changes --- src/cgutils.cpp | 4 ++++ src/codegen.cpp | 21 +++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index a1181ea144ca1..936dc53609f35 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -463,7 +463,11 @@ static void jl_dump_shadow(char *fname, int jit_model, const char *sysimg_data, CodeGenOpt::Aggressive // -O3 )); +#ifdef LLVM38 + legacy::PassManager PM; +#else PassManager PM; +#endif if (!dump_as_bc) { #ifndef LLVM37 PM.add(new TargetLibraryInfo(Triple(TM->getTargetTriple()))); diff --git a/src/codegen.cpp b/src/codegen.cpp index 28fad20181102..92ea56ec97536 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -174,7 +174,11 @@ static Module *jl_Module; #endif static MDBuilder *mbuilder; static std::map argNumberStrings; +#ifdef LLVM38 +static legacy::FunctionPassManager *FPM; +#else static FunctionPassManager *FPM; +#endif #ifdef LLVM37 // No DataLayout pass needed anymore. @@ -1058,7 +1062,7 @@ void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) if (llvmf) { #ifndef LLVM35 new GlobalAlias(llvmf->getType(), GlobalValue::ExternalLinkage, name, llvmf, llvmf->getParent()); - #elif defined(LLVM37) + #elif defined(LLVM37) && !defined(LLVM38) GlobalAlias::create(cast(llvmf->getType()), GlobalValue::ExternalLinkage, name, llvmf, llvmf->getParent()); #else @@ -5601,7 +5605,11 @@ static void init_julia_llvm_env(Module *m) add_named_global(diff_gc_total_bytes_func, (void*)*jl_gc_diff_total_bytes); // set up optimization passes +#ifdef LLVM38 + FPM = new legacy::FunctionPassManager(m); +#else FPM = new FunctionPassManager(m); +#endif #ifdef LLVM37 // No DataLayout pass needed anymore. @@ -5628,9 +5636,18 @@ static void init_julia_llvm_env(Module *m) #ifndef LLVM37 jl_TargetMachine->addAnalysisPasses(*FPM); #endif +#ifdef LLVM38 + FPM->add(createTypeBasedAAWrapperPass()); +#else FPM->add(createTypeBasedAliasAnalysisPass()); - if (jl_options.opt_level>=1) +#endif + if (jl_options.opt_level>=1) { +#ifdef LLVM38 + FPM->add(createBasicAAWrapperPass()); +#else FPM->add(createBasicAliasAnalysisPass()); +#endif + } // list of passes from vmkit FPM->add(createCFGSimplificationPass()); // Clean up disgusting code FPM->add(createPromoteMemoryToRegisterPass());// Kill useless allocas From e11c9142f088d984cabf60aa68c32c2b38adebb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1ll=20Haraldsson?= Date: Tue, 15 Sep 2015 16:54:10 +0000 Subject: [PATCH 0103/1938] Pull request = PR [ci skip] PR may be common Git[Hub] knowledge and then Julia, but not in general programming? Even WIP (and RFC) are explained here, and recently pointed to this document for as not understood.. --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 75c03a5ec63f0..43a4cbfad82c2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,7 +58,7 @@ There are never enough tests. Track [code coverage at Coveralls](https://coveral 4. Run `make test-all` to rebuild Julia and run your new test(s). If you had to fix a bug or add functionality in `base`, this will ensure that your test passes and that you have not introduced extraneous whitespace. -5. Submit the test as a pull request. +5. Submit the test as a pull request (PR). * Code for the buildbot configuration is maintained at: https://github.com/staticfloat/julia-buildbot * You can see the current buildbot setup at: http://buildbot.e.ip.saba.us:8010/builders @@ -98,7 +98,7 @@ The content of the codeblock is spliced directly into the final restructured tex *By contributing code to Julia, you are agreeing to release it under the [MIT License](https://github.com/JuliaLang/julia/tree/master/LICENSE.md).* -The Julia community uses [GitHub issues](https://github.com/JuliaLang/julia/issues) to track and discuss problems, feature requests, and pull requests. You can make pull requests for incomplete features to get code review. The convention is to prefix the pull request title with "WIP:" for Work In Progress, or "RFC:" for Request for Comments when work is completed and ready for merging. This will prevent accidental merging of work that is in progress. +The Julia community uses [GitHub issues](https://github.com/JuliaLang/julia/issues) to track and discuss problems, feature requests, and pull requests (PR). You can make pull requests for incomplete features to get code review. The convention is to prefix the pull request title with "WIP:" for Work In Progress, or "RFC:" for Request for Comments when work is completed and ready for merging. This will prevent accidental merging of work that is in progress. Note: These instructions are for adding to or improving functionality in the base library. Before getting started, it can be helpful to discuss the proposed changes or additions on the mailing list or in a GitHub issue---it's possible your proposed change belongs in a package rather than the core language. Also, keep in mind that changing stuff in the base can potentially break a lot of things. Finally, because of the time required to build Julia, note that it's usually faster to develop your code in stand-alone files, get it working, and then migrate it into the base libraries. From 94487c583c23df65fcf31f482804ee03a0186339 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 15 Sep 2015 13:57:15 -0400 Subject: [PATCH 0104/1938] fix #13144, elementwise min and max for > 3 array arguments --- base/operators.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/base/operators.jl b/base/operators.jl index c2eb8ae900844..ad7e66213b9b3 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -88,8 +88,14 @@ function afoldl(op,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,qs...) y end +immutable ElementwiseMaxFun end +call(::ElementwiseMaxFun, x, y) = max(x,y) + +immutable ElementwiseMinFun end +call(::ElementwiseMinFun, x, y) = min(x, y) + for (op,F) in ((:+,:(AddFun())), (:*,:(MulFun())), (:&,:(AndFun())), (:|,:(OrFun())), - (:$,:$), (:min,:(MinFun())), (:max,:(MaxFun())), (:kron,:kron)) + (:$,:(XorFun())), (:min,:(ElementwiseMinFun())), (:max,:(ElementwiseMaxFun())), (:kron,:kron)) @eval begin # note: these definitions must not cause a dispatch loop when +(a,b) is # not defined, and must only try to call 2-argument definitions, so From 1b6efc7ab297b7b8aea4d7cb0b52fb589392042b Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 15 Sep 2015 15:57:25 -0400 Subject: [PATCH 0105/1938] add the ability to deprecate bindings. fixes #11200 This is a rather minimal implementation that only really works for types and functions, since it relies on printing those objects to show the replacement. Fortunately all existing deprecated bindings are for types. This also avoids tab-completing deprecated names, by excluding them from the result of `names`. The tradeoff is that it's now impossible to enumerate deprecated names by reflection. Hopefully that will be ok. We could add `deprecate(:f)` calls for functions that are entirely deprecated. Then they would be excluded from tab completion as well. --- base/deprecated.jl | 56 +++++++++++++++++++++++---------------------- base/docs/helpdb.jl | 2 +- base/require.jl | 2 +- src/codegen.cpp | 24 +++++++++++++++++-- src/dump.c | 3 ++- src/julia.h | 1 + src/module.c | 39 ++++++++++++++++++++++++++++++- 7 files changed, 94 insertions(+), 33 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 1b0e7faa165bf..e5ed9b2bcc66b 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -101,6 +101,16 @@ function firstcaller(bt::Array{Ptr{Void},1}, funcsym::Symbol) return C_NULL end +deprecate(s::Symbol) = deprecate(current_module(), s) +deprecate(m::Module, s::Symbol) = ccall(:jl_deprecate_binding, Void, (Any, Any), m, s) + +macro deprecate_binding(old, new) + Expr(:toplevel, + Expr(:export, esc(old)), + Expr(:(=), esc(old), esc(new)), + Expr(:call, :deprecate, Expr(:quote, old))) +end + # 0.4 deprecations @deprecate split(x,y,l::Integer,k::Bool) split(x,y;limit=l,keep=k) @@ -111,19 +121,17 @@ end @deprecate rsplit(x,y,l::Integer) rsplit(x,y;limit=l) @deprecate rsplit(x,y,k::Bool) rsplit(x,y;keep=k) -export UdpSocket const TcpSocket = TCPSocket -const UdpSocket = UDPSocket +deprecate(:TcpSocket) const IpAddr = IPAddr +deprecate(:IpAddr) +@deprecate_binding UdpSocket UDPSocket @deprecate isblank(c::Char) c == ' ' || c == '\t' @deprecate isblank(s::AbstractString) all(c -> c == ' ' || c == '\t', s) -export Nothing -const Nothing = Void - -export None -const None = Union{} +@deprecate_binding Nothing Void +@deprecate_binding None Union{} export apply @noinline function apply(f, args...) @@ -150,16 +158,14 @@ end @deprecate inf{T<:AbstractFloat}(::Type{T}) convert(T,Inf) @deprecate nan{T<:AbstractFloat}(::Type{T}) convert(T,NaN) -export String -const String = AbstractString +@deprecate_binding String AbstractString -export Uint, Uint8, Uint16, Uint32, Uint64, Uint128 -const Uint = UInt -const Uint8 = UInt8 -const Uint16 = UInt16 -const Uint32 = UInt32 -const Uint64 = UInt64 -const Uint128 = UInt128 +@deprecate_binding Uint UInt +@deprecate_binding Uint8 UInt8 +@deprecate_binding Uint16 UInt16 +@deprecate_binding Uint32 UInt32 +@deprecate_binding Uint64 UInt64 +@deprecate_binding Uint128 UInt128 @deprecate zero{T}(::Type{Ptr{T}}) Ptr{T}(0) @deprecate zero{T}(x::Ptr{T}) Ptr{T}(0) @@ -178,8 +184,7 @@ const Uint128 = UInt128 @deprecate iround(x) round(Integer,x) @deprecate iround{T}(::Type{T},x) round(T,x) -export Base64Pipe -const Base64Pipe = Base64EncodePipe +@deprecate_binding Base64Pipe Base64EncodePipe @deprecate base64 base64encode @deprecate prevind(a::Any, i::Integer) i-1 @@ -208,8 +213,7 @@ const Base64Pipe = Base64EncodePipe @deprecate error(ex::Exception) throw(ex) @deprecate error{E<:Exception}(::Type{E}) throw(E()) -export MemoryError -const MemoryError = OutOfMemoryError +@deprecate_binding MemoryError OutOfMemoryError @deprecate map!(f::Callable, dest::StridedArray, A::StridedArray, B::Number) broadcast!(f, dest, A, B) @deprecate map!(f::Callable, dest::StridedArray, A::Number, B::StridedArray) broadcast!(f, dest, A, B) @@ -542,17 +546,15 @@ function start_timer(t, d, r) error("start_timer is deprecated. Use Timer(callback, delay, repeat) instead.") end -const UnionType = Union -export UnionType +@deprecate_binding UnionType Union -const MathConst = Irrational +@deprecate_binding MathConst Irrational macro math_const(sym, val, def) depwarn("@math_const is deprecated and renamed to @irrational.", symbol("@math_const")) :(@irrational $(esc(sym)) $(esc(val)) $(esc(def))) end - -export MathConst, @math_const +export @math_const # 11280, mmap @@ -789,8 +791,7 @@ end @deprecate iseltype(x,T) eltype(x) <: T -const FloatingPoint = AbstractFloat -export FloatingPoint +@deprecate_binding FloatingPoint AbstractFloat # 11447 @@ -812,3 +813,4 @@ end # 12839 const AsyncStream = IO +deprecate(:AsyncStream) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 51c4edcbe3d6f..7f0bcabe660bf 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -6144,7 +6144,7 @@ doc""" The distance between `x` and the next larger representable floating-point value of the same `DataType` as `x`. """ -eps(::FloatingPoint) +eps(::AbstractFloat) doc""" rem1(x,m) diff --git a/base/require.jl b/base/require.jl index 50f3510961432..49b4d0d827857 100644 --- a/base/require.jl +++ b/base/require.jl @@ -67,7 +67,7 @@ end # remote/parallel load -function source_path(default::Union(AbstractString,Nothing)="") +function source_path(default::Union(AbstractString,Void)="") t = current_task() while true s = t.storage diff --git a/src/codegen.cpp b/src/codegen.cpp index 28fad20181102..7790fce807564 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1399,6 +1399,21 @@ extern "C" void jl_write_malloc_log(void) // --- constant determination --- +static void show_source_loc(JL_STREAM *out, jl_codectx_t *ctx) +{ + if (ctx == NULL) return; + jl_printf(out, "in %s at %s:%d", ctx->linfo->name->name, ctx->linfo->file->name, ctx->lineno); +} + +extern "C" void jl_binding_deprecation_warning(jl_binding_t *b); + +static void cg_bdw(jl_binding_t *b, jl_codectx_t *ctx) +{ + jl_binding_deprecation_warning(b); + show_source_loc(JL_STDERR, ctx); + jl_printf(JL_STDERR, "\n"); +} + // try to statically evaluate, NULL if not possible extern "C" jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod, @@ -1451,8 +1466,10 @@ jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod, s = (jl_sym_t*)jl_globalref_name(ex); if (s && jl_is_symbol(s)) { jl_binding_t *b = jl_get_binding(jl_globalref_mod(ex), s); - if (b && b->constp) + if (b && b->constp) { + if (b->deprecated) cg_bdw(b, ctx); return b->value; + } } return NULL; } @@ -1467,8 +1484,10 @@ jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod, s = (jl_sym_t*)jl_static_eval(jl_exprarg(e,2),ctx,mod,sp,ast,sparams,allow_alloc); if (m && jl_is_module(m) && s && jl_is_symbol(s)) { jl_binding_t *b = jl_get_binding(m, s); - if (b && b->constp) + if (b && b->constp) { + if (b->deprecated) cg_bdw(b, ctx); return b->value; + } } } else if (fptr == &jl_f_tuple || fptr == &jl_f_instantiate_type) { @@ -2983,6 +3002,7 @@ static Value *global_binding_pointer(jl_module_t *m, jl_sym_t *s, p->addIncoming(bval, not_found); return julia_binding_gv(builder.CreateBitCast(p, jl_ppvalue_llvmt)); } + if (b->deprecated) cg_bdw(b, ctx); } if (pbnd) *pbnd = b; return julia_binding_gv(b); diff --git a/src/dump.c b/src/dump.c index 2929477a555aa..4c17fc693d385 100644 --- a/src/dump.c +++ b/src/dump.c @@ -563,7 +563,7 @@ static void jl_serialize_module(ios_t *s, jl_module_t *m) jl_serialize_value(s, b->value); jl_serialize_value(s, b->globalref); jl_serialize_value(s, b->owner); - write_int8(s, (b->constp<<2) | (b->exportp<<1) | (b->imported)); + write_int8(s, (b->deprecated<<3) | (b->constp<<2) | (b->exportp<<1) | (b->imported)); jl_serialize_gv(s, (jl_value_t*)b); } } @@ -1426,6 +1426,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t b->owner = (jl_module_t*)jl_deserialize_value(s, (jl_value_t**)&b->owner); if (b->owner != NULL) jl_gc_wb(m, b->owner); int8_t flags = read_int8(s); + b->deprecated = (flags>>3) & 1; b->constp = (flags>>2) & 1; b->exportp = (flags>>1) & 1; b->imported = (flags) & 1; diff --git a/src/julia.h b/src/julia.h index 73ffa40fd5b75..bcfb3b1df71bf 100644 --- a/src/julia.h +++ b/src/julia.h @@ -305,6 +305,7 @@ typedef struct { unsigned constp:1; unsigned exportp:1; unsigned imported:1; + unsigned deprecated:1; } jl_binding_t; typedef struct _jl_module_t { diff --git a/src/module.c b/src/module.c index 4162113cbc126..ec21d7f7f7ac5 100644 --- a/src/module.c +++ b/src/module.c @@ -76,6 +76,7 @@ static jl_binding_t *new_binding(jl_sym_t *name) b->constp = 0; b->exportp = 0; b->imported = 0; + b->deprecated = 0; return b; } @@ -232,11 +233,15 @@ DLLEXPORT jl_binding_t *jl_get_binding(jl_module_t *m, jl_sym_t *var) return jl_get_binding_(m, var, NULL); } +void jl_binding_deprecation_warning(jl_binding_t *b); + DLLEXPORT jl_binding_t *jl_get_binding_or_error(jl_module_t *m, jl_sym_t *var) { jl_binding_t *b = jl_get_binding_(m, var, NULL); if (b == NULL) jl_undefined_var_error(var); + if (b->deprecated) + jl_binding_deprecation_warning(b); return b; } @@ -324,6 +329,7 @@ static void module_import_(jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_binding_t *nb = new_binding(s); nb->owner = b->owner; nb->imported = (explici!=0); + nb->deprecated = b->deprecated; *bp = nb; jl_gc_wb_buf(to, nb); } @@ -431,6 +437,7 @@ jl_value_t *jl_get_global(jl_module_t *m, jl_sym_t *var) { jl_binding_t *b = jl_get_binding(m, var); if (b == NULL) return NULL; + if (b->deprecated) jl_binding_deprecation_warning(b); return b->value; } @@ -460,6 +467,35 @@ DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var) return b && b->constp; } +DLLEXPORT void jl_deprecate_binding(jl_module_t *m, jl_sym_t *var) +{ + jl_binding_t *b = jl_get_binding(m, var); + if (b) b->deprecated = 1; +} + +DLLEXPORT int jl_is_binding_deprecated(jl_module_t *m, jl_sym_t *var) +{ + jl_binding_t *b = jl_get_binding(m, var); + return b && b->deprecated; +} + +void jl_binding_deprecation_warning(jl_binding_t *b) +{ + if (b->deprecated) { + if (b->owner) + jl_printf(JL_STDERR, "WARNING: %s.%s is deprecated", b->owner->name->name, b->name->name); + else + jl_printf(JL_STDERR, "WARNING: %s is deprecated", b->name->name); + jl_value_t *v = b->value; + if (v && (jl_is_type(v) || (jl_is_function(v) && jl_is_gf(v)))) { + jl_printf(JL_STDERR, ", use "); + jl_static_show(JL_STDERR, v); + jl_printf(JL_STDERR, " instead"); + } + jl_printf(JL_STDERR, ".\n"); + } +} + DLLEXPORT void jl_checked_assignment(jl_binding_t *b, jl_value_t *rhs) { if (b->constp && b->value != NULL) { @@ -517,7 +553,8 @@ DLLEXPORT jl_value_t *jl_module_names(jl_module_t *m, int all, int imported) for(i=1; i < m->bindings.size; i+=2) { if (table[i] != HT_NOTFOUND) { jl_binding_t *b = (jl_binding_t*)table[i]; - if (b->exportp || ((imported || b->owner == m) && (all || m == jl_main_module))) { + if ((b->exportp || ((imported || b->owner == m) && (all || m == jl_main_module))) && + !b->deprecated) { jl_array_grow_end(a, 1); //XXX: change to jl_arrayset if array storage allocation for Array{Symbols,1} changes: jl_cellset(a, jl_array_dim0(a)-1, (jl_value_t*)b->name); From 391301e185b770839f09132573e425c3b9d277c4 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 15 Sep 2015 17:00:57 -0400 Subject: [PATCH 0106/1938] fix #13075, repair repl ^C mechanism --- base/REPL.jl | 7 ++++--- base/client.jl | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/base/REPL.jl b/base/REPL.jl index 04faf504cf529..1616ca8f85e24 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -57,10 +57,10 @@ function eval_user_input(ast::ANY, backend::REPLBackend) ans = backend.ans # note: value wrapped in a non-syntax value to avoid evaluating # possibly-invalid syntax (issue #6763). - backend.in_eval = true eval(Main, :(ans = $(Any[ans])[1])) - backend.in_eval = false + backend.in_eval = true value = eval(Main, ast) + backend.in_eval = false backend.ans = value put!(backend.response_channel, (value, nothing)) end @@ -159,10 +159,11 @@ immutable REPLBackendRef response_channel::Channel end -function run_repl(repl::AbstractREPL) +function run_repl(repl::AbstractREPL, consumer = x->nothing) repl_channel = Channel(1) response_channel = Channel(1) backend = start_repl_backend(repl_channel, response_channel) + consumer(backend) run_frontend(repl, REPLBackendRef(repl_channel,response_channel)) backend end diff --git a/base/client.jl b/base/client.jl index edc452845069c..1530242c9787d 100644 --- a/base/client.jl +++ b/base/client.jl @@ -452,7 +452,7 @@ function _start() end else _atreplinit(active_repl) - active_repl_backend = REPL.run_repl(active_repl) + REPL.run_repl(active_repl, backend->(global active_repl_backend = backend)) end end catch err From b8d71b7bfa2875f83a0d963f1b15e3667ab540c8 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 15 Sep 2015 17:08:21 -0400 Subject: [PATCH 0107/1938] print Task backtrace in writemime, not show. also use showerror. --- base/replutil.jl | 8 ++++++++ base/task.jl | 4 ---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/base/replutil.jl b/base/replutil.jl index fa9eacda8d615..07879de96b506 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -38,6 +38,14 @@ writemime(io::IO, ::MIME"text/plain", t::Associative) = writemime(io::IO, ::MIME"text/plain", t::Union{KeyIterator, ValueIterator}) = showkv(io, t, limit=true) +function writemime(io::IO, ::MIME"text/plain", t::Task) + show(io, t) + if t.state == :failed + println(io) + showerror(io, CapturedException(t.result, t.backtrace)) + end +end + # showing exception objects as descriptive error messages diff --git a/base/task.jl b/base/task.jl index ab436a2f1c518..0128eaf06394e 100644 --- a/base/task.jl +++ b/base/task.jl @@ -7,10 +7,6 @@ Task(f) = Task(()->f()) function show(io::IO, t::Task) print(io, "Task ($(t.state)) @0x$(hex(convert(UInt, pointer_from_objref(t)), WORD_SIZE>>2))") - if t.state == :failed - println(io) - show(io, CapturedException(t.result, t.backtrace)) - end end # Container for a captured exception and its backtrace. Can be serialized. From e3bfd6b68b06af4f5525139c6932411c2db9fbb0 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 15 Sep 2015 17:30:29 -0400 Subject: [PATCH 0108/1938] fix #13048, neater signatures for `in` of Associative --- base/dict.jl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/base/dict.jl b/base/dict.jl index 22fee19569fb6..ce53f6838e82d 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -8,16 +8,17 @@ const secret_table_token = :__c782dbf1cf4d6a2e5e3865d7e95634f2e09b5902__ haskey(d::Associative, k) = in(k,keys(d)) -function in(p, a::Associative) - if !isa(p,Pair) - error("""Associative collections only contain Pairs; - Either look for e.g. A=>B instead, or use the `keys` or `values` - function if you are looking for a key or value respectively.""") - end +function in(p::Pair, a::Associative) v = get(a,p[1],secret_table_token) !is(v, secret_table_token) && (v == p[2]) end +function in(p, a::Associative) + error("""Associative collections only contain Pairs; + Either look for e.g. A=>B instead, or use the `keys` or `values` + function if you are looking for a key or value respectively.""") +end + function summary(t::Associative) n = length(t) string(typeof(t), " with ", n, (n==1 ? " entry" : " entries")) From 7a00cff23ca9febc1a9acb7f1e425275a130a82d Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 15 Sep 2015 17:44:26 -0400 Subject: [PATCH 0109/1938] better definitions of startswith and endswith, fixes #12917 --- base/strings/util.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/strings/util.jl b/base/strings/util.jl index 4e6901ef8d65f..aefc53f4b2bad 100644 --- a/base/strings/util.jl +++ b/base/strings/util.jl @@ -12,7 +12,7 @@ function startswith(a::AbstractString, b::AbstractString) end done(b,i) end -startswith(str::AbstractString, chars::Chars) = !isempty(str) && str[start(str)] in chars +startswith(str::AbstractString, chars::Chars) = !isempty(str) && first(str) in chars function endswith(a::AbstractString, b::AbstractString) i = endof(a) @@ -28,7 +28,7 @@ function endswith(a::AbstractString, b::AbstractString) end j < b1 end -endswith(str::AbstractString, chars::Chars) = !isempty(str) && str[end] in chars +endswith(str::AbstractString, chars::Chars) = !isempty(str) && last(str) in chars startswith(a::ByteString, b::ByteString) = startswith(a.data, b.data) startswith(a::Vector{UInt8}, b::Vector{UInt8}) = From 0b9ef7d2221f0a2940769c54fa9dbc4a2306f7ef Mon Sep 17 00:00:00 2001 From: Michael Hatherly Date: Tue, 8 Sep 2015 01:08:20 +0200 Subject: [PATCH 0110/1938] Documenting macro-generated code This adds a `@__doc__` macro, not exported currently, for use by macro authors that want their macros to hook into the docsystem properly. Usage: macro example(f) quote $(f)() = 0 @__doc__ $(f)(x) = 1 $(f)(x, y) = 2 end |> esc end "Docs for `g(x)` only." @example g will attach the docstring to the 1-arg method only. Also improve invalid doc expression error message by displaying the unexpanded form of the expression instead of its expansion. Enable docstrings for `at-enum` using the newly added macro. Fixes #12705. --- base/Enums.jl | 2 +- base/docs/Docs.jl | 43 ++++++++++++++++++++++++++++++++++++- base/docs/bootstrap.jl | 2 ++ test/docs.jl | 48 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 2 deletions(-) diff --git a/base/Enums.jl b/base/Enums.jl index af241db044192..07ea8a5b610fe 100644 --- a/base/Enums.jl +++ b/base/Enums.jl @@ -74,7 +74,7 @@ macro enum(T,syms...) end blk = quote # enum definition - bitstype 32 $(esc(T)) <: Enum + Base.@__doc__(bitstype 32 $(esc(T)) <: Enum) function Base.convert(::Type{$(esc(typename))}, x::Integer) $(membershiptest(:x, values)) || enum_argument_error($(Expr(:quote, typename)), x) Intrinsics.box($(esc(typename)), convert(Int32, x)) diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index ce82dc069bc92..b488244c0d852 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -496,9 +496,49 @@ end multidoc(meta, objs) = quote $([:(@doc $(esc(meta)) $(esc(obj))) for obj in objs]...) end +doc""" + @__doc__(ex) + +Low-level macro used to mark expressions returned by a macro that should be documented. If +more than one expression is marked then the same docstring is applied to each expression. + + macro example(f) + quote + $(f)() = 0 + @__doc__ $(f)(x) = 1 + $(f)(x, y) = 2 + end |> esc + end + +`@__doc__` has no effect when a macro that uses it is not documented. +""" +:(Base.@__doc__) + +function __doc__!(meta, def::Expr) + if isexpr(def, :block) && length(def.args) == 2 && def.args[1] == symbol("#doc#") + # Convert `Expr(:block, :#doc#, ...)` created by `@__doc__` to an `@doc`. + def.head = :macrocall + def.args = [symbol("@doc"), meta, def.args[end]] + true + else + found = false + for each in def.args + found |= __doc__!(meta, each) + end + found + end +end +__doc__!(meta, def) = false + fexpr(ex) = isexpr(ex, :function, :stagedfunction, :(=)) && isexpr(ex.args[1], :call) function docm(meta, def, define = true) + + err = ( + "invalid doc expression:", def, isexpr(def, :macrocall) ? + "'$(def.args[1])' is not documentable. See 'help?> Base.@__doc__' for details." : "" + ) + def′ = unblock(def) isexpr(def′, :quote) && isexpr(def′.args[1], :macrocall) && @@ -522,7 +562,8 @@ function docm(meta, def, define = true) :global) ? vardoc(meta, def, namify(def′)) : isvar(def′) ? objdoc(meta, def′) : isexpr(def′, :tuple) ? multidoc(meta, def′.args) : - isa(def′, Expr) ? error("invalid doc expression $def′") : + __doc__!(meta, def′) ? esc(def′) : + isa(def′, Expr) ? error(strip(join(err, "\n\n"))) : objdoc(meta, def′) end diff --git a/base/docs/bootstrap.jl b/base/docs/bootstrap.jl index 7084219bc9596..a55d33028d340 100644 --- a/base/docs/bootstrap.jl +++ b/base/docs/bootstrap.jl @@ -4,6 +4,8 @@ macro doc(args...) DocBootstrap._expand_(args...) end +macro __doc__(ex) esc(Expr(:block, symbol("#doc#"), ex)) end + module DocBootstrap type List diff --git a/test/docs.jl b/test/docs.jl index 3566edcb0ae9a..52b0b84ced72a 100644 --- a/test/docs.jl +++ b/test/docs.jl @@ -270,6 +270,54 @@ let d1 = @doc(DocsTest.val) @test d1 !== nothing end +# Document specific expressions generated by macro calls. +module MacroGenerated + +import Base.@__doc__ + +macro example_1(f) + quote + $(f)() = 0 + @__doc__ $(f)(x) = x + $(f)(x, y) = x + y + end |> esc +end + +"f" +@example_1 f + +@example_1 _f + +macro example_2(f) + quote + $(f)() = 0 + @__doc__ $(f)(x) = x + @__doc__ $(f)(x, y) = x + y + end |> esc +end + +"g" +@example_2 g + +@example_2 _g + +end + +let funcdoc = meta(MacroGenerated)[MacroGenerated.f] + @test funcdoc.order == [Tuple{Any}] + @test funcdoc.meta[Tuple{Any}] == doc"f" +end + +@test isdefined(MacroGenerated, :_f) + +let funcdoc = meta(MacroGenerated)[MacroGenerated.g] + @test funcdoc.order == [Tuple{Any}, Tuple{Any, Any}] + @test funcdoc.meta[Tuple{Any}] == doc"g" + @test funcdoc.meta[Tuple{Any, Any}] == doc"g" +end + +@test isdefined(MacroGenerated, :_g) + # Issue #12700. @test @doc(DocsTest.@m) == doc"Inner.@m" From ab1d09ba72601587f740dad858fb11b55d1b16c8 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 15 Sep 2015 18:15:02 -0400 Subject: [PATCH 0111/1938] better deprecation for `int` etc. fixes #12847 --- base/deprecated.jl | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index e5ed9b2bcc66b..64f1d793767c7 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -254,16 +254,38 @@ end @deprecate float64(s::AbstractString) parse(Float64,s) @deprecate float32(s::AbstractString) parse(Float32,s) -for (f,t) in ((:integer, Integer), (:signed, Signed), - (:unsigned, Unsigned), (:int, Int), (:int8, Int8), (:int16, Int16), - (:int32, Int32), (:int64, Int64), (:int128, Int128), (:uint, UInt), - (:uint8, UInt8), (:uint16, UInt16), (:uint32, UInt32), (:uint64, UInt64), - (:uint128, UInt128)) +for (f,t) in ((:integer, Integer), (:signed, Signed), (:unsigned, Unsigned)) @eval begin @deprecate $f(x::AbstractArray) round($t, x) end end +for (f,t) in ((:int, Int), (:int8, Int8), (:int16, Int16), (:int32, Int32), + (:int64, Int64), (:int128, Int128), (:uint, UInt), (:uint8, UInt8), + (:uint16, UInt16), (:uint32, UInt32), (:uint64, UInt64), (:uint128,UInt128)) + ex1 = sprint(io->show_unquoted(io,:([parse($t,s) for s in a]))) + ex2 = sprint(io->show_unquoted(io,:(round($t, a)))) + name = Expr(:quote,f) + @eval begin + function ($f)(x::AbstractArray) + if all(y->isa(y,AbstractString), x) + depwarn(string($name,"(a::AbstractArray) is deprecated, use ", $ex1, " instead."), $name) + return [parse($t,s) for s in x] + elseif all(y->isa(y,Number), x) + depwarn(string($name,"(a::AbstractArray) is deprecated, use ", $ex2, " instead."), $name) + return round($t, x) + end + y = similar(x,$t) + i = 1 + for e in x + y[i] = ($f)(e) + i += 1 + end + y + end + end +end + for (f,t) in ((:char, Char), (:bool, Bool), (:float16, Float16), (:float32, Float32), (:float64, Float64), (:complex64, Complex64), (:complex128, Complex128)) @eval begin @@ -346,13 +368,6 @@ end @deprecate integer(x::Ptr) convert(UInt, x) @deprecate unsigned(x::Ptr) convert(UInt, x) -for (f,t) in ((:int, Int), (:int8, Int8), (:int16, Int16), (:int32, Int32), - (:int64, Int64), (:int128, Int128), (:uint, UInt), (:uint8, UInt8), - (:uint16, UInt16), (:uint32, UInt32), (:uint64, UInt64), (:uint128,UInt128)) - @eval begin - @deprecate ($f){S<:AbstractString}(a::AbstractArray{S}) [parse($t,s) for s in a] - end -end for (f,t) in ((:float32, Float32), (:float64, Float64)) @eval begin @deprecate ($f){S<:AbstractString}(a::AbstractArray{S}) [parse($t,s) for s in a] From eaf75cbd37cd3cda50b58f8b4f5b951c55cce1d5 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 15 Sep 2015 21:54:19 -0400 Subject: [PATCH 0112/1938] Revert "TypeError is needed by type-intersection during --compile=all" This reverts commit 11009e152383de6f5ce665d537200b2a43f36a07. --- base/base.jl | 7 +++++++ base/boot.jl | 35 +++++++++++++++-------------------- base/essentials.jl | 1 - src/init.c | 2 +- src/toplevel.c | 1 + 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/base/base.jl b/base/base.jl index 909bb062b6a13..18cdf4e4d8f39 100644 --- a/base/base.jl +++ b/base/base.jl @@ -11,6 +11,13 @@ type SystemError <: Exception SystemError(p::AbstractString) = new(p, Libc.errno()) end +type TypeError <: Exception + func::Symbol + context::AbstractString + expected::Type + got +end + type ParseError <: Exception msg::AbstractString end diff --git a/base/boot.jl b/base/boot.jl index e79885196708a..2a687893acebb 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -142,7 +142,7 @@ export # errors BoundsError, DivideError, DomainError, Exception, InexactError, InterruptException, OutOfMemoryError, ReadOnlyMemoryError, OverflowError, - StackOverflowError, SegmentationFault, UndefRefError, UndefVarError, TypeError, + StackOverflowError, SegmentationFault, UndefRefError, UndefVarError, # AST representation Expr, GotoNode, LabelNode, LineNumberNode, QuoteNode, SymbolNode, TopNode, GlobalRef, NewvarNode, GenSym, @@ -210,20 +210,8 @@ else typealias UInt UInt32 end -abstract AbstractString -abstract DirectIndexString <: AbstractString - -immutable ASCIIString <: DirectIndexString - data::Array{UInt8,1} -end - -immutable UTF8String <: AbstractString - data::Array{UInt8,1} -end - -typealias ByteString Union{ASCIIString,UTF8String} - abstract Exception + immutable BoundsError <: Exception a::Any i::Any @@ -244,12 +232,9 @@ immutable UndefVarError <: Exception var::Symbol end immutable InterruptException <: Exception end -type TypeError <: Exception - func::Symbol - context::AbstractString - expected::Type - got -end + +abstract AbstractString +abstract DirectIndexString <: AbstractString type SymbolNode name::Symbol @@ -257,6 +242,16 @@ type SymbolNode SymbolNode(name::Symbol, t::ANY) = new(name, t) end +immutable ASCIIString <: DirectIndexString + data::Array{UInt8,1} +end + +immutable UTF8String <: AbstractString + data::Array{UInt8,1} +end + +typealias ByteString Union{ASCIIString,UTF8String} + include(fname::ByteString) = ccall(:jl_load_, Any, (Any,), fname) # constructors for built-in types diff --git a/base/essentials.jl b/base/essentials.jl index 582f66bb2584a..774fe2c789487 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -32,7 +32,6 @@ call(T::Type{SegmentationFault}) = Core.call(T) call(T::Type{UndefRefError}) = Core.call(T) call(T::Type{UndefVarError}, var::Symbol) = Core.call(T, var) call(T::Type{InterruptException}) = Core.call(T) -call(T::Type{TypeError}, func::Symbol, context::AbstractString, expected::Type, got) = Core.call(T, func, context, expected, got) call(T::Type{SymbolNode}, name::Symbol, t::ANY) = Core.call(T, name, t) call(T::Type{GlobalRef}, modu, name::Symbol) = Core.call(T, modu, name) call(T::Type{ASCIIString}, d::Array{UInt8,1}) = Core.call(T, d) diff --git a/src/init.c b/src/init.c index 5debfbbe0176b..6ba9ae916f90a 100644 --- a/src/init.c +++ b/src/init.c @@ -712,7 +712,6 @@ void jl_get_builtin_hooks(void) jl_boundserror_type = (jl_datatype_t*)core("BoundsError"); jl_memory_exception = jl_new_struct_uninit((jl_datatype_t*)core("OutOfMemoryError")); jl_readonlymemory_exception = jl_new_struct_uninit((jl_datatype_t*)core("ReadOnlyMemoryError")); - jl_typeerror_type = (jl_datatype_t*)core("TypeError"); #ifdef SEGV_EXCEPTION jl_segv_exception = jl_new_struct_uninit((jl_datatype_t*)core("SegmentationFault")); @@ -733,6 +732,7 @@ DLLEXPORT void jl_get_system_hooks(void) jl_errorexception_type = (jl_datatype_t*)basemod("ErrorException"); jl_argumenterror_type = (jl_datatype_t*)basemod("ArgumentError"); + jl_typeerror_type = (jl_datatype_t*)basemod("TypeError"); jl_methoderror_type = (jl_datatype_t*)basemod("MethodError"); jl_loaderror_type = (jl_datatype_t*)basemod("LoadError"); jl_initerror_type = (jl_datatype_t*)basemod("InitError"); diff --git a/src/toplevel.c b/src/toplevel.c index 1e5f34aef1214..6fb35f6aaa1a1 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -138,6 +138,7 @@ jl_value_t *jl_eval_module_expr(jl_expr_t *ex) // to pick up new types from Base jl_errorexception_type = NULL; jl_argumenterror_type = NULL; + jl_typeerror_type = NULL; jl_methoderror_type = NULL; jl_loaderror_type = NULL; jl_initerror_type = NULL; From adbbd2f14b99dbc54f12af799d6cc0e2df9ac311 Mon Sep 17 00:00:00 2001 From: Glen Hertz Date: Tue, 15 Sep 2015 23:47:09 -0400 Subject: [PATCH 0113/1938] Minor doc fix. --- doc/manual/interfaces.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/interfaces.rst b/doc/manual/interfaces.rst index 94b4dbd14702f..51982619d24ce 100644 --- a/doc/manual/interfaces.rst +++ b/doc/manual/interfaces.rst @@ -151,7 +151,7 @@ Abstract Arrays Methods to implement Brief description ========================================================== ============================================ ======================================================================================= :func:`size(A) ` Returns a tuple containing the dimensions of A -:func:`Base.linearindexing(Type) ` Returns either ``Base.LinearFast()`` or ``Base.LinearSlow``. See the description below. +:func:`Base.linearindexing(Type) ` Returns either ``Base.LinearFast()`` or ``Base.LinearSlow()``. See the description below. :func:`getindex(A, i::Int) ` (if ``LinearFast``) Linear scalar indexing :func:`getindex(A, i1::Int, ..., iN::Int) ` (if ``LinearSlow``, where ``N = ndims(A)``) N-dimensional scalar indexing :func:`setindex!(A, v, i::Int) ` (if ``LinearFast``) Scalar indexed assignment From f3de524a7db7e9076f95a7f323af07efce46fffb Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Tue, 15 Sep 2015 22:33:41 -0700 Subject: [PATCH 0114/1938] Add cygpath_w call to fix windows installer build in binary-dist was broken by out-of-tree builds --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d96320c863c89..db6e3a25de68f 100644 --- a/Makefile +++ b/Makefile @@ -465,7 +465,7 @@ ifeq ($(OS), WINNT) cd $(prefix) && find * | sed -e 's/\//\\/g' -e 's/$$/\r/g' > etc/uninstall.log # build nsis package - $(call spawn,$(JULIAHOME)/dist-extras/nsis/makensis.exe) -NOCD -DVersion=$(JULIA_VERSION) -DArch=$(ARCH) -DCommit=$(JULIA_COMMIT) $(JULIAHOME)/contrib/windows/build-installer.nsi + $(call spawn,$(JULIAHOME)/dist-extras/nsis/makensis.exe) -NOCD -DVersion=$(JULIA_VERSION) -DArch=$(ARCH) -DCommit=$(JULIA_COMMIT) $(call cygpath_w,$(JULIAHOME)/contrib/windows/build-installer.nsi) # compress nsis installer and combine with 7zip self-extracting header cd $(BUILDROOT) && $(JULIAHOME)/dist-extras/7z a -mx9 "julia-install-$(JULIA_COMMIT)-$(ARCH).7z" julia-installer.exe From fee7aff43935a27a042f1d2241c92240223a2315 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Tue, 15 Sep 2015 22:51:38 -0700 Subject: [PATCH 0115/1938] Clearly mark where 0.5 deprecations start in base/deprecated.jl --- base/deprecated.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base/deprecated.jl b/base/deprecated.jl index 64f1d793767c7..354907e875ac0 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -826,6 +826,8 @@ end @deprecate cartesianmap(f, dims) for idx in CartesianRange(dims); f(idx.I...); end +# 0.5 deprecations + # 12839 const AsyncStream = IO deprecate(:AsyncStream) From 00c7325491855bf1a8c573cace8696cdeaa36f49 Mon Sep 17 00:00:00 2001 From: tan Date: Wed, 16 Sep 2015 11:29:01 +0530 Subject: [PATCH 0116/1938] fix #13028 --- base/datafmt.jl | 2 +- test/readdlm.jl | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/base/datafmt.jl b/base/datafmt.jl index 6a0a12bdf6141..a5a9a3581236c 100644 --- a/base/datafmt.jl +++ b/base/datafmt.jl @@ -208,7 +208,7 @@ function result{T}(dlmstore::DLMStore{T}) cells = dlmstore.data sbuff = dlmstore.sbuff - if (lastcol < ncols) || (lastrow < nrows) + if (nrows > 0) && ((lastcol < ncols) || (lastrow < nrows)) while lastrow <= nrows (lastcol == ncols) && (lastcol = 0; lastrow += 1) for cidx in (lastcol+1):ncols diff --git a/test/readdlm.jl b/test/readdlm.jl index c0032acc27049..d584f985abd8b 100644 --- a/test/readdlm.jl +++ b/test/readdlm.jl @@ -218,3 +218,10 @@ end reshape(Any[1,2000.1,Float64(22222222222222222222222222222222222222),true,0x3,false,10e6,-10.34], 2, 4), Any) @test isequaldlm(readcsv(IOBuffer("-9223355253176920979,9223355253176920979"), Int64), Int64[-9223355253176920979 9223355253176920979], Int64) + +# fix #13028 +for data in ["A B C", "A B C\n"] + data,hdr = readdlm(IOBuffer(data), header=true) + @test hdr == AbstractString["A" "B" "C"] + @test data == Array(Float64, 0, 3) +end From 5726979d29e1fa635e0d429d13d1fa7b2bd124d0 Mon Sep 17 00:00:00 2001 From: Graham Inggs Date: Wed, 16 Sep 2015 08:18:09 +0200 Subject: [PATCH 0117/1938] Build in the multiarch directories as well This makes $(private_libdir_rel) = $(build_private_libdir_rel), so that patchelf is no longer required (on Debian, at least) --- Make.inc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Make.inc b/Make.inc index 8dfd9b762f550..02f0370b63488 100644 --- a/Make.inc +++ b/Make.inc @@ -174,8 +174,10 @@ INSTALL_M := $(JULIAHOME)/contrib/install.sh 755 # This used for debian packaging, to conform to library layout guidelines ifeq ($(MULTIARCH_INSTALL), 1) MULTIARCH := $(shell gcc -print-multiarch) -private_libdir := $(prefix)/lib/$(MULTIARCH)/julia -libdir := $(prefix)/lib/$(MULTIARCH)/ +libdir = $(prefix)/lib/$(MULTIARCH) +private_libdir = $(prefix)/lib/$(MULTIARCH)/julia +build_libdir = $(build_prefix)/lib/$(MULTIARCH) +build_private_libdir = $(build_prefix)/lib/$(MULTIARCH)/julia endif # LLVM Options From f4ffde64fe950d5ec102817e2fa84219f872c349 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Fri, 26 Jun 2015 22:04:46 -0400 Subject: [PATCH 0118/1938] Make size and offset in jl_fielddesc_t 32bit length --- src/alloc.c | 5 +++-- src/julia.h | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 522bbba61d5d0..2ff8912660391 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -577,11 +577,12 @@ void jl_compute_field_offsets(jl_datatype_t *st) if (al > alignm) alignm = al; } - if (__unlikely(sz > JL_FIELD_MAX_OFFSET)) - jl_throw(jl_overflow_exception); st->fields[i].offset = sz; st->fields[i].size = fsz; sz += fsz; + if (__unlikely(sz >= JL_FIELD_MAX_SIZE)) { + jl_throw(jl_overflow_exception); + } } st->alignment = alignm; st->size = LLT_ALIGN(sz, alignm); diff --git a/src/julia.h b/src/julia.h index bcfb3b1df71bf..b2d65cc301881 100644 --- a/src/julia.h +++ b/src/julia.h @@ -253,13 +253,13 @@ typedef struct { } jl_uniontype_t; typedef struct { - uint16_t offset; // offset relative to data start, excluding type tag - uint16_t size:15; - uint16_t isptr:1; + uint32_t offset; // offset relative to data start, excluding type tag + uint32_t size:31; + uint32_t isptr:1; } jl_fielddesc_t; -#define JL_FIELD_MAX_OFFSET ((1ul << 16) - 1ul) -#define JL_FIELD_MAX_SIZE ((1ul << 15) - 1ul) +// For both field size and total size +#define JL_FIELD_MAX_SIZE ((((uint32_t)1) << 31) - 1) typedef struct _jl_datatype_t { JL_DATA_TYPE From 1d963c55c8e08e92d568efe85de0dd9ae9082405 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sun, 28 Jun 2015 18:28:02 -0400 Subject: [PATCH 0119/1938] Use wrapper functions for field info access --- src/alloc.c | 11 ++++++----- src/builtins.c | 14 +++++++------- src/cgutils.cpp | 13 +++++++------ src/codegen.cpp | 4 ++-- src/dump.c | 6 +++--- src/gc.c | 6 +++--- src/julia.h | 13 ++++++++++--- src/sys.c | 2 +- 8 files changed, 39 insertions(+), 30 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 2ff8912660391..66e9b707c90b1 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -298,8 +298,9 @@ DLLEXPORT jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args, uin jl_set_nth_field(jv, i, args[i]); } for(size_t i=na; i < nf; i++) { - if (type->fields[i].isptr) + if (jl_field_isptr(type, i)) { *(jl_value_t**)((char*)jl_data_ptr(jv)+jl_field_offset(type,i)) = NULL; + } } return jv; } @@ -557,7 +558,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) if (__unlikely(fsz > JL_FIELD_MAX_SIZE)) jl_throw(jl_overflow_exception); al = ((jl_datatype_t*)ty)->alignment; - st->fields[i].isptr = 0; + jl_field_setisptr(st, i, 0); if (((jl_datatype_t*)ty)->haspadding) st->haspadding = 1; } @@ -566,7 +567,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) if (fsz > MAX_ALIGN) fsz = MAX_ALIGN; al = fsz; - st->fields[i].isptr = 1; + jl_field_setisptr(st, i, 1); ptrfree = 0; } if (al != 0) { @@ -577,8 +578,8 @@ void jl_compute_field_offsets(jl_datatype_t *st) if (al > alignm) alignm = al; } - st->fields[i].offset = sz; - st->fields[i].size = fsz; + jl_field_setoffset(st, i, sz); + jl_field_setsize(st, i, fsz); sz += fsz; if (__unlikely(sz >= JL_FIELD_MAX_SIZE)) { jl_throw(jl_overflow_exception); diff --git a/src/builtins.c b/src/builtins.c index 56b0cef763a49..7dc6319a7aa1d 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -254,11 +254,11 @@ static int NOINLINE compare_fields(jl_value_t *a, jl_value_t *b, jl_datatype_t * { size_t nf = jl_datatype_nfields(dt); for (size_t f=0; f < nf; f++) { - size_t offs = dt->fields[f].offset; + size_t offs = jl_field_offset(dt, f); char *ao = (char*)jl_data_ptr(a) + offs; char *bo = (char*)jl_data_ptr(b) + offs; int eq; - if (dt->fields[f].isptr) { + if (jl_field_isptr(dt, f)) { jl_value_t *af = *(jl_value_t**)ao; jl_value_t *bf = *(jl_value_t**)bo; if (af == bf) eq = 1; @@ -268,7 +268,7 @@ static int NOINLINE compare_fields(jl_value_t *a, jl_value_t *b, jl_datatype_t * else { jl_datatype_t *ft = (jl_datatype_t*)jl_field_type(dt, f); if (!ft->haspadding) { - eq = bits_equal(ao, bo, dt->fields[f].size); + eq = bits_equal(ao, bo, jl_field_size(dt, f)); } else { assert(jl_datatype_nfields(ft) > 0); @@ -1178,20 +1178,20 @@ static uptrint_t jl_object_id_(jl_value_t *tv, jl_value_t *v) return bits_hash(jl_data_ptr(v), sz) ^ h; } for (size_t f=0; f < nf; f++) { - size_t offs = dt->fields[f].offset; + size_t offs = jl_field_offset(dt, f); char *vo = (char*)jl_data_ptr(v) + offs; uptrint_t u; - if (dt->fields[f].isptr) { + if (jl_field_isptr(dt, f)) { jl_value_t *f = *(jl_value_t**)vo; u = f==NULL ? 0 : jl_object_id(f); } else { - jl_datatype_t *fieldtype = (jl_datatype_t*)jl_svecref(dt->types, f); + jl_datatype_t *fieldtype = (jl_datatype_t*)jl_field_type(dt, f); assert(jl_is_datatype(fieldtype) && !fieldtype->abstract && !fieldtype->mutabl); if (fieldtype->haspadding) u = jl_object_id_((jl_value_t*)fieldtype, (jl_value_t*)vo); else - u = bits_hash(vo, dt->fields[f].size); + u = bits_hash(vo, jl_field_size(dt, f)); } h = bitmix(h, u); } diff --git a/src/cgutils.cpp b/src/cgutils.cpp index a1181ea144ca1..df41e151d7692 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -727,7 +727,7 @@ static Type *julia_struct_to_llvm(jl_value_t *jt) for(i = 0; i < ntypes; i++) { jl_value_t *ty = jl_svecref(jst->types, i); Type *lty; - if (jst->fields[i].isptr) + if (jl_field_isptr(jst, i)) lty = jl_pvalue_llvmt; else lty = ty==(jl_value_t*)jl_bool_type ? T_int8 : julia_type_to_llvm(ty); @@ -764,8 +764,9 @@ static bool is_datatype_all_pointers(jl_datatype_t *dt) { size_t i, l = jl_datatype_nfields(dt); for(i=0; i < l; i++) { - if (!dt->fields[i].isptr) + if (!jl_field_isptr(dt, i)) { return false; + } } return true; } @@ -1848,7 +1849,7 @@ static void emit_setfield(jl_datatype_t *sty, const jl_cgval_t &strct, size_t id builder.CreateGEP(builder.CreateBitCast(strct.V, T_pint8), ConstantInt::get(T_size, jl_field_offset(sty,idx0))); jl_value_t *jfty = jl_svecref(sty->types, idx0); - if (sty->fields[idx0].isptr) { + if (jl_field_isptr(sty, idx0)) { Value *r = boxed(rhs, ctx); builder.CreateStore(r, builder.CreateBitCast(addr, jl_ppvalue_llvmt)); @@ -1912,7 +1913,7 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg } } size_t j = 0; - if (nf > 0 && sty->fields[0].isptr && nargs>1) { + if (nf > 0 && jl_field_isptr(sty, 0) && nargs>1) { // emit first field before allocating struct to save // a couple store instructions. avoids initializing // the first field to NULL, and sometimes the GC root @@ -1940,7 +1941,7 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg make_gcroot(strct, ctx); } for(size_t i=j; i < nf; i++) { - if (sty->fields[i].isptr) { + if (jl_field_isptr(sty, i)) { builder.CreateStore( V_null, builder.CreatePointerCast( @@ -1952,7 +1953,7 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg bool need_wb = false; for(size_t i=j+1; i < nargs; i++) { jl_cgval_t rhs = emit_expr(args[i],ctx); - if (sty->fields[i-1].isptr && !rhs.isboxed) { + if (jl_field_isptr(sty, i - 1) && !rhs.isboxed) { if (!needroots) { // if this struct element needs boxing and we haven't rooted // the struct, root it now. diff --git a/src/codegen.cpp b/src/codegen.cpp index 7790fce807564..41236b3b9cae5 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1756,7 +1756,7 @@ static bool is_getfield_nonallocating(jl_datatype_t *ty, jl_value_t *fld) else if (jl_is_quotenode(fld) && jl_is_long(jl_fieldref(fld,0))) idx = jl_unbox_long(jl_fieldref(fld,0))-1; for(size_t i=0; i < jl_svec_len(ty->types); i++) { - if (!(ty->fields[i].isptr || (idx >= 0 && (size_t)idx != i))) + if (!(jl_field_isptr(ty,i) || (idx >= 0 && (size_t)idx != i))) return false; } return true; @@ -2570,7 +2570,7 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, if (jl_is_leaf_type((jl_value_t*)sty) && jl_subtype(rhst, ft, 0)) { // TODO: attempt better codegen for approximate types jl_cgval_t strct = emit_expr(args[1], ctx); // emit lhs - if (sty->fields[idx].isptr) // emit rhs + if (jl_field_isptr(sty, idx)) // emit rhs *ret = emit_expr(args[3], ctx); else *ret = emit_unboxed(args[3], ctx); diff --git a/src/dump.c b/src/dump.c index 4c17fc693d385..be7f96c4cc443 100644 --- a/src/dump.c +++ b/src/dump.c @@ -896,7 +896,7 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) } else { for(size_t i=0; i < nf; i++) { - if (t->fields[i].size > 0) { + if (jl_field_size(t, i) > 0) { jl_serialize_value(s, jl_get_nth_field(v, i)); } } @@ -1511,8 +1511,8 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t else { char *data = (char*)jl_data_ptr(v); for(i=0; i < nf; i++) { - if (dt->fields[i].size > 0) { - if (dt->fields[i].isptr) { + if (jl_field_size(dt,i) > 0) { + if (jl_field_isptr(dt,i)) { jl_value_t **fld = (jl_value_t**)(data+jl_field_offset(dt, i)); *fld = jl_deserialize_value(s, fld); } diff --git a/src/gc.c b/src/gc.c index 30cc8da014119..9fed119746a9c 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1744,11 +1744,11 @@ static int push_root(jl_value_t *v, int d, int bits) // int fdsz = sizeof(void*)*nf; // void** children = alloca(fdsz); // int ci = 0; - jl_fielddesc_t* fields = dt->fields; for(int i=0; i < nf; i++) { - if (fields[i].isptr) { + if (jl_field_isptr(dt, i)) { nptr++; - jl_value_t **slot = (jl_value_t**)((char*)v + fields[i].offset); + jl_value_t **slot = (jl_value_t**)((char*)v + + jl_field_offset(dt, i)); jl_value_t *fld = *slot; if (fld) { verify_parent2("object", v, slot, "field(%d)", i); diff --git a/src/julia.h b/src/julia.h index b2d65cc301881..e0d386ca9761d 100644 --- a/src/julia.h +++ b/src/julia.h @@ -695,14 +695,21 @@ STATIC_INLINE jl_value_t *jl_cellset(void *a, size_t i, void *x) #define jl_data_ptr(v) (((jl_value_t*)v)->fieldptr) // struct type info -#define jl_field_offset(st,i) (((jl_datatype_t*)st)->fields[i].offset) -#define jl_field_size(st,i) (((jl_datatype_t*)st)->fields[i].size) -#define jl_field_isptr(st,i) (((jl_datatype_t*)st)->fields[i].isptr) #define jl_field_name(st,i) (jl_sym_t*)jl_svecref(((jl_datatype_t*)st)->name->names, (i)) #define jl_field_type(st,i) jl_svecref(((jl_datatype_t*)st)->types, (i)) #define jl_datatype_size(t) (((jl_datatype_t*)t)->size) #define jl_datatype_nfields(t) (((jl_datatype_t*)(t))->nfields) +#define jl_field_offset(st,i) (((jl_datatype_t*)st)->fields[i].offset) +#define jl_field_size(st,i) (((jl_datatype_t*)st)->fields[i].size) +#define jl_field_isptr(st,i) (((jl_datatype_t*)st)->fields[i].isptr) +#define jl_field_setoffset(st, i, v) \ + (((jl_datatype_t*)st)->fields[i].offset = v) +#define jl_field_setsize(st, i, v) \ + (((jl_datatype_t*)st)->fields[i].size = v) +#define jl_field_setisptr(st, i, v) \ + (((jl_datatype_t*)st)->fields[i].isptr = v) + // basic predicates ----------------------------------------------------------- #define jl_is_nothing(v) (((jl_value_t*)(v)) == ((jl_value_t*)jl_nothing)) #define jl_is_tuple(v) (((jl_datatype_t*)jl_typeof(v))->name == jl_tuple_typename) diff --git a/src/sys.c b/src/sys.c index 1953f06e810b3..4d88301b8aed3 100644 --- a/src/sys.c +++ b/src/sys.c @@ -614,7 +614,7 @@ DLLEXPORT size_t jl_get_field_offset(jl_datatype_t *ty, int field) { if (field > jl_datatype_nfields(ty)) jl_error("This type does not have that many fields"); - return ty->fields[field].offset; + return jl_field_offset(ty, field); } DLLEXPORT size_t jl_get_alignment(jl_datatype_t *ty) From 12fa893179e140a6d50f6bcc3f8d78d843993985 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sun, 28 Jun 2015 22:54:42 -0400 Subject: [PATCH 0120/1938] Support variable length field descriptors --- src/abi_x86_64.cpp | 3 +- src/alloc.c | 14 ++++++-- src/dump.c | 10 ++++-- src/gc.c | 10 ++++-- src/jltypes.c | 10 +++--- src/julia.h | 80 ++++++++++++++++++++++++++++++++++++++-------- 6 files changed, 99 insertions(+), 28 deletions(-) diff --git a/src/abi_x86_64.cpp b/src/abi_x86_64.cpp index 08aac96aa52cc..845401183885d 100644 --- a/src/abi_x86_64.cpp +++ b/src/abi_x86_64.cpp @@ -133,7 +133,8 @@ void classifyType(Classification& accum, jl_value_t* ty, uint64_t offset) { else if (jl_datatype_size(ty) <= 16) { size_t i; for (i = 0; i < jl_datatype_nfields(ty); ++i) { - classifyType(accum, jl_field_type(ty,i), offset + jl_field_offset(ty,i)); + classifyType(accum, jl_field_type((jl_datatype_t*)ty,i), + offset + jl_field_offset((jl_datatype_t*)ty,i)); } } else { diff --git a/src/alloc.c b/src/alloc.c index 66e9b707c90b1..076763a152f2c 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -536,17 +536,25 @@ jl_datatype_t *jl_new_abstracttype(jl_value_t *name, jl_datatype_t *super, return dt; } -jl_datatype_t *jl_new_uninitialized_datatype(size_t nfields) +jl_datatype_t *jl_new_uninitialized_datatype(size_t nfields, + int8_t fielddesc_type) { + // fielddesc_type is specified manually for builtin types + // and is (will be) calculated automatically for user defined types. + uint32_t fielddesc_size = jl_fielddesc_size(fielddesc_type); jl_datatype_t *t = (jl_datatype_t*) newobj((jl_value_t*)jl_datatype_type, - NWORDS(sizeof(jl_datatype_t) + nfields*sizeof(jl_fielddesc_t))); + NWORDS(sizeof(jl_datatype_t) + nfields * fielddesc_size)); + // fielddesc_type should only be assigned here. It can cause data + // corruption otherwise. + t->fielddesc_type = fielddesc_type; t->nfields = nfields; return t; } void jl_compute_field_offsets(jl_datatype_t *st) { + // TODO overflow check size_t sz = 0, alignm = 1; int ptrfree = 1; @@ -614,7 +622,7 @@ jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_datatype_t *super, t = jl_bool_type; } if (t == NULL) - t = jl_new_uninitialized_datatype(jl_svec_len(fnames)); + t = jl_new_uninitialized_datatype(jl_svec_len(fnames), 2); // TODO else tn = t->name; // init before possibly calling jl_new_typename diff --git a/src/dump.c b/src/dump.c index be7f96c4cc443..ac223e305681d 100644 --- a/src/dump.c +++ b/src/dump.c @@ -514,6 +514,7 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) write_int32(s, dt->size); int has_instance = !!(dt->instance != NULL); write_uint8(s, dt->abstract | (dt->mutabl<<1) | (dt->pointerfree<<2) | (has_instance<<3)); + write_int8(s, dt->fielddesc_type); if (!dt->abstract) { write_uint16(s, dt->ninitialized); if (mode != MODE_MODULE && mode != MODE_MODULE_POSTWORK) { @@ -525,7 +526,8 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) if (nf > 0) { write_int32(s, dt->alignment); write_int8(s, dt->haspadding); - ios_write(s, (char*)&dt->fields[0], nf*sizeof(jl_fielddesc_t)); + size_t fieldsize = jl_fielddesc_size(dt->fielddesc_type); + ios_write(s, (char*)&dt->fields32[0], nf * fieldsize); jl_serialize_value(s, dt->types); } @@ -1082,6 +1084,7 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) uint16_t nf = read_uint16(s); size_t size = read_int32(s); uint8_t flags = read_uint8(s); + uint8_t fielddesc_type = read_int8(s); jl_datatype_t *dt; if (tag == 2) dt = jl_int32_type; @@ -1090,7 +1093,7 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) else if (tag == 4) dt = jl_int64_type; else - dt = jl_new_uninitialized_datatype(nf); + dt = jl_new_uninitialized_datatype(nf, fielddesc_type); assert(tree_literal_values==NULL && mode != MODE_AST); backref_list.items[pos] = dt; dt->size = size; @@ -1125,7 +1128,8 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) if (nf > 0) { dt->alignment = read_int32(s); dt->haspadding = read_int8(s); - ios_read(s, (char*)&dt->fields[0], nf*sizeof(jl_fielddesc_t)); + size_t fieldsize = jl_fielddesc_size(fielddesc_type); + ios_read(s, (char*)&dt->fields32[0], nf * fieldsize); dt->types = (jl_svec_t*)jl_deserialize_value(s, (jl_value_t**)&dt->types); jl_gc_wb(dt, dt->types); } diff --git a/src/gc.c b/src/gc.c index 9fed119746a9c..d05dd4dc11d50 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1734,10 +1734,14 @@ static int push_root(jl_value_t *v, int d, int bits) else if (gc_typeof(vt) == (jl_value_t*)jl_datatype_type) { jl_datatype_t *dt = (jl_datatype_t*)vt; size_t dtsz; - if (dt == jl_datatype_type) - dtsz = NWORDS(sizeof(jl_datatype_t) + jl_datatype_nfields(v)*sizeof(jl_fielddesc_t))*sizeof(void*); - else + if (dt == jl_datatype_type) { + size_t fieldsize = + jl_fielddesc_size(((jl_datatype_t*)v)->fielddesc_type); + dtsz = NWORDS(sizeof(jl_datatype_t) + + jl_datatype_nfields(v) * fieldsize) * sizeof(void*); + } else { dtsz = jl_datatype_size(dt); + } MARK(v, bits = gc_setmark(v, dtsz, GC_MARKED_NOESC)); int nf = (int)jl_datatype_nfields(dt); // TODO check if there is a perf improvement for objects with a lot of fields diff --git a/src/jltypes.c b/src/jltypes.c index da8cbbfd14f7e..7e62e8d383bdb 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2059,7 +2059,7 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i } // create and initialize new type - ndt = jl_new_uninitialized_datatype(istuple ? ntp : dt->nfields); + ndt = jl_new_uninitialized_datatype(istuple ? ntp : dt->nfields, 2); // TODO // associate these parameters with the new type on // the stack, in case one of its field types references it. top.tt = (jl_datatype_t*)ndt; @@ -3148,12 +3148,12 @@ extern void jl_init_int32_int64_cache(void); void jl_init_types(void) { // create base objects - jl_datatype_type = jl_new_uninitialized_datatype(10); + jl_datatype_type = jl_new_uninitialized_datatype(10, 1); jl_set_typeof(jl_datatype_type, jl_datatype_type); - jl_typename_type = jl_new_uninitialized_datatype(7); - jl_sym_type = jl_new_uninitialized_datatype(0); + jl_typename_type = jl_new_uninitialized_datatype(7, 1); + jl_sym_type = jl_new_uninitialized_datatype(0, 1); jl_symbol_type = jl_sym_type; - jl_simplevector_type = jl_new_uninitialized_datatype(1); + jl_simplevector_type = jl_new_uninitialized_datatype(1, 1); jl_emptysvec = (jl_svec_t*)newobj((jl_value_t*)jl_simplevector_type, 1); jl_svec_set_len_unsafe(jl_emptysvec, 0); diff --git a/src/julia.h b/src/julia.h index e0d386ca9761d..2cda92abeac0b 100644 --- a/src/julia.h +++ b/src/julia.h @@ -252,11 +252,23 @@ typedef struct { jl_svec_t *types; } jl_uniontype_t; +typedef struct { + uint8_t offset; // offset relative to data start, excluding type tag + uint8_t size:7; + uint8_t isptr:1; +} jl_fielddesc8_t; + +typedef struct { + uint16_t offset; // offset relative to data start, excluding type tag + uint16_t size:15; + uint16_t isptr:1; +} jl_fielddesc16_t; + typedef struct { uint32_t offset; // offset relative to data start, excluding type tag uint32_t size:31; uint32_t isptr:1; -} jl_fielddesc_t; +} jl_fielddesc32_t; // For both field size and total size #define JL_FIELD_MAX_SIZE ((((uint32_t)1) << 31) - 1) @@ -275,12 +287,17 @@ typedef struct _jl_datatype_t { int32_t ninitialized; // hidden fields: uint32_t nfields; - uint32_t alignment : 31; // strictest alignment over all fields + uint32_t alignment : 29; // strictest alignment over all fields uint32_t haspadding : 1; // has internal undefined bytes + uint32_t fielddesc_type : 2; // 0 -> 8, 1 -> 16, 2 -> 32 uint32_t uid; void *struct_decl; //llvm::Value* void *ditype; // llvm::MDNode* to be used as llvm::DIType(ditype) - jl_fielddesc_t fields[]; + union { + jl_fielddesc8_t fields8[0]; + jl_fielddesc16_t fields16[0]; + jl_fielddesc32_t fields32[0]; + }; } jl_datatype_t; typedef struct { @@ -700,15 +717,51 @@ STATIC_INLINE jl_value_t *jl_cellset(void *a, size_t i, void *x) #define jl_datatype_size(t) (((jl_datatype_t*)t)->size) #define jl_datatype_nfields(t) (((jl_datatype_t*)(t))->nfields) -#define jl_field_offset(st,i) (((jl_datatype_t*)st)->fields[i].offset) -#define jl_field_size(st,i) (((jl_datatype_t*)st)->fields[i].size) -#define jl_field_isptr(st,i) (((jl_datatype_t*)st)->fields[i].isptr) -#define jl_field_setoffset(st, i, v) \ - (((jl_datatype_t*)st)->fields[i].offset = v) -#define jl_field_setsize(st, i, v) \ - (((jl_datatype_t*)st)->fields[i].size = v) -#define jl_field_setisptr(st, i, v) \ - (((jl_datatype_t*)st)->fields[i].isptr = v) +#define DEFINE_FIELD_ACCESSORS(f) \ + static inline uint32_t jl_field_##f(jl_datatype_t *st, int i) \ + { \ + if (st->fielddesc_type == 0) { \ + return st->fields8[i].f; \ + } \ + else if (st->fielddesc_type == 1) { \ + return st->fields16[i].f; \ + } \ + else { \ + return st->fields32[i].f; \ + } \ + } \ + static inline void jl_field_set##f(jl_datatype_t *st, int i, \ + uint32_t val) \ + { \ + if (st->fielddesc_type == 0) { \ + st->fields8[i].f = val; \ + } \ + else if (st->fielddesc_type == 1) { \ + st->fields16[i].f = val; \ + } \ + else { \ + st->fields32[i].f = val; \ + } \ + } + +DEFINE_FIELD_ACCESSORS(offset) +DEFINE_FIELD_ACCESSORS(size) +DEFINE_FIELD_ACCESSORS(isptr) + +static inline uint32_t jl_fielddesc_size(int8_t fielddesc_type) +{ + if (fielddesc_type == 0) { + return sizeof(jl_fielddesc8_t); + } + else if (fielddesc_type == 1) { + return sizeof(jl_fielddesc16_t); + } + else { + return sizeof(jl_fielddesc32_t); + } +} + +#undef DEFINE_FIELD_ACCESSORS // basic predicates ----------------------------------------------------------- #define jl_is_nothing(v) (((jl_value_t*)(v)) == ((jl_value_t*)jl_nothing)) @@ -894,7 +947,8 @@ jl_value_t *jl_apply_type_(jl_value_t *tc, jl_value_t **params, size_t n); jl_value_t *jl_instantiate_type_with(jl_value_t *t, jl_value_t **env, size_t n); jl_datatype_t *jl_new_abstracttype(jl_value_t *name, jl_datatype_t *super, jl_svec_t *parameters); -DLLEXPORT jl_datatype_t *jl_new_uninitialized_datatype(size_t nfields); +DLLEXPORT jl_datatype_t *jl_new_uninitialized_datatype(size_t nfields, + int8_t fielddesc_type); DLLEXPORT jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_datatype_t *super, jl_svec_t *parameters, jl_svec_t *fnames, jl_svec_t *ftypes, From 2b00b5ef7cf362627873fec80afa504a69c33791 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Tue, 1 Sep 2015 19:07:19 -0400 Subject: [PATCH 0121/1938] Fix overflow check in type construction --- src/alloc.c | 15 ++++++++++----- src/julia.h | 3 --- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 076763a152f2c..c0a5c22c8fdb8 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -554,16 +554,22 @@ jl_datatype_t *jl_new_uninitialized_datatype(size_t nfields, void jl_compute_field_offsets(jl_datatype_t *st) { - // TODO overflow check size_t sz = 0, alignm = 1; int ptrfree = 1; + assert(0 <= st->fielddesc_type && st->fielddesc_type <= 2); + + uint64_t max_offset = (((uint64_t)1) << + (1 << (3 + st->fielddesc_type))) - 1; + uint64_t max_size = max_offset >> 1; + for(size_t i=0; i < jl_datatype_nfields(st); i++) { jl_value_t *ty = jl_field_type(st, i); size_t fsz, al; if (jl_isbits(ty) && jl_is_leaf_type(ty)) { fsz = jl_datatype_size(ty); - if (__unlikely(fsz > JL_FIELD_MAX_SIZE)) + // Should never happen + if (__unlikely(fsz > max_size)) jl_throw(jl_overflow_exception); al = ((jl_datatype_t*)ty)->alignment; jl_field_setisptr(st, i, 0); @@ -588,10 +594,9 @@ void jl_compute_field_offsets(jl_datatype_t *st) } jl_field_setoffset(st, i, sz); jl_field_setsize(st, i, fsz); - sz += fsz; - if (__unlikely(sz >= JL_FIELD_MAX_SIZE)) { + if (__unlikely(max_offset - sz < fsz)) jl_throw(jl_overflow_exception); - } + sz += fsz; } st->alignment = alignm; st->size = LLT_ALIGN(sz, alignm); diff --git a/src/julia.h b/src/julia.h index 2cda92abeac0b..9f1988ffcf867 100644 --- a/src/julia.h +++ b/src/julia.h @@ -270,9 +270,6 @@ typedef struct { uint32_t isptr:1; } jl_fielddesc32_t; -// For both field size and total size -#define JL_FIELD_MAX_SIZE ((((uint32_t)1) << 31) - 1) - typedef struct _jl_datatype_t { JL_DATA_TYPE jl_typename_t *name; From a0a1946fa9c7b26737dd5e7f0a83808e015518b6 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Tue, 1 Sep 2015 19:30:50 -0400 Subject: [PATCH 0122/1938] Add test for constructing huge types. --- test/core.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/core.jl b/test/core.jl index 363789ef9f0b3..d3b44c6bd7dee 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3357,3 +3357,12 @@ for j = 1:1 continue end end + +# PR 11888 +immutable A11888{T} + a::NTuple{16,T} +end + +typealias B11888{T} A11888{A11888{A11888{T}}} + +@test sizeof(B11888{B11888{Int64}}) == (1 << 24) * 8 From 23122445acebcf3f7609c25be12eca8bc3cecc45 Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Wed, 16 Sep 2015 14:18:31 +0200 Subject: [PATCH 0123/1938] Add det(Symmetric/Hermitian) based on bkfact. Fixes 13088. --- base/linalg/bunchkaufman.jl | 25 +++++++++++++++++++++++++ base/linalg/symmetric.jl | 10 +++++++++- test/linalg/symmetric.jl | 10 ++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/base/linalg/bunchkaufman.jl b/base/linalg/bunchkaufman.jl index 86468ff961b60..f2a069c861870 100644 --- a/base/linalg/bunchkaufman.jl +++ b/base/linalg/bunchkaufman.jl @@ -50,6 +50,31 @@ function A_ldiv_B!{T<:BlasComplex}(B::BunchKaufman{T}, R::StridedVecOrMat{T}) (issym(B) ? LAPACK.sytrs! : LAPACK.hetrs!)(B.uplo, B.LD, B.ipiv, R) end +function det(F::BunchKaufman) + M = F.LD + p = F.ipiv + n = size(F.LD, 1) + + i = 1 + d = one(eltype(F)) + while i <= n + if p[i] > 0 + # 1x1 pivot case + d *= M[i,i] + i += 1 + else + # 2x2 pivot case. Make sure not to square before the subtraction by scaling with the off-diagonal element. This is safe because the off diagonal is always large for 2x2 pivots. + if F.uplo == 'U' + d *= M[i, i + 1]*(M[i,i]/M[i, i + 1]*M[i + 1, i + 1] - (issym(F) ? M[i, i + 1] : conj(M[i, i + 1]))) + else + d *= M[i + 1,i]*(M[i, i]/M[i + 1, i]*M[i + 1, i + 1] - (issym(F) ? M[i + 1, i] : conj(M[i + 1, i]))) + end + i += 2 + end + end + return d +end + ## reconstruct the original matrix ## TODO: understand the procedure described at ## http://www.nag.com/numeric/FL/nagdoc_fl22/pdf/F07/f07mdf.pdf diff --git a/base/linalg/symmetric.jl b/base/linalg/symmetric.jl index bbd5bf3b19b1e..ad7d51da9476e 100644 --- a/base/linalg/symmetric.jl +++ b/base/linalg/symmetric.jl @@ -117,8 +117,16 @@ A_mul_B!{T<:BlasComplex,S<:StridedMatrix}(C::StridedMatrix{T}, A::StridedMatrix{ *(A::HermOrSym, B::HermOrSym) = full(A)*full(B) *(A::StridedMatrix, B::HermOrSym) = A*full(B) -factorize(A::HermOrSym) = bkfact(A.data, symbol(A.uplo), issym(A)) +bkfact(A::HermOrSym) = bkfact(A.data, symbol(A.uplo), issym(A)) +factorize(A::HermOrSym) = bkfact(A) + +# Is just RealHermSymComplexHerm, but type alias seems to be broken +det{T<:Real,S}(A::Union{Hermitian{T,S}, Symmetric{T,S}, Hermitian{Complex{T},S}}) = real(det(bkfact(A))) +det{T<:Real}(A::Symmetric{T}) = det(bkfact(A)) +det(A::Symmetric) = det(bkfact(A)) + \{T,S<:StridedMatrix}(A::HermOrSym{T,S}, B::StridedVecOrMat) = \(bkfact(A.data, symbol(A.uplo), issym(A)), B) + inv{T<:BlasFloat,S<:StridedMatrix}(A::Hermitian{T,S}) = Hermitian{T,S}(inv(bkfact(A.data, symbol(A.uplo))), A.uplo) inv{T<:BlasFloat,S<:StridedMatrix}(A::Symmetric{T,S}) = Symmetric{T,S}(inv(bkfact(A.data, symbol(A.uplo), true)), A.uplo) diff --git a/test/linalg/symmetric.jl b/test/linalg/symmetric.jl index 3d820b44fa12c..19b71f6ebad9e 100644 --- a/test/linalg/symmetric.jl +++ b/test/linalg/symmetric.jl @@ -101,6 +101,16 @@ let n=10 # cond @test cond(Hermitian(asym)) ≈ cond(asym) + # det + @test det(asym) ≈ det(Hermitian(asym, :U)) + @test det(asym) ≈ det(Hermitian(asym, :L)) + if eltya <: Real + @test det(asym) ≈ det(Symmetric(asym, :U)) + @test det(asym) ≈ det(Symmetric(asym, :L)) + end + @test det(a + a.') ≈ det(Symmetric(a + a.', :U)) + @test det(a + a.') ≈ det(Symmetric(a + a.', :L)) + # rank let A = a[:,1:5]*a[:,1:5]' @test rank(A) == rank(Hermitian(A)) From a3c58ff40931870aea91ded998e11c29d822aa42 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Wed, 16 Sep 2015 09:47:43 -0500 Subject: [PATCH 0124/1938] Optionally log compilation --- src/codegen.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/codegen.cpp b/src/codegen.cpp index 7790fce807564..82365c46cc321 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -753,6 +753,15 @@ static void maybe_alloc_arrayvar(jl_sym_t *s, jl_codectx_t *ctx) } } +// Snooping on which functions are being compiled, and how long it takes +JL_STREAM *dump_compiles_stream = NULL; +uint64_t last_time = 0; +extern "C" DLLEXPORT +void jl_dump_compiles(void *s) +{ + dump_compiles_stream = (JL_STREAM*)s; +} + // --- entry point --- //static int n_emit=0; static Function *emit_function(jl_lambda_info_t *lam); @@ -764,6 +773,8 @@ static Function *to_function(jl_lambda_info_t *li) BasicBlock *old = nested_compile ? builder.GetInsertBlock() : NULL; DebugLoc olddl = builder.getCurrentDebugLocation(); bool last_n_c = nested_compile; + if (!nested_compile && dump_compiles_stream != NULL) + last_time = jl_hrtime(); nested_compile = true; jl_gc_inhibit_finalizers(nested_compile); Function *f = NULL; @@ -817,6 +828,13 @@ static Function *to_function(jl_lambda_info_t *li) nested_compile = last_n_c; jl_gc_inhibit_finalizers(nested_compile); JL_SIGATOMIC_END(); + if (dump_compiles_stream != NULL) { + uint64_t this_time = jl_hrtime(); + jl_printf(dump_compiles_stream, "%lu\t\"", this_time-last_time); + jl_static_show(dump_compiles_stream, (jl_value_t*)li); + jl_printf(dump_compiles_stream, "\"\n"); + last_time = this_time; + } return f; } From 55e840ba6951d95ebd751cf1d7d65169a83622ba Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 16 Sep 2015 12:51:58 -0400 Subject: [PATCH 0125/1938] fix #13168, segfault when scheduling an already-finished task --- base/task.jl | 6 +++++- src/task.c | 4 +++- test/parallel.jl | 10 ++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/base/task.jl b/base/task.jl index 0128eaf06394e..b360a3a119a20 100644 --- a/base/task.jl +++ b/base/task.jl @@ -164,7 +164,11 @@ function task_done_hook(t::Task) end end end - wait() + # if a finished task accidentally gets into the queue, wait() + # could return. in that case just take the next task off the queue. + while true + wait() + end end end diff --git a/src/task.c b/src/task.c index 4fc7b28a003d6..31c3dc8b91085 100644 --- a/src/task.c +++ b/src/task.c @@ -368,7 +368,9 @@ JL_THREAD jl_value_t * volatile jl_task_arg_in_transit; extern int jl_in_gc; DLLEXPORT jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg) { - if (t->state == done_sym || t->state == failed_sym) { + if (t->state == done_sym || t->state == failed_sym || + // task started but stkbuf NULL'd => finish_task ran + (t->last != NULL && t->stkbuf == NULL && t != jl_current_task)) { if (t->exception != jl_nothing) jl_throw(t->exception); return t->result; diff --git a/test/parallel.jl b/test/parallel.jl index 021fb2ed23a54..e0a15ed78af98 100644 --- a/test/parallel.jl +++ b/test/parallel.jl @@ -517,3 +517,13 @@ let A = Any[] end @test length(A) == 1 end + +# issue #13168 +function f13168(n) + val = 0 + for i=1:n val+=sum(rand(n,n)^2) end + val +end +let t = schedule(@task f13168(100)) + @test schedule(t) === t +end From af98334c8da0b98a737126ad64e6b8cb93f28fa5 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 16 Sep 2015 13:38:20 -0400 Subject: [PATCH 0126/1938] Revert "Revert "TypeError is needed by type-intersection during --compile=all"" This reverts commit eaf75cbd37cd3cda50b58f8b4f5b951c55cce1d5. --- base/base.jl | 7 ------- base/boot.jl | 35 ++++++++++++++++++++--------------- base/essentials.jl | 1 + src/init.c | 2 +- src/toplevel.c | 1 - 5 files changed, 22 insertions(+), 24 deletions(-) diff --git a/base/base.jl b/base/base.jl index 18cdf4e4d8f39..909bb062b6a13 100644 --- a/base/base.jl +++ b/base/base.jl @@ -11,13 +11,6 @@ type SystemError <: Exception SystemError(p::AbstractString) = new(p, Libc.errno()) end -type TypeError <: Exception - func::Symbol - context::AbstractString - expected::Type - got -end - type ParseError <: Exception msg::AbstractString end diff --git a/base/boot.jl b/base/boot.jl index 2a687893acebb..e79885196708a 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -142,7 +142,7 @@ export # errors BoundsError, DivideError, DomainError, Exception, InexactError, InterruptException, OutOfMemoryError, ReadOnlyMemoryError, OverflowError, - StackOverflowError, SegmentationFault, UndefRefError, UndefVarError, + StackOverflowError, SegmentationFault, UndefRefError, UndefVarError, TypeError, # AST representation Expr, GotoNode, LabelNode, LineNumberNode, QuoteNode, SymbolNode, TopNode, GlobalRef, NewvarNode, GenSym, @@ -210,8 +210,20 @@ else typealias UInt UInt32 end -abstract Exception +abstract AbstractString +abstract DirectIndexString <: AbstractString + +immutable ASCIIString <: DirectIndexString + data::Array{UInt8,1} +end +immutable UTF8String <: AbstractString + data::Array{UInt8,1} +end + +typealias ByteString Union{ASCIIString,UTF8String} + +abstract Exception immutable BoundsError <: Exception a::Any i::Any @@ -232,9 +244,12 @@ immutable UndefVarError <: Exception var::Symbol end immutable InterruptException <: Exception end - -abstract AbstractString -abstract DirectIndexString <: AbstractString +type TypeError <: Exception + func::Symbol + context::AbstractString + expected::Type + got +end type SymbolNode name::Symbol @@ -242,16 +257,6 @@ type SymbolNode SymbolNode(name::Symbol, t::ANY) = new(name, t) end -immutable ASCIIString <: DirectIndexString - data::Array{UInt8,1} -end - -immutable UTF8String <: AbstractString - data::Array{UInt8,1} -end - -typealias ByteString Union{ASCIIString,UTF8String} - include(fname::ByteString) = ccall(:jl_load_, Any, (Any,), fname) # constructors for built-in types diff --git a/base/essentials.jl b/base/essentials.jl index 774fe2c789487..582f66bb2584a 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -32,6 +32,7 @@ call(T::Type{SegmentationFault}) = Core.call(T) call(T::Type{UndefRefError}) = Core.call(T) call(T::Type{UndefVarError}, var::Symbol) = Core.call(T, var) call(T::Type{InterruptException}) = Core.call(T) +call(T::Type{TypeError}, func::Symbol, context::AbstractString, expected::Type, got) = Core.call(T, func, context, expected, got) call(T::Type{SymbolNode}, name::Symbol, t::ANY) = Core.call(T, name, t) call(T::Type{GlobalRef}, modu, name::Symbol) = Core.call(T, modu, name) call(T::Type{ASCIIString}, d::Array{UInt8,1}) = Core.call(T, d) diff --git a/src/init.c b/src/init.c index 6ba9ae916f90a..5debfbbe0176b 100644 --- a/src/init.c +++ b/src/init.c @@ -712,6 +712,7 @@ void jl_get_builtin_hooks(void) jl_boundserror_type = (jl_datatype_t*)core("BoundsError"); jl_memory_exception = jl_new_struct_uninit((jl_datatype_t*)core("OutOfMemoryError")); jl_readonlymemory_exception = jl_new_struct_uninit((jl_datatype_t*)core("ReadOnlyMemoryError")); + jl_typeerror_type = (jl_datatype_t*)core("TypeError"); #ifdef SEGV_EXCEPTION jl_segv_exception = jl_new_struct_uninit((jl_datatype_t*)core("SegmentationFault")); @@ -732,7 +733,6 @@ DLLEXPORT void jl_get_system_hooks(void) jl_errorexception_type = (jl_datatype_t*)basemod("ErrorException"); jl_argumenterror_type = (jl_datatype_t*)basemod("ArgumentError"); - jl_typeerror_type = (jl_datatype_t*)basemod("TypeError"); jl_methoderror_type = (jl_datatype_t*)basemod("MethodError"); jl_loaderror_type = (jl_datatype_t*)basemod("LoadError"); jl_initerror_type = (jl_datatype_t*)basemod("InitError"); diff --git a/src/toplevel.c b/src/toplevel.c index 6fb35f6aaa1a1..1e5f34aef1214 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -138,7 +138,6 @@ jl_value_t *jl_eval_module_expr(jl_expr_t *ex) // to pick up new types from Base jl_errorexception_type = NULL; jl_argumenterror_type = NULL; - jl_typeerror_type = NULL; jl_methoderror_type = NULL; jl_loaderror_type = NULL; jl_initerror_type = NULL; From 9c0af821c9c72f879e229a16cec8c28bc0b913fa Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 16 Sep 2015 16:30:15 -0400 Subject: [PATCH 0127/1938] fix mysterious compiler performance regression 11009e152383de6f5ce665d537200b2a43f36a07 caused a large compiler performance regression for some code. By isolating each piece of the change, @vtjnash and I determined that this seemingly meaningless movement of declarations in boot.jl was the cause. After poking at it for a couple hours we still have no idea why. It's possibly related to the order of methods in Core.call. --- base/boot.jl | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/base/boot.jl b/base/boot.jl index e79885196708a..3dd891a22d519 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -211,17 +211,6 @@ else end abstract AbstractString -abstract DirectIndexString <: AbstractString - -immutable ASCIIString <: DirectIndexString - data::Array{UInt8,1} -end - -immutable UTF8String <: AbstractString - data::Array{UInt8,1} -end - -typealias ByteString Union{ASCIIString,UTF8String} abstract Exception immutable BoundsError <: Exception @@ -257,6 +246,18 @@ type SymbolNode SymbolNode(name::Symbol, t::ANY) = new(name, t) end +abstract DirectIndexString <: AbstractString + +immutable ASCIIString <: DirectIndexString + data::Array{UInt8,1} +end + +immutable UTF8String <: AbstractString + data::Array{UInt8,1} +end + +typealias ByteString Union{ASCIIString,UTF8String} + include(fname::ByteString) = ccall(:jl_load_, Any, (Any,), fname) # constructors for built-in types From 3d6c872f74aae4b81e549d9b75aec1fc0775d5b8 Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Tue, 15 Sep 2015 09:37:16 +0530 Subject: [PATCH 0128/1938] fix tracking of serialization state for Function types --- base/serialize.jl | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/base/serialize.jl b/base/serialize.jl index afd57c6fd6bec..752d06da984d2 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -507,24 +507,30 @@ function deserialize(s::SerializationState, ::Type{Function}) if b==0 name = deserialize(s)::Symbol if !isdefined(Base,name) - return (args...)->error("function $name not defined on process $(myid())") + f = (args...)->error("function $name not defined on process $(myid())") + else + f = getfield(Base,name)::Function end - return getfield(Base,name)::Function elseif b==2 mod = deserialize(s)::Module name = deserialize(s)::Symbol if !isdefined(mod,name) - return (args...)->error("function $name not defined on process $(myid())") + f = (args...)->error("function $name not defined on process $(myid())") + else + f = getfield(mod,name)::Function end - return getfield(mod,name)::Function elseif b==3 env = deserialize(s) - return ccall(:jl_new_gf_internal, Any, (Any,), env)::Function + f = ccall(:jl_new_gf_internal, Any, (Any,), env)::Function + else + linfo = deserialize(s) + f = ccall(:jl_new_closure, Any, (Ptr{Void}, Ptr{Void}, Any), C_NULL, C_NULL, linfo)::Function + deserialize_cycle(s, f) + f.env = deserialize(s) + return f end - linfo = deserialize(s) - f = ccall(:jl_new_closure, Any, (Ptr{Void}, Ptr{Void}, Any), C_NULL, C_NULL, linfo)::Function + deserialize_cycle(s, f) - f.env = deserialize(s) return f end From 3b8d41aa1ba1935b7c324e77b2557cc9dfbb8ede Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Wed, 16 Sep 2015 12:24:35 +0530 Subject: [PATCH 0129/1938] added tests [ci skip] --- base/serialize.jl | 2 +- test/serialize.jl | 24 +++++++++++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/base/serialize.jl b/base/serialize.jl index 752d06da984d2..aeb6e89adb43a 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -529,7 +529,7 @@ function deserialize(s::SerializationState, ::Type{Function}) f.env = deserialize(s) return f end - + deserialize_cycle(s, f) return f end diff --git a/test/serialize.jl b/test/serialize.jl index 0d5a6554c04fc..1b63b7fe25734 100644 --- a/test/serialize.jl +++ b/test/serialize.jl @@ -280,16 +280,34 @@ create_serialization_stream() do s end # cycles +module CycleFoo + echo(x)=x +end create_serialization_stream() do s - A = Any[1,2,3,4,5] + echo(x) = x + afunc = (x)->x + A = Any[1,2,3,abs,abs,afunc,afunc,echo,echo,CycleFoo.echo,CycleFoo.echo,4,5] A[3] = A serialize(s, A) seekstart(s) b = deserialize(s) @test b[3] === b @test b[1] == 1 - @test b[5] == 5 - @test length(b) == 5 + @test b[4] === abs + @test b[5] === b[4] + @test b[5](-1) == 1 + + @test b[6] === b[7] + @test b[6]("Hello") == "Hello" + + @test b[8] === b[9] + @test b[8]("World") == "World" + + @test b[10] === b[11] + @test b[10]("foobar") == "foobar" + + @test b[end] == 5 + @test length(b) == length(A) @test isa(b,Vector{Any}) end From a9ae0adce6c90dd8bb84befef21291af3938a0c4 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 16 Sep 2015 19:14:27 -0400 Subject: [PATCH 0130/1938] fix serializing functions with cycles, and a bug in serializing Expr fixes #12848 --- base/serialize.jl | 25 +++++++++++++++---------- src/alloc.c | 5 +++++ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/base/serialize.jl b/base/serialize.jl index aeb6e89adb43a..5681286d1d6fb 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -224,7 +224,7 @@ function serialize(s::SerializationState, n::BigFloat) end function serialize(s::SerializationState, ex::Expr) - serialize_cycle(s, e) && return + serialize_cycle(s, ex) && return l = length(ex.args) if l <= 255 writetag(s.io, EXPR_TAG) @@ -265,8 +265,6 @@ function serialize(s::SerializationState, m::Module) end function serialize(s::SerializationState, f::Function) - serialize_cycle(s, f) && return - writetag(s.io, FUNCTION_TAG) name = false if isgeneric(f) name = f.env.name @@ -275,6 +273,7 @@ function serialize(s::SerializationState, f::Function) end if isa(name,Symbol) if isdefined(Base,name) && is(f,getfield(Base,name)) + writetag(s.io, FUNCTION_TAG) write(s.io, UInt8(0)) serialize(s, name) return @@ -288,18 +287,23 @@ function serialize(s::SerializationState, f::Function) if mod !== () if isdefined(mod,name) && is(f,getfield(mod,name)) # toplevel named func + writetag(s.io, FUNCTION_TAG) write(s.io, UInt8(2)) serialize(s, mod) serialize(s, name) return end end + serialize_cycle(s, f) && return + writetag(s.io, FUNCTION_TAG) write(s.io, UInt8(3)) serialize(s, f.env) else + serialize_cycle(s, f) && return + writetag(s.io, FUNCTION_TAG) + write(s.io, UInt8(1)) linfo = f.code @assert isa(linfo,LambdaStaticData) - write(s.io, UInt8(1)) serialize(s, linfo) serialize(s, f.env) end @@ -520,17 +524,18 @@ function deserialize(s::SerializationState, ::Type{Function}) f = getfield(mod,name)::Function end elseif b==3 - env = deserialize(s) - f = ccall(:jl_new_gf_internal, Any, (Any,), env)::Function + f = ccall(:jl_new_gf_internal, Any, (Any,), nothing)::Function + deserialize_cycle(s, f) + f.env = deserialize(s) else - linfo = deserialize(s) - f = ccall(:jl_new_closure, Any, (Ptr{Void}, Ptr{Void}, Any), C_NULL, C_NULL, linfo)::Function + f = ccall(:jl_new_closure, Any, (Ptr{Void}, Ptr{Void}, Any), + cglobal(:jl_trampoline), C_NULL, nothing)::Function deserialize_cycle(s, f) + f.code = li = deserialize(s) + f.fptr = ccall(:jl_linfo_fptr, Ptr{Void}, (Any,), li) f.env = deserialize(s) - return f end - deserialize_cycle(s, f) return f end diff --git a/src/alloc.c b/src/alloc.c index c0a5c22c8fdb8..f0621a9a5e674 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -325,6 +325,11 @@ DLLEXPORT jl_function_t *jl_new_closure(jl_fptr_t fptr, jl_value_t *env, return f; } +DLLEXPORT jl_fptr_t *jl_linfo_fptr(jl_lambda_info_t *linfo) +{ + return linfo->fptr; +} + DLLEXPORT jl_lambda_info_t *jl_new_lambda_info(jl_value_t *ast, jl_svec_t *sparams, jl_module_t *ctx) { From 5d0ba2d9ff1c26e7bf826d0cd226dd6033229665 Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Wed, 16 Sep 2015 21:34:03 -0400 Subject: [PATCH 0131/1938] =?UTF-8?q?Fix=20#13175,=20segfault=20with=20?= =?UTF-8?q?=F0=9F=91=BB=20fields?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/codegen.cpp | 2 +- test/core.jl | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index fd16cb4208efa..d482dbad70b9d 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2048,7 +2048,7 @@ static Value *emit_bits_compare(const jl_cgval_t &arg1, const jl_cgval_t &arg2, Value *subAns, *fld1, *fld2; fld1 = builder.CreateConstGEP2_32(LLVM37_param(at) varg1, 0, i); fld2 = builder.CreateConstGEP2_32(LLVM37_param(at) varg2, 0, i); - if (type_is_ghost(fld1->getType())) + if (type_is_ghost(fld1->getType()->getPointerElementType())) continue; subAns = emit_bits_compare(mark_julia_slot(fld1, fldty), mark_julia_slot(fld2, fldty), ctx); answer = builder.CreateAnd(answer, subAns); diff --git a/test/core.jl b/test/core.jl index d3b44c6bd7dee..2659dece308d1 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3366,3 +3366,12 @@ end typealias B11888{T} A11888{A11888{A11888{T}}} @test sizeof(B11888{B11888{Int64}}) == (1 << 24) * 8 + +# issue #13175 +immutable EmptyImmutable13175 end +immutable EmptyIIOtherField13175 + x::EmptyImmutable13175 + y::Float64 +end +@test EmptyIIOtherField13175(EmptyImmutable13175(), 1.0) == EmptyIIOtherField13175(EmptyImmutable13175(), 1.0) +@test EmptyIIOtherField13175(EmptyImmutable13175(), 1.0) != EmptyIIOtherField13175(EmptyImmutable13175(), 2.0) From 3e3f3533f6ff889a0aafeca258e095816b727efa Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Wed, 16 Sep 2015 19:21:38 -0700 Subject: [PATCH 0132/1938] Constify deprecated bindings hopefully fixes auto_unbox: unable to determine argument type ref https://github.com/JuliaLang/julia/pull/13107#issuecomment-140874488 --- base/deprecated.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 354907e875ac0..5a78ef2b120cf 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -107,7 +107,7 @@ deprecate(m::Module, s::Symbol) = ccall(:jl_deprecate_binding, Void, (Any, Any), macro deprecate_binding(old, new) Expr(:toplevel, Expr(:export, esc(old)), - Expr(:(=), esc(old), esc(new)), + Expr(:const, Expr(:(=), esc(old), esc(new))), Expr(:call, :deprecate, Expr(:quote, old))) end From 04e94c287aad80ecf8e8413e9d533ba645f45bf6 Mon Sep 17 00:00:00 2001 From: catawbasam Date: Thu, 17 Sep 2015 01:31:31 +0000 Subject: [PATCH 0133/1938] basedocs: abstract, bitstype, module, baremodule, macro macro 4 spaces --- base/docs/basedocs.jl | 55 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/base/docs/basedocs.jl b/base/docs/basedocs.jl index 15916b020c09c..cd2a536c3dc01 100644 --- a/base/docs/basedocs.jl +++ b/base/docs/basedocs.jl @@ -70,6 +70,61 @@ keywords[:export] = doc""" Test.bar(1) # 1 """ +keywords[:abstract] = doc""" + `abstract` declares a type that cannot be instantiated, and serves only as a node in the type graph, + thereby describing sets of related concrete types: those concrete types which are their descendants. + Abstract types form the conceptual hierarchy which makes Julia’s type system more than just a collection of object implementations. + For example: + + abstract Number + abstract Real <: Number + + `abstract Number` has no supertype, whereas `abstract Real` is an abstract subtype of `Number`. + """ + +keywords[:module] = doc""" + `module` declares a Module, which is a separate global variable workspace. Within a module, you can control which names from other modules are visible (via importing), and specify which of your names are intended to be public (via exporting). For example: + + module + import Base.show + export MyType, foo + + type MyType + x + end + + bar(x) = 2x + foo(a::MyType) = bar(a.x) + 1 + show(io, a::MyType) = print(io, "MyType $(a.x)") + end + + Modules allow you to create top-level definitions without worrying about name conflicts when your code is used together with somebody else’s. + """ + +keywords[:baremodule] = doc""" + `baremodule` declares a module that does not contain `using Base`, `import Base.call`, + or a definition of `eval`. It does still import `Core`. + """ + +keywords[:bitstype] = doc""" + `bitstype` declares a concrete type whose data consists of plain old bits. Classic examples of bits types are integers and floating-point values. Some example built-in bits type declarations: + + bitstype 32 Char + bitstype 8 Bool <: Integer + + The first parameter indicates how many bits of storage the type requires. Currently, only sizes that are multiples of 8 bits are supported. The second parameter gives the name of the type. The `Bool` declaration shows how a bits type can be optionally declared to be a subtype of some supertype. + """ + +keywords[:macro] = doc""" + `macro` defines a method to include generated code in the final body of a program. A macro maps a tuple of arguments to a returned expression, and the resulting expression is compiled directly rather than requiring a runtime `eval()` call. Macro arguments may include expressions, literal values, and symbols. For example: + + macro sayhello(name) + return :( println("Hello, ", $name) ) + end + + This macro takes one argument: `name`. When `@sayhello` is encountered, the quoted expression is expanded to interpolate the value of the argument into the final expression. + """ + keywords[:const] = doc""" `const` is used to declare global variables which are also constant. In almost all code (and particularly performance sensitive code) From db02cda5919ca53d03b86d13fea92efc62b6e0fb Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 16 Sep 2015 16:47:10 -0400 Subject: [PATCH 0134/1938] handle passing of Ref{GhostType} to cfunction signature (fix #13031) --- src/codegen.cpp | 21 ++++++++++++++++----- test/ccall.jl | 9 +++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index d482dbad70b9d..fbfa60b1a47b1 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3841,9 +3841,10 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t Function::arg_iterator AI = cw->arg_begin(); Value *sretPtr = NULL; if (sret) - sretPtr = AI++; //const Argument &fArg = *AI++; + sretPtr = AI++; - for (size_t i=0; i < nargs; i++) { + size_t FParamIndex = 0; + for (size_t i = 0; i < nargs; i++) { Value *val = AI++; jl_value_t *jargty = jl_nth_slot_type(lam->specTypes, i); @@ -3854,8 +3855,18 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t } else { Type *t = julia_type_to_llvm(jargty); - val = builder.CreatePointerCast(val, t->getPointerTo()); - val = builder.CreateAlignedLoad(val, 1); // make no alignment assumption about pointer from C + if (type_is_ghost(t)) { + if (specsig) { + continue; // ghost types are skipped by the specsig method signature + } + else { + val = boxed(ghostValue(jargty), &ctx, jargty); + } + } + else { + val = builder.CreatePointerCast(val, t->getPointerTo()); + val = builder.CreateAlignedLoad(val, 1); // make no alignment assumption about pointer from C + } } } else { @@ -3865,7 +3876,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t } // figure out how to repack this type - Type *at = specsig ? theFptr->getFunctionType()->getParamType(i) : jl_pvalue_llvmt; + Type *at = specsig ? theFptr->getFunctionType()->getParamType(FParamIndex++) : jl_pvalue_llvmt; if (val->getType() != at) { if (at == jl_pvalue_llvmt) { assert(jl_is_leaf_type(jargty)); diff --git a/test/ccall.jl b/test/ccall.jl index 4dfa2d58c3dfe..2335e60b4f0ca 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -209,3 +209,12 @@ for (t,v) in ((Complex{Int32},:ci32),(Complex{Int64},:ci64), #b = ccall(cfunction($fname,Any,(Ref{Any},)),Any,(Ref{Any},),$v) # unimplemented end end + +# issue 13031 +foo13031(x) = Cint(1) +foo13031p = cfunction(foo13031, Cint, (Ref{Tuple{}},)) +ccall(foo13031p, Cint, (Ref{Tuple{}},), ()) + +foo13031(x,y,z) = z +foo13031p = cfunction(foo13031, Cint, (Ref{Tuple{}},Ref{Tuple{}},Cint)) +ccall(foo13031p, Cint, (Ref{Tuple{}},Ref{Tuple{}},Cint), (), (), 8) From 7ed546e754732ee516054269c69cdc98e9ad2f11 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 16 Sep 2015 17:36:28 -0400 Subject: [PATCH 0135/1938] handle TypeVar showing up in more invalid places (as symbolnode.typ) after inference passes (fix #13173) --- src/codegen.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index fbfa60b1a47b1..592e68c6f450d 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3024,7 +3024,7 @@ static jl_cgval_t emit_checked_var(Value *bp, jl_sym_t *name, jl_codectx_t *ctx, return mark_julia_type(v, jl_any_type); } -static jl_cgval_t emit_var(jl_sym_t *sym, jl_value_t *ty, jl_codectx_t *ctx, bool isboxed) +static jl_cgval_t emit_var(jl_sym_t *sym, jl_codectx_t *ctx, bool isboxed) { bool isglobal = is_global(sym, ctx); if (isglobal) { @@ -3260,14 +3260,18 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx, bool isboxed, b if (jl_is_symbol(expr)) { if (!valuepos) return jl_cgval_t(); // value not used, no point in doing codegen for it jl_sym_t *sym = (jl_sym_t*)expr; - return emit_var(sym, (jl_value_t*)jl_any_type, ctx, isboxed); + return emit_var(sym, ctx, isboxed); } if (jl_is_symbolnode(expr)) { if (!valuepos) return jl_cgval_t(); // value not used, no point in doing codegen for it jl_sym_t *sym = jl_symbolnode_sym(expr); - return remark_julia_type( - emit_var(sym, jl_symbolnode_type(expr), ctx, isboxed), - jl_symbolnode_type(expr)); // patch up typ to match SymbolNode.type + jl_value_t *typ = jl_symbolnode_type(expr); + if (jl_is_typevar(typ)) + typ = ((jl_tvar_t*)typ)->ub; + jl_cgval_t val = emit_var(sym, ctx, isboxed); + if (val.isboxed) + val = remark_julia_type(val, typ); // patch up typ to match SymbolNode.type + return val; // patch up typ to match SymbolNode.type } if (jl_is_gensym(expr)) { if (!valuepos) return jl_cgval_t(); // value not used, no point in doing codegen for it From 31424ea9c5aa1492331de1b2be58f38bc3373ba6 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 16 Sep 2015 18:13:07 -0400 Subject: [PATCH 0136/1938] Makefile src/libccalltest needs to depend on julia-deps target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index db6e3a25de68f..69770e3d5986d 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,7 @@ julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia julia-base: julia-deps $(build_sysconfdir)/julia/juliarc.jl $(build_man1dir)/julia.1 @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/base -julia-libccalltest: +julia-libccalltest: julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src libccalltest julia-src-release julia-src-debug : julia-src-% : julia-deps From c501ac537d661636e0a7e13d6a9832c61b2942ba Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 16 Sep 2015 18:40:54 -0400 Subject: [PATCH 0137/1938] remove unused emit_reg2mem function --- src/ccall.cpp | 2 +- src/cgutils.cpp | 3 +-- src/codegen.cpp | 8 ++------ 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 1a5b7083cb313..e460462059149 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -788,7 +788,7 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c jl_error("Return type of llvmcall'ed function does not match declared return type"); } - return mark_julia_type(emit_reg2mem(inst, ctx), rtt); + return mark_julia_type(inst, rtt); } // --- code generator for ccall itself --- diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 4e52e57898f8d..496d04d692acf 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1107,7 +1107,6 @@ static jl_cgval_t typed_load(Value *ptr, Value *idx_0based, jl_value_t *jltype, if (elty == jl_pvalue_llvmt) { null_pointer_check(elt, ctx); } - elt = emit_reg2mem(elt, ctx); //} if (isbool) return mark_julia_type(builder.CreateTrunc(elt, T_int1), jltype); @@ -1905,7 +1904,7 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg } idx++; } - return mark_julia_type(emit_reg2mem(strct, ctx), ty); + return mark_julia_type(strct, ty); } Value *f1 = NULL; int fieldStart = ctx->gc.argDepth; diff --git a/src/codegen.cpp b/src/codegen.cpp index 592e68c6f450d..63f87ace09c91 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -636,10 +636,6 @@ static inline jl_cgval_t mark_julia_const(jl_value_t *jv) } -static Value *emit_reg2mem(Value *v, jl_codectx_t *ctx) { - return v; -} - // --- utilities --- extern "C" { @@ -2771,8 +2767,8 @@ static jl_cgval_t emit_call_function_object(jl_function_t *f, Value *theF, Value idx++; } assert(idx == nfargs); - result = builder.CreateCall(prepare_call(cf), ArrayRef(&argvals[0],nfargs)); - return mark_julia_type(emit_reg2mem(result, ctx), jlretty); + result = builder.CreateCall(prepare_call(cf), ArrayRef(&argvals[0], nfargs)); + return mark_julia_type(result, jlretty); } return mark_julia_type(emit_jlcall(theFptr, theF, &args[1], nargs, ctx), jl_any_type); // (typ will be patched up by caller) } From d801b571af8a2f0f073ea184a788e510fccebac4 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 17 Sep 2015 00:22:06 -0400 Subject: [PATCH 0138/1938] fix a codegen bug where a ghost root could get replaced by an undef root --- src/codegen.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 63f87ace09c91..1a17a160db997 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3161,7 +3161,7 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx) // it's a local variable or closure variable jl_varinfo_t &vi = ctx->vars[s]; if (!vi.memloc && !vi.hasGCRoot && vi.used - && !vi.isArgument && !is_stable_expr(r, ctx) && !vi.value.isghost) { + && !vi.isArgument && !is_stable_expr(r, ctx)) { Instruction *newroot = cast(emit_local_slot(ctx->gc.argSpaceSize++, ctx)); newroot->removeFromParent(); // move it to the gc frame basic block so it can be reused as needed newroot->insertAfter(ctx->gc.last_gcframe_inst); @@ -3201,6 +3201,7 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx) } else { // SSA variable w/o gcroot, just track the value info + assert(vi.isSA); if (store_unboxed_p(vi.value.typ) && rval_info.ispointer) { // emit a copy of boxed isbits values. TODO: elid this copy if unnecessary rval_info = mark_julia_type( emit_unbox(julia_type_to_llvm(vi.value.typ), rval_info, vi.value.typ), @@ -4164,6 +4165,7 @@ static Function *emit_function(jl_lambda_info_t *lam) if (!jl_is_type(typ)) typ = (jl_value_t*)jl_any_type; varinfo.value = mark_julia_type((Value*)NULL, typ); + varinfo.hasGCRoot = true; } // step 3. some variable analysis @@ -4600,10 +4602,11 @@ static Function *emit_function(jl_lambda_info_t *lam) jl_sym_t *vname = ((jl_sym_t*)jl_cellref(vi,0)); assert(jl_is_symbol(vname)); jl_varinfo_t &varinfo = ctx.vars[vname]; - varinfo.memloc = builder.CreatePointerCast( - emit_nthptr_addr(envArg, i + offsetof(jl_svec_t,data) / sizeof(void*)), - jl_ppvalue_llvmt); - varinfo.hasGCRoot = true; + if (!varinfo.value.isghost) { + varinfo.memloc = builder.CreatePointerCast( + emit_nthptr_addr(envArg, i + offsetof(jl_svec_t,data) / sizeof(void*)), + jl_ppvalue_llvmt); + } } } @@ -4618,6 +4621,7 @@ static Function *emit_function(jl_lambda_info_t *lam) assert(!varinfo.memloc); // arguments shouldn't also be in the closure env if (varinfo.value.isghost) { // no need to explicitly load/store a ghost value + varinfo.hasGCRoot = true; continue; } if (store_unboxed_p(s, &ctx)) { @@ -4634,8 +4638,9 @@ static Function *emit_function(jl_lambda_info_t *lam) jl_varinfo_t &varinfo = ctx.vars[s]; if (varinfo.memloc || varinfo.isArgument) continue; // gc root already created - if (store_unboxed_p(s, &ctx)) { - varinfo.hasGCRoot = true; + if (store_unboxed_p(s, &ctx) || + (varinfo.value.isghost && !varinfo.usedUndef)) { + varinfo.hasGCRoot = true; // will never need a gc-root for this } else if (varinfo.hasGCRoot) { Value *lv = emit_local_slot(varnum, &ctx); From 13c83db372527c9c489d751cfa3bd061f8ecd5f0 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 16 Sep 2015 20:15:07 -0400 Subject: [PATCH 0139/1938] in codegen, use StructRet where appropriate it seems that the llvm 3.3 return type legalizer may not be properly handling large types. this is more efficient anyways. fixes #8932, should fix #12163, probably fixes #7434 --- src/cgutils.cpp | 6 +++ src/codegen.cpp | 110 ++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 93 insertions(+), 23 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 496d04d692acf..32612a2f0a3fc 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -790,6 +790,12 @@ static bool is_tupletype_homogeneous(jl_svec_t *t) return true; } +static bool deserves_sret(jl_value_t *dt, Type *T) +{ + assert(jl_is_datatype(dt)); + return jl_datatype_size(dt) > sizeof(void*) && !T->isFloatingPointTy(); +} + // --- generating various field accessors --- static Value *emit_nthptr_addr(Value *v, ssize_t n) diff --git a/src/codegen.cpp b/src/codegen.cpp index 1a17a160db997..94e548667fad3 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -530,6 +530,7 @@ typedef struct { std::string funcName; jl_sym_t *vaName; // name of vararg argument bool vaStack; // varargs stack-allocated + bool sret; int nReqArgs; int lineno; std::vector boundsCheck; @@ -2714,7 +2715,6 @@ static jl_cgval_t emit_call_function_object(jl_function_t *f, Value *theF, Value jl_value_t **args, size_t nargs, jl_codectx_t *ctx) { - Value *result; if (f!=NULL && specialized && f->linfo!=NULL && f->linfo->specFunctionObject!=NULL) { // emit specialized call site jl_value_t *jlretty = jl_ast_rettype(f->linfo, f->linfo->ast); @@ -2722,7 +2722,14 @@ static jl_cgval_t emit_call_function_object(jl_function_t *f, Value *theF, Value FunctionType *cft = cf->getFunctionType(); size_t nfargs = cft->getNumParams(); Value **argvals = (Value**) alloca(nfargs*sizeof(Value*)); + bool sret = cf->hasStructRetAttr(); unsigned idx = 0; + Value *result; + if (sret) { + result = emit_static_alloca(cft->getParamType(0)->getContainedType(0), ctx); + argvals[idx] = result; + idx++; + } for(size_t i=0; i < nargs; i++) { Type *at = cft->getParamType(idx); jl_value_t *jt = jl_nth_slot_type(f->linfo->specTypes,i); @@ -2767,8 +2774,9 @@ static jl_cgval_t emit_call_function_object(jl_function_t *f, Value *theF, Value idx++; } assert(idx == nfargs); - result = builder.CreateCall(prepare_call(cf), ArrayRef(&argvals[0], nfargs)); - return mark_julia_type(result, jlretty); + CallInst *call = builder.CreateCall(prepare_call(cf), ArrayRef(&argvals[0], nfargs)); + call->setAttributes(cf->getAttributes()); + return sret ? mark_julia_slot(result, jlretty) : mark_julia_type(call, jlretty); } return mark_julia_type(emit_jlcall(theFptr, theF, &args[1], nargs, ctx), jl_any_type); // (typ will be patched up by caller) } @@ -3811,6 +3819,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t jl_codectx_t ctx; ctx.f = cw; ctx.linfo = lam; + ctx.sret = false; allocate_gc_frame(0, b0, &ctx); // Save the Function object reference @@ -3824,15 +3833,17 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t lam->cFunctionList = list2; // See whether this function is specsig or jlcall - bool specsig; + bool specsig, jlfunc_sret; Function *theFptr; if (lam->specFunctionObject != NULL) { theFptr = (Function*)lam->specFunctionObject; specsig = true; + jlfunc_sret = theFptr->hasStructRetAttr(); } else { theFptr = (Function*)lam->functionObject; specsig = false; + jlfunc_sret = false; } assert(theFptr); @@ -3844,7 +3855,17 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t if (sret) sretPtr = AI++; + Value *result; size_t FParamIndex = 0; + if (jlfunc_sret) { + if (sret) + result = sretPtr; + else + result = builder.CreateAlloca(theFptr->getFunctionType()->getParamType(0)->getContainedType(0)); + args.push_back(result); + FParamIndex++; + } + for (size_t i = 0; i < nargs; i++) { Value *val = AI++; jl_value_t *jargty = jl_nth_slot_type(lam->specTypes, i); @@ -3920,7 +3941,12 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t // Create the call Value *r; if (specsig) { - r = builder.CreateCall(prepare_call(theFptr), ArrayRef(args)); + CallInst *call = builder.CreateCall(prepare_call(theFptr), ArrayRef(args)); + call->setAttributes(theFptr->getAttributes()); + if (jlfunc_sret) + r = builder.CreateLoad(result); + else + r = call; } else { r = emit_jlcall(theFptr, literal_pointer_val((jl_value_t*)ff), 0, nargs, &ctx); @@ -3934,6 +3960,9 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t r = boxed(mark_julia_type(r, jlrettype), &ctx, jlrettype); } } + else if (sret && jlfunc_sret) { + // nothing to do + } else if (!type_is_ghost(crt)) { if (sret) prt = fargt_sig[0]->getContainedType(0); // sret is a PointerType @@ -3998,7 +4027,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t } // generate a julia-callable function that calls f (AKA lam) -static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Function *f) +static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Function *f, bool sret) { std::stringstream funcName; const std::string &fname = f->getName().str(); @@ -4012,9 +4041,9 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Funct funcName.str(), f->getParent()); addComdat(w); Function::arg_iterator AI = w->arg_begin(); - AI++; //const Argument &fArg = *AI++; + /* const Argument &fArg = */ *AI++; Value *argArray = AI++; - //const Argument &argCount = *AI++; + /* const Argument &argCount = *AI++; */ BasicBlock *b0 = BasicBlock::Create(jl_LLVMContext, "top", w); builder.SetInsertPoint(b0); @@ -4022,13 +4051,21 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Funct builder.SetCurrentDebugLocation(noDbg); jl_codectx_t ctx; + ctx.f = w; ctx.linfo = lam; + ctx.sret = false; allocate_gc_frame(0, b0, &ctx); size_t nargs = jl_array_dim0(jl_lam_args(ast)); size_t nfargs = f->getFunctionType()->getNumParams(); Value **args = (Value**) alloca(nfargs*sizeof(Value*)); unsigned idx = 0; + Value *result; + if (sret) { + result = builder.CreateAlloca(f->getFunctionType()->getParamType(0)->getContainedType(0)); + args[idx] = result; + idx++; + } for(size_t i=0; i < nargs; i++) { jl_value_t *ty = jl_nth_slot_type(lam->specTypes, i); Type *lty = julia_type_to_llvm(ty); @@ -4048,9 +4085,16 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Funct } // TODO: consider pulling the function pointer out of fArg so these // wrappers can be reused for different functions of the same type. - Value *r = builder.CreateCall(prepare_call(f), ArrayRef(&args[0], nfargs)); - if (r->getType() != jl_pvalue_llvmt) { - r = boxed(mark_julia_type(r, jl_ast_rettype(lam, (jl_value_t*)ast)), &ctx); + CallInst *call = builder.CreateCall(prepare_call(f), ArrayRef(&args[0], nfargs)); + call->setAttributes(f->getAttributes()); + Value *r; + if (sret || call->getType() != jl_pvalue_llvmt) { + jl_value_t *ty = jl_ast_rettype(lam, (jl_value_t*)ast); + jl_cgval_t r_typed = sret ? mark_julia_slot(result, ty) : mark_julia_type(call, ty); + r = boxed(r_typed, &ctx); + } + else { + r = call; } builder.CreateRet(r); @@ -4215,8 +4259,15 @@ static Function *emit_function(jl_lambda_info_t *lam) #endif funcName << "_" << globalUnique++; + ctx.sret = false; if (specsig) { // assumes !va std::vector fsig(0); + Type *rt = (jlrettype == (jl_value_t*)jl_void_type ? T_void : julia_type_to_llvm(jlrettype)); + if (rt != jl_pvalue_llvmt && rt != T_void && deserves_sret(jlrettype, rt)) { + ctx.sret = true; + fsig.push_back(rt->getPointerTo()); + rt = T_void; + } for(size_t i=0; i < jl_nparams(lam->specTypes); i++) { Type *ty = julia_type_to_llvm(jl_tparam(lam->specTypes,i)); if (type_is_ghost(ty)) @@ -4225,17 +4276,18 @@ static Function *emit_function(jl_lambda_info_t *lam) ty = PointerType::get(ty,0); fsig.push_back(ty); } - Type *rt = (jlrettype == (jl_value_t*)jl_void_type ? T_void : julia_type_to_llvm(jlrettype)); f = Function::Create(FunctionType::get(rt, fsig, false), imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, funcName.str(), m); + if (ctx.sret) + f->addAttribute(1, Attribute::StructRet); addComdat(f); if (lam->specFunctionObject == NULL) { lam->specFunctionObject = (void*)f; lam->specFunctionID = jl_assign_functionID(f); } if (lam->functionObject == NULL) { - Function *fwrap = gen_jlcall_wrapper(lam, ast, f); + Function *fwrap = gen_jlcall_wrapper(lam, ast, f, ctx.sret); lam->functionObject = (void*)fwrap; lam->functionID = jl_assign_functionID(fwrap); } @@ -4401,7 +4453,7 @@ static Function *emit_function(jl_lambda_info_t *lam) varinfo.dinfo = ctx.dbuilder->createParameterVariable( SP, // Scope (current function will be fill in later) argname->name, // Variable name - i+1, // Argument number (1-based) + ctx.sret + i + 1, // Argument number (1-based) topfile, // File ctx.lineno == -1 ? 0 : ctx.lineno, // Line // Variable type @@ -4416,7 +4468,7 @@ static Function *emit_function(jl_lambda_info_t *lam) julia_type_to_di(varinfo.value.typ, ctx.dbuilder, specsig), // Variable type false, // May be optimized out 0, // Flags (TODO: Do we need any) - i+1); // Argument number (1-based) + ctx.sret + i + 1); // Argument number (1-based) #endif } if (va) { @@ -4424,10 +4476,10 @@ static Function *emit_function(jl_lambda_info_t *lam) ctx.vars[ctx.vaName].dinfo = ctx.dbuilder->createParameterVariable( SP, // Scope (current function will be fill in later) ctx.vaName->name, // Variable name - nreq + 1, // Argument number (1-based) + ctx.sret + nreq + 1, // Argument number (1-based) topfile, // File ctx.lineno == -1 ? 0 : ctx.lineno, // Line (for now, use lineno of the function) - julia_type_to_di(ctx.vars[ctx.vaName].value.typ,ctx.dbuilder,false)); + julia_type_to_di(ctx.vars[ctx.vaName].value.typ, ctx.dbuilder, false)); #else ctx.vars[ctx.vaName].dinfo = ctx.dbuilder->createLocalVariable( llvm::dwarf::DW_TAG_arg_variable, // Tag @@ -4435,10 +4487,10 @@ static Function *emit_function(jl_lambda_info_t *lam) ctx.vaName->name, // Variable name topfile, // File ctx.lineno == -1 ? 0 : ctx.lineno, // Line (for now, use lineno of the function) - julia_type_to_di(ctx.vars[ctx.vaName].value.typ,ctx.dbuilder,false), // Variable type + julia_type_to_di(ctx.vars[ctx.vaName].value.typ, ctx.dbuilder, false), // Variable type false, // May be optimized out 0, // Flags (TODO: Do we need any) - nreq + 1); // Argument number (1-based) + ctx.sret + nreq + 1); // Argument number (1-based) #endif } for(i=0; i < vinfoslen; i++) { @@ -4697,6 +4749,8 @@ static Function *emit_function(jl_lambda_info_t *lam) // step 12. move args into local variables Function::arg_iterator AI = f->arg_begin(); + if (ctx.sret) + AI++; // skip sret slot for(i=0; i < nreq; i++) { jl_sym_t *s = jl_decl_var(jl_cellref(largs,i)); jl_varinfo_t &vi = ctx.vars[s]; @@ -4923,7 +4977,7 @@ static Function *emit_function(jl_lambda_info_t *lam) if (jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == return_sym) { jl_expr_t *ex = (jl_expr_t*)stmt; Value *retval; - Type *retty = f->getReturnType(); + Type *retty = ctx.sret ? f->getFunctionType()->getParamType(0)->getContainedType(0) : f->getReturnType(); if (retty == jl_pvalue_llvmt) { retval = boxed(emit_expr(jl_exprarg(ex,0), &ctx, true),&ctx,expr_type(stmt,&ctx)); } @@ -4937,7 +4991,9 @@ static Function *emit_function(jl_lambda_info_t *lam) } if (do_malloc_log && lno != -1) mallocVisitLine(filename, lno); - if (type_is_ghost(retty)) + if (ctx.sret) + builder.CreateStore(retval, ctx.f->arg_begin()); + if (type_is_ghost(retty) || ctx.sret) builder.CreateRetVoid(); else builder.CreateRet(retval); @@ -5023,8 +5079,15 @@ extern "C" void jl_fptr_to_llvm(void *fptr, jl_lambda_info_t *lam, int specsig) std::string funcName = lam->name->name; funcName = "julia_" + funcName; if (specsig) { // assumes !va - jl_value_t *jlrettype = jl_ast_rettype(lam, (jl_value_t*)lam->ast); std::vector fsig(0); + jl_value_t *jlrettype = jl_ast_rettype(lam, (jl_value_t*)lam->ast); + Type *rt = (jlrettype == (jl_value_t*)jl_void_type ? T_void : julia_type_to_llvm(jlrettype)); + bool sret = false; + if (rt != jl_pvalue_llvmt && rt != T_void && deserves_sret(jlrettype, rt)) { + sret = true; + fsig.push_back(rt->getPointerTo()); + rt = T_void; + } for (size_t i=0; i < jl_nparams(lam->specTypes); i++) { Type *ty = julia_type_to_llvm(jl_tparam(lam->specTypes,i)); if (type_is_ghost(ty)) @@ -5033,9 +5096,10 @@ extern "C" void jl_fptr_to_llvm(void *fptr, jl_lambda_info_t *lam, int specsig) ty = PointerType::get(ty,0); fsig.push_back(ty); } - Type *rt = (jlrettype == (jl_value_t*)jl_void_type ? T_void : julia_type_to_llvm(jlrettype)); Function *f = Function::Create(FunctionType::get(rt, fsig, false), Function::ExternalLinkage, funcName, shadow_module); + if (sret) + f->addAttribute(1, Attribute::StructRet); if (lam->specFunctionObject == NULL) { lam->specFunctionObject = (void*)f; From 5bebf4127aeebac3c07c7ec1946fdab03a4729c5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 17 Sep 2015 03:27:48 -0400 Subject: [PATCH 0140/1938] remove leading heisenzero from grisu output (fix #12899, fix #10908, fix #10931) --- base/grisu/bignum.jl | 9 +++++++++ base/show.jl | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/base/grisu/bignum.jl b/base/grisu/bignum.jl index 0c9f6a28d85d1..2182602163537 100644 --- a/base/grisu/bignum.jl +++ b/base/grisu/bignum.jl @@ -170,6 +170,9 @@ function init3!( Bignums.shiftleft!(plus,exponent) Bignums.assignuint16!(minus,UInt16(1)) Bignums.shiftleft!(minus,exponent) + else + Bignums.zero!(plus) + Bignums.zero!(minus) end return end @@ -186,6 +189,9 @@ function init1!( Bignums.shiftleft!(num,1) Bignums.assignuint16!(plus,UInt16(1)) Bignums.assignuint16!(minus,UInt16(1)) + else + Bignums.zero!(plus) + Bignums.zero!(minus) end return end @@ -198,6 +204,9 @@ function init2!( if need_boundary_deltas Bignums.assignbignum!(plus,power_ten) Bignums.assignbignum!(minus,power_ten) + else + Bignums.zero!(plus) + Bignums.zero!(minus) end Bignums.multiplybyuint64!(num,UInt64(significand)) Bignums.assignuint16!(den,UInt16(1)) diff --git a/base/show.jl b/base/show.jl index 4e118d8491214..177ef7e7e43a0 100644 --- a/base/show.jl +++ b/base/show.jl @@ -983,7 +983,7 @@ function alignment( rows::AbstractVector, cols::AbstractVector, cols_if_complete::Integer, cols_otherwise::Integer, sep::Integer ) - a = [] + a = Tuple{Int, Int}[] for j in cols l = r = 0 for i in rows From bb4f0d46ffa1a85ec386e1af177dc241294c6b91 Mon Sep 17 00:00:00 2001 From: michele Date: Thu, 17 Sep 2015 14:16:34 +0200 Subject: [PATCH 0141/1938] DOC: missing ncv optional argument in eigs --- doc/stdlib/linalg.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index a04a057f69c88..5f388022cd153 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -901,7 +901,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f Conjugate transpose array ``src`` and store the result in the preallocated array ``dest``\ , which should have a size corresponding to ``(size(src,2),size(src,1))``\ . No in-place transposition is supported and unexpected results will happen if ``src`` and ``dest`` have overlapping memory regions. -.. function:: eigs(A, [B,]; nev=6, which="LM", tol=0.0, maxiter=300, sigma=nothing, ritzvec=true, v0=zeros((0,))) -> (d,[v,],nconv,niter,nmult,resid) +.. function:: eigs(A, [B,]; nev=6, ncv=max(20,2*nev+1), which="LM", tol=0.0, maxiter=300, sigma=nothing, ritzvec=true, v0=zeros((0,))) -> (d,[v,],nconv,niter,nmult,resid) .. Docstring generated from Julia source From 9eed76063b32aeb1898872bd47c64395e6a3bd59 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Thu, 17 Sep 2015 09:24:16 -0400 Subject: [PATCH 0142/1938] Doc: note convention for upper-case Module names [ci skip] --- doc/manual/variables.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/variables.rst b/doc/manual/variables.rst index a347f6789676d..506b88709c1f5 100644 --- a/doc/manual/variables.rst +++ b/doc/manual/variables.rst @@ -142,7 +142,7 @@ adopt the following conventions: - Names of variables are in lower case. - Word separation can be indicated by underscores (``'_'``), but use of underscores is discouraged unless the name would be hard to read otherwise. -- Names of ``Type``\ s begin with a capital letter and word separation is +- Names of ``Type``\ s and ``Modules``\ s begin with a capital letter and word separation is shown with upper camel case instead of underscores. - Names of ``function``\ s and ``macro``\s are in lower case, without underscores. From ca5c7df2e3c1184efc2bea05f6467cc6194e7910 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Thu, 17 Sep 2015 09:28:48 -0400 Subject: [PATCH 0143/1938] Doc: simple typo fix [ci skip] --- doc/manual/variables.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/variables.rst b/doc/manual/variables.rst index 506b88709c1f5..c2d374de36094 100644 --- a/doc/manual/variables.rst +++ b/doc/manual/variables.rst @@ -142,7 +142,7 @@ adopt the following conventions: - Names of variables are in lower case. - Word separation can be indicated by underscores (``'_'``), but use of underscores is discouraged unless the name would be hard to read otherwise. -- Names of ``Type``\ s and ``Modules``\ s begin with a capital letter and word separation is +- Names of ``Type``\ s and ``Module``\ s begin with a capital letter and word separation is shown with upper camel case instead of underscores. - Names of ``function``\ s and ``macro``\s are in lower case, without underscores. From 1700195e7fd7b5b624cbaca64ade634ac4a4da3c Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 17 Sep 2015 09:05:47 -0500 Subject: [PATCH 0144/1938] Improving documentation typo I believe the path should be `doc/stdlib` and not `rst/stdlib`. --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 43a4cbfad82c2..fb3ac0c0168ec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -88,7 +88,7 @@ If you want to edit the body of a method docstring, run the `doc/genstdlib.jl` s If you want to edit an existing docstring signature, you **first** have to change the signature in the `doc/stdlib` `..function` or `..data` definition (not the auto-generated content) and *then* edit the helpdb.jl or inline method docstrings. The existing signatures in the `doc/stdlib/*.rst` files are pattern matched to base docstrings and the new content overwrites the content in `doc/stdlib/`. The signature definitions **must** be in sync or else the pattern match will fail and documentation will be lost in the result. -To add entirely new methods to the `stdlib` documentation, first add the signature in the appropriate `rst/stdlib/*.rst` file before writing the docstring, rebuilding Julia, and re-running `doc/genstdlib.jl`. +To add entirely new methods to the `stdlib` documentation, first add the signature in the appropriate `doc/stdlib/*.rst` file before writing the docstring, rebuilding Julia, and re-running `doc/genstdlib.jl`. It is encouraged to write all new docstrings in Markdown markup. If you need to write a more complicated docstring that contains cross-references or citations it can be written in a restructured text codeblock. Many of the existing docstrings are currently restructured text codeblocks and these will be transitioned to Markdown over time. RST codeblocks are delineated with the triple-quote (\`\`\`rst \`\`\`) Makdown codeblock syntax. From ec059230cd7da9ecba875f261922df2ca5135dda Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 17 Sep 2015 00:05:24 -0400 Subject: [PATCH 0145/1938] Try fixing c++ compiler warnings about unknown union size. Fix #13172 --- src/dump.c | 4 ++-- src/julia.h | 18 +++++++----------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/dump.c b/src/dump.c index ac223e305681d..91021482bcec0 100644 --- a/src/dump.c +++ b/src/dump.c @@ -527,7 +527,7 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) write_int32(s, dt->alignment); write_int8(s, dt->haspadding); size_t fieldsize = jl_fielddesc_size(dt->fielddesc_type); - ios_write(s, (char*)&dt->fields32[0], nf * fieldsize); + ios_write(s, (char*)dt->fields, nf * fieldsize); jl_serialize_value(s, dt->types); } @@ -1129,7 +1129,7 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) dt->alignment = read_int32(s); dt->haspadding = read_int8(s); size_t fieldsize = jl_fielddesc_size(fielddesc_type); - ios_read(s, (char*)&dt->fields32[0], nf * fieldsize); + ios_read(s, (char*)dt->fields, nf * fieldsize); dt->types = (jl_svec_t*)jl_deserialize_value(s, (jl_value_t**)&dt->types); jl_gc_wb(dt, dt->types); } diff --git a/src/julia.h b/src/julia.h index 9f1988ffcf867..ea9416c66e946 100644 --- a/src/julia.h +++ b/src/julia.h @@ -290,11 +290,7 @@ typedef struct _jl_datatype_t { uint32_t uid; void *struct_decl; //llvm::Value* void *ditype; // llvm::MDNode* to be used as llvm::DIType(ditype) - union { - jl_fielddesc8_t fields8[0]; - jl_fielddesc16_t fields16[0]; - jl_fielddesc32_t fields32[0]; - }; + size_t fields[]; } jl_datatype_t; typedef struct { @@ -718,26 +714,26 @@ STATIC_INLINE jl_value_t *jl_cellset(void *a, size_t i, void *x) static inline uint32_t jl_field_##f(jl_datatype_t *st, int i) \ { \ if (st->fielddesc_type == 0) { \ - return st->fields8[i].f; \ + return ((jl_fielddesc8_t*)st->fields)[i].f; \ } \ else if (st->fielddesc_type == 1) { \ - return st->fields16[i].f; \ + return ((jl_fielddesc16_t*)st->fields)[i].f; \ } \ else { \ - return st->fields32[i].f; \ + return ((jl_fielddesc32_t*)st->fields)[i].f; \ } \ } \ static inline void jl_field_set##f(jl_datatype_t *st, int i, \ uint32_t val) \ { \ if (st->fielddesc_type == 0) { \ - st->fields8[i].f = val; \ + ((jl_fielddesc8_t*)st->fields)[i].f = val; \ } \ else if (st->fielddesc_type == 1) { \ - st->fields16[i].f = val; \ + ((jl_fielddesc16_t*)st->fields)[i].f = val; \ } \ else { \ - st->fields32[i].f = val; \ + ((jl_fielddesc32_t*)st->fields)[i].f = val; \ } \ } From 46107d9fd2188a002021880feb3c24b0b59341f9 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 17 Sep 2015 14:20:22 -0400 Subject: [PATCH 0146/1938] Fix compiler warning about incompatible pointer types. jl_fptr_t is already a function pointer. --- src/alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index f0621a9a5e674..a098f230933aa 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -325,7 +325,7 @@ DLLEXPORT jl_function_t *jl_new_closure(jl_fptr_t fptr, jl_value_t *env, return f; } -DLLEXPORT jl_fptr_t *jl_linfo_fptr(jl_lambda_info_t *linfo) +DLLEXPORT jl_fptr_t jl_linfo_fptr(jl_lambda_info_t *linfo) { return linfo->fptr; } From 9f72db1a1a94cbb8e365e00dd92b141e956c1b74 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 17 Sep 2015 14:48:41 -0400 Subject: [PATCH 0147/1938] fix #13183, infinite recursion in compiler via static parameter The static parameter was causing a type to be wrapped in a TypeVar, hiding it from limit_tuple_depth. --- base/inference.jl | 6 +++++- test/core.jl | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/base/inference.jl b/base/inference.jl index fc75634665398..9609ad741fa1a 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -225,7 +225,8 @@ function limit_type_depth(t::ANY, d::Int, cov::Bool, vars) if d > MAX_TYPE_DEPTH R = t.name.primary else - Q = map(x->limit_type_depth(x, d+1, false, vars), P) + stillcov = cov && (t.name === Tuple.name) + Q = map(x->limit_type_depth(x, d+1, stillcov, vars), P) if !cov && any(p->contains_is(vars,p), Q) R = t.name.primary inexact = true @@ -578,6 +579,9 @@ const limit_tuple_depth_ = function (t,d::Int) # may have to recur into other stuff in the future too. return Union{map(x->limit_tuple_depth_(x,d+1), t.types)...} end + if isa(t,TypeVar) + return limit_tuple_depth_(t.ub, d) + end if !(isa(t,DataType) && t.name === Tuple.name) return t end diff --git a/test/core.jl b/test/core.jl index 2659dece308d1..6e53a57e547d4 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3375,3 +3375,7 @@ immutable EmptyIIOtherField13175 end @test EmptyIIOtherField13175(EmptyImmutable13175(), 1.0) == EmptyIIOtherField13175(EmptyImmutable13175(), 1.0) @test EmptyIIOtherField13175(EmptyImmutable13175(), 1.0) != EmptyIIOtherField13175(EmptyImmutable13175(), 2.0) + +# issue #13183 +gg13183{X}(x::X...) = 1==0 ? gg13183(x, x) : 0 +@test gg13183(5) == 0 From d4761549c1c0c0e7353d34279fd44b1fb7a8dce5 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 17 Sep 2015 15:24:37 -0400 Subject: [PATCH 0148/1938] fix compiler warning --- src/cgutils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 32612a2f0a3fc..a01bb9cd1df09 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -793,7 +793,7 @@ static bool is_tupletype_homogeneous(jl_svec_t *t) static bool deserves_sret(jl_value_t *dt, Type *T) { assert(jl_is_datatype(dt)); - return jl_datatype_size(dt) > sizeof(void*) && !T->isFloatingPointTy(); + return (size_t)jl_datatype_size(dt) > sizeof(void*) && !T->isFloatingPointTy(); } // --- generating various field accessors --- From 53fcec2cb2bdae88f3ef445c1a856f5bf4462462 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 9 Sep 2015 10:18:59 -0500 Subject: [PATCH 0149/1938] Update edit to better support spaces and args When working with Sublime as my default editor on OS X I had set my editor to: EDITOR=/Applications/Sublime\ Text.app/Contents/SharedSupport/bin/subl Since the `ispath` function did not recognize this as a path (because contained a backslash) this resulted in my editor not being recognized as "subl" and line number support being unavailable. The section of `edit` that deals with determining the user's editor has been abstracted into a new function `editor`. Test cases have been added for `editor` along with some basic tests for `shell_split`. Note that `startswith` is used to identify alternative editor names: - Sublime Text can refer to "subl" or "sublime". - Emacs can refer to "emacs" or "emacsclient". --- base/interactiveutil.jl | 51 ++++++++++++++++++++++------------------- test/replutil.jl | 40 ++++++++++++++++++++++++++++++++ test/spawn.jl | 22 ++++++++++++++++++ 3 files changed, 90 insertions(+), 23 deletions(-) diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 162b3261d12ae..74dbed0b5de15 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -2,7 +2,14 @@ # editing files -function edit(file::AbstractString, line::Integer) +doc""" + editor() + +Determines the editor to use when running functions like `edit`. Returns an Array compatible +for use within backticks. You can change the editor by setting JULIA_EDITOR, VISUAL, or +EDITOR as an environmental variable. +""" +function editor() if OS_NAME == :Windows || OS_NAME == :Darwin default_editor = "open" elseif isreadable("/etc/alternatives/editor") @@ -10,39 +17,37 @@ function edit(file::AbstractString, line::Integer) else default_editor = "emacs" end - editor = get(ENV,"JULIA_EDITOR", get(ENV,"VISUAL", get(ENV,"EDITOR", default_editor))) - if ispath(editor) - if isreadable(editor) - edpath = realpath(editor) - edname = basename(edpath) - else - error("can't find \"$editor\"") - end - else - edpath = edname = editor - end + # Note: the editor path can include spaces (if escaped) and flags. + command = shell_split(get(ENV,"JULIA_EDITOR", get(ENV,"VISUAL", get(ENV,"EDITOR", default_editor)))) + isempty(command) && error("editor is empty") + return command +end + +function edit(file::AbstractString, line::Integer) + command = editor() + name = basename(first(command)) issrc = length(file)>2 && file[end-2:end] == ".jl" if issrc f = find_source_file(file) f !== nothing && (file = f) end const no_line_msg = "Unknown editor: no line number information passed.\nThe method is defined at line $line." - if startswith(edname, "emacs") || edname == "gedit" - spawn(`$edpath +$line $file`) - elseif edname == "vi" || edname == "vim" || edname == "nvim" || edname == "mvim" || edname == "nano" - run(`$edpath +$line $file`) - elseif edname == "textmate" || edname == "mate" || edname == "kate" - spawn(`$edpath $file -l $line`) - elseif startswith(edname, "subl") || edname == "atom" - spawn(`$(shell_split(edpath)) $file:$line`) - elseif OS_NAME == :Windows && (edname == "start" || edname == "open") + if startswith(name, "emacs") || name == "gedit" + spawn(`$command +$line $file`) + elseif name == "vi" || name == "vim" || name == "nvim" || name == "mvim" || name == "nano" + run(`$command +$line $file`) + elseif name == "textmate" || name == "mate" || name == "kate" + spawn(`$command $file -l $line`) + elseif startswith(name, "subl") || name == "atom" + spawn(`$command $file:$line`) + elseif OS_NAME == :Windows && (name == "start" || name == "open") spawn(`cmd /c start /b $file`) println(no_line_msg) - elseif OS_NAME == :Darwin && (edname == "start" || edname == "open") + elseif OS_NAME == :Darwin && (name == "start" || name == "open") spawn(`open -t $file`) println(no_line_msg) else - run(`$(shell_split(edpath)) $file`) + run(`$command $file`) println(no_line_msg) end nothing diff --git a/test/replutil.jl b/test/replutil.jl index c0c8760d10d17..dc054ad9787b7 100644 --- a/test/replutil.jl +++ b/test/replutil.jl @@ -132,3 +132,43 @@ let showerror(buff, MethodError(convert, Tuple{Type, Float64})) showerror(buff, MethodError(convert, Tuple{DataType, Float64})) end + +# Issue #13032 +withenv("JULIA_EDITOR" => nothing, "VISUAL" => nothing, "EDITOR" => nothing) do + + # Make sure editor doesn't error when no ENV editor is set. + @test isa(Base.editor(), Array) + + # Invalid editor + ENV["JULIA_EDITOR"] = "" + @test_throws ErrorException Base.editor() + + # Note: The following testcases should work regardless of whether these editors are + # installed or not. + + # Editor on the path. + ENV["JULIA_EDITOR"] = "vim" + @test Base.editor() == ["vim"] + + # Absolute path to editor. + ENV["JULIA_EDITOR"] = "/usr/bin/vim" + @test Base.editor() == ["/usr/bin/vim"] + + # Editor on the path using arguments. + ENV["JULIA_EDITOR"] = "subl -w" + @test Base.editor() == ["subl", "-w"] + + # Absolute path to editor with spaces. + ENV["JULIA_EDITOR"] = "/Applications/Sublime\\ Text.app/Contents/SharedSupport/bin/subl" + @test Base.editor() == ["/Applications/Sublime Text.app/Contents/SharedSupport/bin/subl"] + + # Paths with spaces and arguments (#13032). + ENV["JULIA_EDITOR"] = "/Applications/Sublime\\ Text.app/Contents/SharedSupport/bin/subl -w" + @test Base.editor() == ["/Applications/Sublime Text.app/Contents/SharedSupport/bin/subl", "-w"] + + ENV["JULIA_EDITOR"] = "'/Applications/Sublime Text.app/Contents/SharedSupport/bin/subl' -w" + @test Base.editor() == ["/Applications/Sublime Text.app/Contents/SharedSupport/bin/subl", "-w"] + + ENV["JULIA_EDITOR"] = "\"/Applications/Sublime Text.app/Contents/SharedSupport/bin/subl\" -w" + @test Base.editor() == ["/Applications/Sublime Text.app/Contents/SharedSupport/bin/subl", "-w"] +end \ No newline at end of file diff --git a/test/spawn.jl b/test/spawn.jl index 1856df40147d1..d50658caba14f 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -271,3 +271,25 @@ let fname = tempname() @test success(pipeline(`cat $fname`, `$exename -e $code`)) rm(fname) end + +let cmd = AbstractString[] + # Ensure that quoting works + @test Base.shell_split("foo bar baz") == ["foo", "bar", "baz"] + @test Base.shell_split("foo\\ bar baz") == ["foo bar", "baz"] + @test Base.shell_split("'foo bar' baz") == ["foo bar", "baz"] + @test Base.shell_split("\"foo bar\" baz") == ["foo bar", "baz"] + + # "Over quoted" + @test Base.shell_split("'foo\\ bar' baz") == ["foo\\ bar", "baz"] + @test Base.shell_split("\"foo\\ bar\" baz") == ["foo\\ bar", "baz"] + + # Ensure that shell_split handles quoted spaces + cmd = ["/Volumes/External HD/program", "-a"] + @test Base.shell_split("/Volumes/External\\ HD/program -a") == cmd + @test Base.shell_split("'/Volumes/External HD/program' -a") == cmd + @test Base.shell_split("\"/Volumes/External HD/program\" -a") == cmd + + # Backticks should automatically quote where necessary + cmd = ["foo bar", "baz"] + @test string(`$cmd`) == "`'foo bar' baz`" +end From 81451d53f489b1e2664bfbc9927e3c8c3b696d98 Mon Sep 17 00:00:00 2001 From: Graham Inggs Date: Thu, 17 Sep 2015 22:07:02 +0200 Subject: [PATCH 0150/1938] Switch to := , move debian section --- Make.inc | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/Make.inc b/Make.inc index 02f0370b63488..b5d1d8c0a2548 100644 --- a/Make.inc +++ b/Make.inc @@ -135,7 +135,6 @@ USE_GPL_LIBS ?= 1 prefix ?= $(abspath julia-$(JULIA_COMMIT)) bindir := $(prefix)/bin libdir := $(prefix)/lib -private_libdir := $(libdir)/julia libexecdir := $(prefix)/libexec datarootdir := $(prefix)/share docdir := $(datarootdir)/doc/julia @@ -149,7 +148,6 @@ build_prefix := $(BUILDROOT)/usr build_staging := $(build_prefix)-staging build_bindir := $(build_prefix)/bin build_libdir := $(build_prefix)/lib -build_private_libdir := $(build_prefix)/lib/julia build_libexecdir := $(build_prefix)/libexec build_datarootdir := $(build_prefix)/share build_docdir := $(build_datarootdir)/doc/julia @@ -158,6 +156,16 @@ build_man1dir := $(build_mandir)/man1 build_includedir := $(build_prefix)/include build_sysconfdir := $(build_prefix)/etc +# This used for debian packaging, to conform to library layout guidelines +ifeq ($(MULTIARCH_INSTALL), 1) +MULTIARCH := $(shell gcc -print-multiarch) +libdir := $(prefix)/lib/$(MULTIARCH) +build_libdir := $(build_prefix)/lib/$(MULTIARCH) +endif + +# Private library directories +private_libdir := $(libdir)/julia +build_private_libdir := $(build_libdir)/julia # Calculate relative paths to libdir, private_libdir, datarootdir, and sysconfdir build_libdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(build_bindir) $(build_libdir)) @@ -171,15 +179,6 @@ sysconfdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(sysc INSTALL_F := $(JULIAHOME)/contrib/install.sh 644 INSTALL_M := $(JULIAHOME)/contrib/install.sh 755 -# This used for debian packaging, to conform to library layout guidelines -ifeq ($(MULTIARCH_INSTALL), 1) -MULTIARCH := $(shell gcc -print-multiarch) -libdir = $(prefix)/lib/$(MULTIARCH) -private_libdir = $(prefix)/lib/$(MULTIARCH)/julia -build_libdir = $(build_prefix)/lib/$(MULTIARCH) -build_private_libdir = $(build_prefix)/lib/$(MULTIARCH)/julia -endif - # LLVM Options LLVMROOT := $(build_prefix) LLVM_ASSERTIONS := 0 From 1fa28584c93ec3a2b554f1fcc167e23b152c6f6e Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 17 Sep 2015 15:12:38 -0400 Subject: [PATCH 0151/1938] improve type-inference/codegen for setindex! closes #10954 --- base/array.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/array.jl b/base/array.jl index 26949644727a1..733eb00d5a25c 100644 --- a/base/array.jl +++ b/base/array.jl @@ -310,10 +310,10 @@ function getindex{T<:Real}(A::Array, I::Range{T}) end ## Indexing: setindex! ## -setindex!{T}(A::Array{T}, x, i1::Real) = arrayset(A, convert(T,x), to_index(i1)) -setindex!{T}(A::Array{T}, x, i1::Real, i2::Real, I::Real...) = arrayset(A, convert(T,x), to_index(i1), to_index(i2), to_indexes(I...)...) +setindex!{T}(A::Array{T}, x, i1::Real) = arrayset(A, convert(T,x)::T, to_index(i1)) +setindex!{T}(A::Array{T}, x, i1::Real, i2::Real, I::Real...) = arrayset(A, convert(T,x)::T, to_index(i1), to_index(i2), to_indexes(I...)...) -unsafe_setindex!{T}(A::Array{T}, x, i1::Real, I::Real...) = @inbounds return arrayset(A, convert(T,x), to_index(i1), to_indexes(I...)...) +unsafe_setindex!{T}(A::Array{T}, x, i1::Real, I::Real...) = @inbounds return arrayset(A, convert(T,x)::T, to_index(i1), to_indexes(I...)...) # These are redundant with the abstract fallbacks but needed for bootstrap function setindex!(A::Array, x, I::AbstractVector{Int}) From f8c8135efa440e48591d530cc2be1ee6a74ca7de Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 17 Sep 2015 15:18:00 -0400 Subject: [PATCH 0152/1938] add test for #8932 --- test/core.jl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/core.jl b/test/core.jl index 6e53a57e547d4..c25fd8b99c77b 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3379,3 +3379,14 @@ end # issue #13183 gg13183{X}(x::X...) = 1==0 ? gg13183(x, x) : 0 @test gg13183(5) == 0 + +# issue 8932 (llvm return type legalizer error) +immutable Vec3_8932 + x::Float32 + y::Float32 + z::Float32 +end +f8932(a::Vec3_8932, b::Vec3_8932) = Vec3_8932(a.x % b.x, a.y % b.y, a.z % b.z) +a8932 = Vec3_8932(1,1,1) +b8932 = Vec3_8932(2,2,2) +@test f8932(a8932, b8932) == Vec3_8932(1.0, 1.0, 1.0) From 7427de625d7cb1206241ce06df7bb0ec1aa186ff Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 17 Sep 2015 17:32:51 -0400 Subject: [PATCH 0153/1938] fix an incorrect assertion when interrupting output with ^C further helps e.g. #12883 --- base/stream.jl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/base/stream.jl b/base/stream.jl index 6ecf499db9dbc..24485279aeea3 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -1000,12 +1000,13 @@ write(s::LibuvStream, p::Ptr, n::Integer) = buffer_or_write(s, p, n) function uv_writecb_task(req::Ptr{Void}, status::Cint) d = uv_req_data(req) - @assert d != C_NULL - if status < 0 - err = UVError("write",status) - schedule(unsafe_pointer_to_objref(d)::Task,err,error=true) - else - schedule(unsafe_pointer_to_objref(d)::Task) + if d != C_NULL + if status < 0 + err = UVError("write",status) + schedule(unsafe_pointer_to_objref(d)::Task,err,error=true) + else + schedule(unsafe_pointer_to_objref(d)::Task) + end end Libc.free(req) nothing From 06e6b3738264f6b458b8491bb84295d813ec2cec Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 17 Sep 2015 17:35:40 -0400 Subject: [PATCH 0154/1938] fix missing `io` argument to `println` in umfpack this should fix #12837 --- base/sparse/umfpack.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/sparse/umfpack.jl b/base/sparse/umfpack.jl index b36cfced934e1..6310fb8f1d0a5 100644 --- a/base/sparse/umfpack.jl +++ b/base/sparse/umfpack.jl @@ -117,7 +117,7 @@ lufact(A::SparseMatrixCSC) = lufact(float(A)) function show(io::IO, f::UmfpackLU) println(io, "UMFPACK LU Factorization of a $(f.m)-by-$(f.n) sparse matrix") - f.numeric != C_NULL && println(f.numeric) + f.numeric != C_NULL && println(io, f.numeric) end ## Wrappers for UMFPACK functions From 64d0cc9387af20ee956a7f67ab2422338e0f1042 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 17 Sep 2015 18:04:05 -0400 Subject: [PATCH 0155/1938] add deprecation for Union() --- base/deprecated.jl | 2 ++ base/essentials.jl | 2 -- base/require.jl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 5a78ef2b120cf..cdcb77fa60522 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -826,6 +826,8 @@ end @deprecate cartesianmap(f, dims) for idx in CartesianRange(dims); f(idx.I...); end +@deprecate Union(args...) Union{args...} + # 0.5 deprecations # 12839 diff --git a/base/essentials.jl b/base/essentials.jl index 582f66bb2584a..036a2b2bbdd0e 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -6,8 +6,6 @@ typealias Callable Union{Function,DataType} const Bottom = Union{} -call(::Type{Union}, args...) = Union{args...} - # The real @inline macro is not available until after array.jl, so this # internal macro splices the meta Expr directly into the function body. macro _inline_meta() diff --git a/base/require.jl b/base/require.jl index 49b4d0d827857..d8dcff0f9e9cb 100644 --- a/base/require.jl +++ b/base/require.jl @@ -67,7 +67,7 @@ end # remote/parallel load -function source_path(default::Union(AbstractString,Void)="") +function source_path(default::Union{AbstractString,Void}="") t = current_task() while true s = t.storage From 7bcc9471ef1988fabf7b7cb66aef9fd57ac7af1c Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Fri, 18 Sep 2015 05:40:21 -0500 Subject: [PATCH 0156/1938] Fix algorithms that assume linear indexing --- base/abstractarray.jl | 21 ++++++++++++++++++--- base/array.jl | 28 ++++++++++++++++------------ base/multidimensional.jl | 4 +++- base/reduce.jl | 16 +++++++++------- base/sparse/sparsematrix.jl | 2 +- 5 files changed, 47 insertions(+), 24 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index f202fe6e7067c..d460af2ed1632 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -300,7 +300,10 @@ end ## copy between abstract arrays - generally more efficient ## since a single index variable can be used. -function copy!(dest::AbstractArray, src::AbstractArray) +copy!(dest::AbstractArray, src::AbstractArray) = + copy!(linearindexing(dest), dest, linearindexing(src), src) + +function copy!(::LinearIndexing, dest::AbstractArray, ::LinearIndexing, src::AbstractArray) n = length(src) n > length(dest) && throw(BoundsError(dest, n)) @inbounds for i = 1:n @@ -309,6 +312,16 @@ function copy!(dest::AbstractArray, src::AbstractArray) return dest end +function copy!(::LinearIndexing, dest::AbstractArray, ::LinearSlow, src::AbstractArray) + n = length(src) + n > length(dest) && throw(BoundsError(dest, n)) + i = 0 + @inbounds for a in src + dest[i+=1] = a + end + return dest +end + function copy!(dest::AbstractArray, doffs::Integer, src::AbstractArray) copy!(dest, doffs, src, 1, length(src)) end @@ -395,6 +408,8 @@ start(A::AbstractArray) = (@_inline_meta(); itr = eachindex(A); (itr, start(itr) next(A::AbstractArray,i) = (@_inline_meta(); (idx, s) = next(i[1], i[2]); (A[idx], (i[1], s))) done(A::AbstractArray,i) = done(i[1], i[2]) +iterstate(i) = i + # eachindex iterates over all indices. LinearSlow definitions are later. eachindex(A::AbstractArray) = (@_inline_meta(); eachindex(linearindexing(A), A)) eachindex(::LinearFast, A::AbstractArray) = 1:length(A) @@ -1013,7 +1028,7 @@ function isequal(A::AbstractArray, B::AbstractArray) if isa(A,Range) != isa(B,Range) return false end - for i in eachindex(A) + for i in eachindex(A,B) if !isequal(A[i], B[i]) return false end @@ -1037,7 +1052,7 @@ function (==)(A::AbstractArray, B::AbstractArray) if isa(A,Range) != isa(B,Range) return false end - for i in eachindex(A) + for i in eachindex(A,B) if !(A[i]==B[i]) return false end diff --git a/base/array.jl b/base/array.jl index 733eb00d5a25c..f1f8a3958acc0 100644 --- a/base/array.jl +++ b/base/array.jl @@ -827,32 +827,36 @@ function findmax(a) if isempty(a) throw(ArgumentError("collection must be non-empty")) end - m = a[1] - mi = 1 - for i=2:length(a) - ai = a[i] + i = start(a) + mi = i + m, i = next(a, i) + while !done(a, i) + iold = i + ai, i = next(a, i) if ai > m || m!=m m = ai - mi = i + mi = iold end end - return (m, mi) + return (m, iterstate(mi)) end function findmin(a) if isempty(a) throw(ArgumentError("collection must be non-empty")) end - m = a[1] - mi = 1 - for i=2:length(a) - ai = a[i] + i = start(a) + mi = i + m, i = next(a, i) + while !done(a, i) + iold = i + ai, i = next(a, i) if ai < m || m!=m m = ai - mi = i + mi = iold end end - return (m, mi) + return (m, iterstate(mi)) end indmax(a) = findmax(a)[2] diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 9b127007a0772..073204ae682de 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -3,7 +3,7 @@ ### Multidimensional iterators module IteratorsMD -import Base: eltype, length, start, done, next, last, getindex, setindex!, linearindexing, min, max, eachindex, ndims +import Base: eltype, length, start, done, next, last, getindex, setindex!, linearindexing, min, max, eachindex, ndims, iterstate importall ..Base.Operators import Base: simd_outer_range, simd_inner_length, simd_index, @generated import Base: @nref, @ncall, @nif, @nexprs, LinearFast, LinearSlow, to_index @@ -59,6 +59,8 @@ immutable CartesianRange{I<:CartesianIndex} stop::I end +iterstate{CR<:CartesianRange,CI<:CartesianIndex}(i::Tuple{CR,CI}) = i[2] + @generated function CartesianRange{N}(I::CartesianIndex{N}) startargs = fill(1, N) :(CartesianRange($I($(startargs...)), I)) diff --git a/base/reduce.jl b/base/reduce.jl index ce34985e74d46..09099a954e670 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -131,7 +131,9 @@ mr_empty(::Abs2Fun, op::MaxFun, T) = abs2(zero(T)::T) mr_empty(f, op::AndFun, T) = true mr_empty(f, op::OrFun, T) = false -function _mapreduce{T}(f, op, A::AbstractArray{T}) +_mapreduce(f, op, A::AbstractArray) = _mapreduce(f, op, linearindexing(A), A) + +function _mapreduce{T}(f, op, ::LinearFast, A::AbstractArray{T}) n = Int(length(A)) if n == 0 return mr_empty(f, op, T) @@ -152,7 +154,9 @@ function _mapreduce{T}(f, op, A::AbstractArray{T}) end end -mapreduce(f, op, A::AbstractArray) = _mapreduce(f, op, A) +_mapreduce{T}(f, op, ::LinearSlow, A::AbstractArray{T}) = mapfoldl(f, op, A) + +mapreduce(f, op, A::AbstractArray) = _mapreduce(f, op, linearindexing(A), A) mapreduce(f, op, a::Number) = f(a) mapreduce(f, op::Function, A::AbstractArray) = mapreduce(f, specialized_binary(op), A) @@ -395,12 +399,10 @@ function count(pred, itr) return n end -function count(pred, a::AbstractArray) +function count(pred, A::AbstractArray) n = 0 - for i = 1:length(a) - @inbounds if pred(a[i]) - n += 1 - end + @inbounds for a in A + pred(a) && (n += 1) end return n end diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 592e00d7aaea9..e63d3e1bd5149 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1017,7 +1017,7 @@ function _mapreducezeros(f, op, T::Type, nzeros::Int, v0) v end -function Base._mapreduce{T}(f, op, A::SparseMatrixCSC{T}) +function Base._mapreduce{T}(f, op, ::Base.LinearSlow, A::SparseMatrixCSC{T}) z = nnz(A) n = length(A) if z == 0 From 108d72457f5c68ba6c6bab311690250c525f2f7b Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Fri, 18 Sep 2015 09:08:32 -0500 Subject: [PATCH 0157/1938] iterstate returns an integer (findmin/findmax) This avoids breaking 0.4 APIs. --- base/multidimensional.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 073204ae682de..56f882338ec36 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -59,7 +59,7 @@ immutable CartesianRange{I<:CartesianIndex} stop::I end -iterstate{CR<:CartesianRange,CI<:CartesianIndex}(i::Tuple{CR,CI}) = i[2] +iterstate{CR<:CartesianRange,CI<:CartesianIndex}(i::Tuple{CR,CI}) = Base._sub2ind(i[1].stop.I, i[2].I) @generated function CartesianRange{N}(I::CartesianIndex{N}) startargs = fill(1, N) From 761e20198ed7bfd39c32ee4ed293e852ebf6e686 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Fri, 18 Sep 2015 09:09:23 -0500 Subject: [PATCH 0158/1938] Add tests --- test/arrayops.jl | 75 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/test/arrayops.jl b/test/arrayops.jl index 132e3275cdc56..20e239ab380bc 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1265,3 +1265,78 @@ B = 1.5:5.5 #slice dim error A = zeros(5,5) @test_throws ArgumentError slicedim(A,0,1) + +### +### LinearSlow workout +### +immutable LinSlowMatrix{T} <: DenseArray{T,2} + data::Matrix{T} +end + +# This is the default, but just to be sure +Base.linearindexing{A<:LinSlowMatrix}(::Type{A}) = Base.LinearSlow() + +Base.size(A::LinSlowMatrix) = size(A.data) + +Base.getindex(A::LinSlowMatrix, i::Integer) = error("Not defined") +Base.getindex(A::LinSlowMatrix, i::Integer, j::Integer) = A.data[i,j] + +Base.setindex!(A::LinSlowMatrix, v, i::Integer) = error("Not defined") +Base.setindex!(A::LinSlowMatrix, v, i::Integer, j::Integer) = A.data[i,j] = v + +A = rand(3,5) +B = LinSlowMatrix(A) + +@test A == B +@test B == A +@test isequal(A, B) +@test isequal(B, A) + +for (a,b) in zip(A, B) + @test a == b +end + +C = copy(B) +@test A == C +@test B == C + +@test vec(A) == vec(B) +@test minimum(A) == minimum(B) +@test maximum(A) == maximum(B) + +a, ai = findmin(A) +b, bi = findmin(B) +@test a == b +@test ai == bi + +a, ai = findmax(A) +b, bi = findmax(B) +@test a == b +@test ai == bi + +fill!(B, 2) +@test all(x->x==2, B) + +i,j = findn(B) +iall = (1:size(A,1)).*ones(Int,size(A,2))' +jall = ones(Int,size(A,1)).*(1:size(A,2))' +@test vec(i) == vec(iall) +@test vec(j) == vec(jall) + +copy!(B, A) + +@test cat(1, A, B) == cat(1, A, A) +@test cat(2, A, B) == cat(2, A, A) + +@test cumsum(A, 1) == cumsum(B, 1) +@test cumsum(A, 2) == cumsum(B, 2) + +@test mapslices(v->sort(v), A, 1) == mapslices(v->sort(v), B, 1) +@test mapslices(v->sort(v), A, 2) == mapslices(v->sort(v), B, 2) + +@test flipdim(A, 1) == flipdim(B, 1) +@test flipdim(A, 2) == flipdim(B, 2) + +@test A + 1 == B + 1 +@test 2*A == 2*B +@test A/3 == B/3 From 9d57a1eec385f3f972a68a3f56892f53ee44957d Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Fri, 18 Sep 2015 12:08:09 -0400 Subject: [PATCH 0159/1938] adds short paragraph describing the benefits of defining similar() for AbstractArray subtypes --- doc/manual/interfaces.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/interfaces.rst b/doc/manual/interfaces.rst index 51982619d24ce..233dd69baa6cf 100644 --- a/doc/manual/interfaces.rst +++ b/doc/manual/interfaces.rst @@ -173,6 +173,8 @@ A key part in defining an ``AbstractArray`` subtype is :func:`Base.linearindexin This distinction determines which scalar indexing methods the type must define. ``LinearFast()`` arrays are simple: just define :func:`getindex(A::ArrayType, i::Int) `. When the array is subsequently indexed with a multidimensional set of indices, the fallback :func:`getindex(A::AbstractArray, I...)` efficiently converts the indices into one linear index and then calls the above method. ``LinearSlow()`` arrays, on the other hand, require methods to be defined for each supported dimensionality with ``ndims(A)`` ``Int`` indices. For example, the builtin ``SparseMatrix`` type only supports two dimensions, so it just defines :func:`getindex(A::SparseMatrix, i::Int, j::Int)`. The same holds for :func:`setindex!`. +The result of indexing an AbstractArray can itself be an array (for instance when indexing by a ``Range``). The AbstractArray fallback methods use :func:`similar` to create an ``Array`` of the appropriate size and element type, which is filled in using the basic indexing method described above. When implementing an array wrapper you often want the result to be wrapped as well, which can be accomplished by defining ``Base.similar{T}(A::YourArrayType, ::Type{T}, dims::Dims)`` to create the appropriate wrapped array. + Returning to the sequence of squares from above, we could instead define it as a subtype of an ``AbstractArray{Int, 1}``: .. doctest:: From 47820923403344f4a10f2acf604993324455a69a Mon Sep 17 00:00:00 2001 From: Andy Hayden Date: Fri, 18 Sep 2015 09:54:48 -0700 Subject: [PATCH 0160/1938] Remove references to GPL methods in markdown/docs --- base/precompile.jl | 8 +++----- test/markdown.jl | 14 +++++--------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/base/precompile.jl b/base/precompile.jl index bf3aed56cf6a7..72722cb42907a 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -501,8 +501,6 @@ precompile(Base.set_valid_processes, (Array{Int, 1}, )) # Speed up repl help -if Base.USE_GPL_LIBS - sprint(Markdown.term, @doc fft) - sprint(Docs.repl_search, "fft") - sprint(Docs.repl_corrections, "fft") -end +sprint(Markdown.term, @doc mean) +sprint(Docs.repl_search, "mean") +sprint(Docs.repl_corrections, "meen") diff --git a/test/markdown.jl b/test/markdown.jl index b02755dbc86b5..9129962c1b7c6 100644 --- a/test/markdown.jl +++ b/test/markdown.jl @@ -213,24 +213,20 @@ end ref(x) = Reference(x) -if Base.USE_GPL_LIBS - -ref(fft) +ref(mean) writemime(io::IO, m::MIME"text/plain", r::Reference) = print(io, "$(r.ref) (see Julia docs)") -fft_ref = md"Behaves like $(ref(fft))" -@test plain(fft_ref) == "Behaves like fft (see Julia docs)\n" -@test html(fft_ref) == "

Behaves like fft (see Julia docs)

\n" +mean_ref = md"Behaves like $(ref(mean))" +@test plain(mean_ref) == "Behaves like mean (see Julia docs)\n" +@test html(mean_ref) == "

Behaves like mean (see Julia docs)

\n" writemime(io::IO, m::MIME"text/html", r::Reference) = Markdown.withtag(io, :a, :href=>"test") do Markdown.htmlesc(io, Markdown.plaininline(r)) end -@test html(fft_ref) == "

Behaves like fft (see Julia docs)

\n" - -end # USE_GPL_LIBS +@test html(mean_ref) == "

Behaves like mean (see Julia docs)

\n" @test md""" ````julia From 66da8394d7e09f92f94c2a958c68928134b80353 Mon Sep 17 00:00:00 2001 From: nkottary Date: Mon, 14 Sep 2015 15:23:28 +0530 Subject: [PATCH 0161/1938] Grouped docstrings for FFTW in an if USE_GPL_LIBS block --- base/docs/helpdb.jl | 428 ++++++++++++++++++++++---------------------- 1 file changed, 216 insertions(+), 212 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 7f0bcabe660bf..69e4f5cba90ff 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -559,6 +559,8 @@ Test.with_handler # Base.FFTW +if USE_GPL_LIBS + doc""" ```rst .. r2r(A, kind [, dims]) @@ -617,6 +619,220 @@ correspond to :func:`r2r` and :func:`r2r!`, respectively. """ FFTW.plan_r2r +doc""" +```rst +.. plan_bfft!(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Same as :func:`plan_bfft`, but operates in-place on ``A``. +``` +""" +plan_bfft! + +doc""" +```rst +.. plan_irfft(A, d [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Pre-plan an optimized inverse real-input FFT, similar to :func:`plan_rfft` +except for :func:`irfft` and :func:`brfft`, respectively. The first +three arguments have the same meaning as for :func:`irfft`. +``` +""" +plan_irfft + +doc""" +```rst +.. plan_rfft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Pre-plan an optimized real-input FFT, similar to :func:`plan_fft` +except for :func:`rfft` instead of :func:`fft`. The first two +arguments, and the size of the transformed result, are the same as +for :func:`rfft`. +``` +""" +plan_rfft + +doc""" +```rst +.. plan_fft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Pre-plan an optimized FFT along given dimensions (``dims``) of arrays +matching the shape and type of ``A``. (The first two arguments have +the same meaning as for :func:`fft`.) Returns an object ``P`` which +represents the linear operator computed by the FFT, and which contains +all of the information needed to compute ``fft(A, dims)`` quickly. + +To apply ``P`` to an array ``A``, use ``P * A``; in general, the +syntax for applying plans is much like that of matrices. (A plan +can only be applied to arrays of the same size as the ``A`` for +which the plan was created.) You can also apply a plan with a +preallocated output array ``Â`` by calling ``A_mul_B!(Â, plan, +A)``. You can compute the inverse-transform plan by ``inv(P)`` and +apply the inverse plan with ``P \ Â`` (the inverse plan is cached +and reused for subsequent calls to ``inv`` or ``\``), and apply the +inverse plan to a pre-allocated output array ``A`` with +``A_ldiv_B!(A, P, Â)``. + +The ``flags`` argument is a bitwise-or of FFTW planner flags, defaulting +to ``FFTW.ESTIMATE``. e.g. passing ``FFTW.MEASURE`` or ``FFTW.PATIENT`` +will instead spend several seconds (or more) benchmarking different +possible FFT algorithms and picking the fastest one; see the FFTW manual +for more information on planner flags. The optional ``timelimit`` argument +specifies a rough upper bound on the allowed planning time, in seconds. +Passing ``FFTW.MEASURE`` or ``FFTW.PATIENT`` may cause the input array ``A`` +to be overwritten with zeros during plan creation. + +:func:`plan_fft!` is the same as :func:`plan_fft` but creates a plan +that operates in-place on its argument (which must be an array of +complex floating-point numbers). :func:`plan_ifft` and so on +are similar but produce plans that perform the equivalent of +the inverse transforms :func:`ifft` and so on. +``` +""" +plan_fft + +doc""" +```rst +.. plan_bfft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Same as :func:`plan_fft`, but produces a plan that performs an unnormalized +backwards transform :func:`bfft`. +``` +""" +plan_bfft + +doc""" +```rst +.. plan_fft!(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Same as :func:`plan_fft`, but operates in-place on ``A``. +``` +""" +plan_fft! + +doc""" +```rst +.. plan_ifft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Same as :func:`plan_fft`, but produces a plan that performs inverse transforms +:func:`ifft`. +``` +""" +plan_ifft + +doc""" +```rst +.. plan_brfft(A, d [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Pre-plan an optimized real-input unnormalized transform, similar to +:func:`plan_rfft` except for :func:`brfft` instead of :func:`rfft`. +The first two arguments and the size of the transformed result, are +the same as for :func:`brfft`. +``` +""" +plan_brfft + +doc""" +```rst +.. plan_ifft!(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Same as :func:`plan_ifft`, but operates in-place on ``A``. +``` +""" +plan_ifft! + +doc""" +```rst +.. dct(A [, dims]) + +Performs a multidimensional type-II discrete cosine transform (DCT) +of the array ``A``, using the unitary normalization of the DCT. +The optional ``dims`` argument specifies an iterable subset of +dimensions (e.g. an integer, range, tuple, or array) to transform +along. Most efficient if the size of ``A`` along the transformed +dimensions is a product of small primes; see :func:`nextprod`. See +also :func:`plan_dct` for even greater efficiency. +``` +""" +dct + +doc""" +```rst +.. idct(A [, dims]) + +Computes the multidimensional inverse discrete cosine transform (DCT) +of the array ``A`` (technically, a type-III DCT with the unitary +normalization). +The optional ``dims`` argument specifies an iterable subset of +dimensions (e.g. an integer, range, tuple, or array) to transform +along. Most efficient if the size of ``A`` along the transformed +dimensions is a product of small primes; see :func:`nextprod`. See +also :func:`plan_idct` for even greater efficiency. +``` +""" +idct + +doc""" +```rst +.. dct!(A [, dims]) + +Same as :func:`dct!`, except that it operates in-place +on ``A``, which must be an array of real or complex floating-point +values. +``` +""" +dct! + +doc""" +```rst +.. idct!(A [, dims]) + +Same as :func:`idct!`, but operates in-place on ``A``. +``` +""" +idct! + +doc""" +```rst +.. plan_dct!(A [, dims [, flags [, timelimit]]]) + +Same as :func:`plan_dct`, but operates in-place on ``A``. +``` +""" +plan_dct! + +doc""" +```rst +.. plan_idct(A [, dims [, flags [, timelimit]]]) + +Pre-plan an optimized inverse discrete cosine transform (DCT), similar to +:func:`plan_fft` except producing a function that computes :func:`idct`. +The first two arguments have the same meaning as for :func:`idct`. +``` +""" +plan_idct + +doc""" +```rst +.. plan_dct(A [, dims [, flags [, timelimit]]]) + +Pre-plan an optimized discrete cosine transform (DCT), similar to +:func:`plan_fft` except producing a function that computes :func:`dct`. +The first two arguments have the same meaning as for :func:`dct`. +``` +""" +plan_dct + +doc""" +```rst +.. plan_idct!(A [, dims [, flags [, timelimit]]]) + +Same as :func:`plan_idct`, but operates in-place on ``A``. +``` +""" +plan_idct! + +end + # Base.Profile doc""" @@ -963,15 +1179,6 @@ Compute a type that contains the intersection of `T` and `S`. Usually this will """ typeintersect -doc""" -```rst -.. plan_bfft!(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Same as :func:`plan_bfft`, but operates in-place on ``A``. -``` -""" -plan_bfft! - doc""" pointer(array [, index]) @@ -995,17 +1202,6 @@ Test whether a floating point number is not a number (NaN) """ isnan -doc""" -```rst -.. plan_irfft(A, d [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Pre-plan an optimized inverse real-input FFT, similar to :func:`plan_rfft` -except for :func:`irfft` and :func:`brfft`, respectively. The first -three arguments have the same meaning as for :func:`irfft`. -``` -""" -plan_irfft - doc""" ```rst .. println(x) @@ -3796,21 +3992,6 @@ Perform `wait(remotecall(...))` in one message. """ remotecall_wait -doc""" -```rst -.. dct(A [, dims]) - -Performs a multidimensional type-II discrete cosine transform (DCT) -of the array ``A``, using the unitary normalization of the DCT. -The optional ``dims`` argument specifies an iterable subset of -dimensions (e.g. an integer, range, tuple, or array) to transform -along. Most efficient if the size of ``A`` along the transformed -dimensions is a product of small primes; see :func:`nextprod`. See -also :func:`plan_dct` for even greater efficiency. -``` -""" -dct - doc""" ```rst .. append!(collection, collection2) -> collection. @@ -5408,22 +5589,6 @@ Like uperm but gets the permissions of the group owning the file """ gperm -doc""" -```rst -.. idct(A [, dims]) - -Computes the multidimensional inverse discrete cosine transform (DCT) -of the array ``A`` (technically, a type-III DCT with the unitary -normalization). -The optional ``dims`` argument specifies an iterable subset of -dimensions (e.g. an integer, range, tuple, or array) to transform -along. Most efficient if the size of ``A`` along the transformed -dimensions is a product of small primes; see :func:`nextprod`. See -also :func:`plan_idct` for even greater efficiency. -``` -""" -idct - doc""" nb_available(stream) @@ -5988,17 +6153,6 @@ Tests whether a character is alphanumeric, or whether this is true for all eleme """ isalnum -doc""" -```rst -.. dct!(A [, dims]) - -Same as :func:`dct!`, except that it operates in-place -on ``A``, which must be an array of real or complex floating-point -values. -``` -""" -dct! - doc""" @sprintf("%Fmt", args...) @@ -6478,15 +6632,6 @@ Create a tuple of length `n`, computing each element as `f(i)`, where `i` is the """ ntuple -doc""" -```rst -.. idct!(A [, dims]) - -Same as :func:`idct!`, but operates in-place on ``A``. -``` -""" -idct! - doc""" Ac_rdiv_Bc(A, B) @@ -6905,15 +7050,6 @@ Create an IOBuffer, which may optionally operate on a pre-existing array. If the """ IOBuffer(data=?) -doc""" -```rst -.. plan_dct!(A [, dims [, flags [, timelimit]]]) - -Same as :func:`plan_dct`, but operates in-place on ``A``. -``` -""" -plan_dct! - doc""" findmax(itr) -> (x, index) @@ -7095,18 +7231,6 @@ Read a series of values of the given type from a stream, in canonical binary rep """ read(stream, t, dims) -doc""" -```rst -.. plan_rfft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Pre-plan an optimized real-input FFT, similar to :func:`plan_fft` -except for :func:`rfft` instead of :func:`fft`. The first two -arguments, and the size of the transformed result, are the same as -for :func:`rfft`. -``` -""" -plan_rfft - doc""" @timev @@ -8747,45 +8871,6 @@ A type assertion failure, or calling an intrinsic function with an incorrect arg """ TypeError -doc""" -```rst -.. plan_fft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Pre-plan an optimized FFT along given dimensions (``dims``) of arrays -matching the shape and type of ``A``. (The first two arguments have -the same meaning as for :func:`fft`.) Returns an object ``P`` which -represents the linear operator computed by the FFT, and which contains -all of the information needed to compute ``fft(A, dims)`` quickly. - -To apply ``P`` to an array ``A``, use ``P * A``; in general, the -syntax for applying plans is much like that of matrices. (A plan -can only be applied to arrays of the same size as the ``A`` for -which the plan was created.) You can also apply a plan with a -preallocated output array ``Â`` by calling ``A_mul_B!(Â, plan, -A)``. You can compute the inverse-transform plan by ``inv(P)`` and -apply the inverse plan with ``P \ Â`` (the inverse plan is cached -and reused for subsequent calls to ``inv`` or ``\``), and apply the -inverse plan to a pre-allocated output array ``A`` with -``A_ldiv_B!(A, P, Â)``. - -The ``flags`` argument is a bitwise-or of FFTW planner flags, defaulting -to ``FFTW.ESTIMATE``. e.g. passing ``FFTW.MEASURE`` or ``FFTW.PATIENT`` -will instead spend several seconds (or more) benchmarking different -possible FFT algorithms and picking the fastest one; see the FFTW manual -for more information on planner flags. The optional ``timelimit`` argument -specifies a rough upper bound on the allowed planning time, in seconds. -Passing ``FFTW.MEASURE`` or ``FFTW.PATIENT`` may cause the input array ``A`` -to be overwritten with zeros during plan creation. - -:func:`plan_fft!` is the same as :func:`plan_fft` but creates a plan -that operates in-place on its argument (which must be an array of -complex floating-point numbers). :func:`plan_ifft` and so on -are similar but produce plans that perform the equivalent of -the inverse transforms :func:`ifft` and so on. -``` -""" -plan_fft - doc""" A_rdiv_Bt(A, B) @@ -9012,16 +9097,6 @@ Get the vector of processes that have mapped the shared array """ procs(::SharedArray) -doc""" -```rst -.. plan_bfft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Same as :func:`plan_fft`, but produces a plan that performs an unnormalized -backwards transform :func:`bfft`. -``` -""" -plan_bfft - doc""" mod(x, y) @@ -9998,17 +10073,6 @@ Create an ASCII string from the address of a C (0-terminated) string encoded in """ ascii(::Ptr{UInt8},?) -doc""" -```rst -.. plan_idct(A [, dims [, flags [, timelimit]]]) - -Pre-plan an optimized inverse discrete cosine transform (DCT), similar to -:func:`plan_fft` except producing a function that computes :func:`idct`. -The first two arguments have the same meaning as for :func:`idct`. -``` -""" -plan_idct - doc""" maxabs(itr) @@ -10116,15 +10180,6 @@ Create a `RandomDevice` RNG object. Two such objects will always generate differ """ RandomDevice -doc""" -```rst -.. plan_fft!(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Same as :func:`plan_fft`, but operates in-place on ``A``. -``` -""" -plan_fft! - doc""" fma(x, y, z) @@ -10758,16 +10813,6 @@ Determine whether predicate `p` returns `true` for any elements of `itr`. """ any(p,itr) -doc""" -```rst -.. plan_ifft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Same as :func:`plan_fft`, but produces a plan that performs inverse transforms -:func:`ifft`. -``` -""" -plan_ifft - doc""" cosc(x) @@ -11038,18 +11083,6 @@ k]``.) """ eigfact(A,B) -doc""" -```rst -.. plan_brfft(A, d [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Pre-plan an optimized real-input unnormalized transform, similar to -:func:`plan_rfft` except for :func:`brfft` instead of :func:`rfft`. -The first two arguments and the size of the transformed result, are -the same as for :func:`brfft`. -``` -""" -plan_brfft - doc""" rowvals(A) @@ -11283,17 +11316,6 @@ Return `x` if `lo <= x <= hi`. If `x < lo`, return `lo`. If `x > hi`, return `hi """ clamp -doc""" -```rst -.. plan_dct(A [, dims [, flags [, timelimit]]]) - -Pre-plan an optimized discrete cosine transform (DCT), similar to -:func:`plan_fft` except producing a function that computes :func:`dct`. -The first two arguments have the same meaning as for :func:`dct`. -``` -""" -plan_dct - doc""" cscd(x) @@ -11589,15 +11611,6 @@ Return a copy of `collection`, removing elements for which `function` is `false` """ filter -doc""" -```rst -.. plan_idct!(A [, dims [, flags [, timelimit]]]) - -Same as :func:`plan_idct`, but operates in-place on ``A``. -``` -""" -plan_idct! - doc""" ```rst .. randperm([rng,] n) @@ -11615,15 +11628,6 @@ Seek a stream to its end. """ seekend -doc""" -```rst -.. plan_ifft!(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Same as :func:`plan_ifft`, but operates in-place on ``A``. -``` -""" -plan_ifft! - doc""" DivideError() From fb15f5a9697b6aa91554744d45a21e4f7215e87d Mon Sep 17 00:00:00 2001 From: nkottary Date: Mon, 14 Sep 2015 16:43:06 +0530 Subject: [PATCH 0162/1938] Moved fft docstrings from helpdb.jl to appropriate files in base/fft/ --- base/docs/helpdb.jl | 155 -------------------------------------------- base/fft/FFTW.jl | 58 +++++++++++++++++ base/fft/dct.jl | 91 ++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 155 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 69e4f5cba90ff..49f590f590730 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -557,68 +557,6 @@ Run the function `f` using the `handler` as the handler. """ Test.with_handler -# Base.FFTW - -if USE_GPL_LIBS - -doc""" -```rst -.. r2r(A, kind [, dims]) - -Performs a multidimensional real-input/real-output (r2r) transform -of type ``kind`` of the array ``A``, as defined in the FFTW manual. -``kind`` specifies either a discrete cosine transform of various types -(``FFTW.REDFT00``, ``FFTW.REDFT01``, ``FFTW.REDFT10``, or -``FFTW.REDFT11``), a discrete sine transform of various types -(``FFTW.RODFT00``, ``FFTW.RODFT01``, ``FFTW.RODFT10``, or -``FFTW.RODFT11``), a real-input DFT with halfcomplex-format output -(``FFTW.R2HC`` and its inverse ``FFTW.HC2R``), or a discrete -Hartley transform (``FFTW.DHT``). The ``kind`` argument may be -an array or tuple in order to specify different transform types -along the different dimensions of ``A``; ``kind[end]`` is used -for any unspecified dimensions. See the FFTW manual for precise -definitions of these transform types, at http://www.fftw.org/doc. - -The optional ``dims`` argument specifies an iterable subset of -dimensions (e.g. an integer, range, tuple, or array) to transform -along. ``kind[i]`` is then the transform type for ``dims[i]``, -with ``kind[end]`` being used for ``i > length(kind)``. - -See also :func:`plan_r2r` to pre-plan optimized r2r transforms. -``` -""" -FFTW.r2r - -doc""" -```rst -.. r2r!(A, kind [, dims]) - -Same as :func:`r2r`, but operates in-place on ``A``, which must be -an array of real or complex floating-point numbers. -``` -""" -FFTW.r2r! - -doc""" -```rst -.. plan_r2r!(A, kind [, dims [, flags [, timelimit]]]) - -Similar to :func:`Base.plan_fft`, but corresponds to :func:`r2r!`. -``` -""" -FFTW.plan_r2r! - -doc""" -```rst -.. plan_r2r(A, kind [, dims [, flags [, timelimit]]]) - -Pre-plan an optimized r2r transform, similar to :func:`Base.plan_fft` -except that the transforms (and the first three arguments) -correspond to :func:`r2r` and :func:`r2r!`, respectively. -``` -""" -FFTW.plan_r2r - doc""" ```rst .. plan_bfft!(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) @@ -740,99 +678,6 @@ Same as :func:`plan_ifft`, but operates in-place on ``A``. """ plan_ifft! -doc""" -```rst -.. dct(A [, dims]) - -Performs a multidimensional type-II discrete cosine transform (DCT) -of the array ``A``, using the unitary normalization of the DCT. -The optional ``dims`` argument specifies an iterable subset of -dimensions (e.g. an integer, range, tuple, or array) to transform -along. Most efficient if the size of ``A`` along the transformed -dimensions is a product of small primes; see :func:`nextprod`. See -also :func:`plan_dct` for even greater efficiency. -``` -""" -dct - -doc""" -```rst -.. idct(A [, dims]) - -Computes the multidimensional inverse discrete cosine transform (DCT) -of the array ``A`` (technically, a type-III DCT with the unitary -normalization). -The optional ``dims`` argument specifies an iterable subset of -dimensions (e.g. an integer, range, tuple, or array) to transform -along. Most efficient if the size of ``A`` along the transformed -dimensions is a product of small primes; see :func:`nextprod`. See -also :func:`plan_idct` for even greater efficiency. -``` -""" -idct - -doc""" -```rst -.. dct!(A [, dims]) - -Same as :func:`dct!`, except that it operates in-place -on ``A``, which must be an array of real or complex floating-point -values. -``` -""" -dct! - -doc""" -```rst -.. idct!(A [, dims]) - -Same as :func:`idct!`, but operates in-place on ``A``. -``` -""" -idct! - -doc""" -```rst -.. plan_dct!(A [, dims [, flags [, timelimit]]]) - -Same as :func:`plan_dct`, but operates in-place on ``A``. -``` -""" -plan_dct! - -doc""" -```rst -.. plan_idct(A [, dims [, flags [, timelimit]]]) - -Pre-plan an optimized inverse discrete cosine transform (DCT), similar to -:func:`plan_fft` except producing a function that computes :func:`idct`. -The first two arguments have the same meaning as for :func:`idct`. -``` -""" -plan_idct - -doc""" -```rst -.. plan_dct(A [, dims [, flags [, timelimit]]]) - -Pre-plan an optimized discrete cosine transform (DCT), similar to -:func:`plan_fft` except producing a function that computes :func:`dct`. -The first two arguments have the same meaning as for :func:`dct`. -``` -""" -plan_dct - -doc""" -```rst -.. plan_idct!(A [, dims [, flags [, timelimit]]]) - -Same as :func:`plan_idct`, but operates in-place on ``A``. -``` -""" -plan_idct! - -end - # Base.Profile doc""" diff --git a/base/fft/FFTW.jl b/base/fft/FFTW.jl index 694c9eafff451..f6f5e9fdb51ed 100644 --- a/base/fft/FFTW.jl +++ b/base/fft/FFTW.jl @@ -745,6 +745,64 @@ function plan_r2r!{T<:fftwNumber,N}(X::StridedArray{T,N}, kinds, region; r2rFFTWPlan{T,ANY,true,N}(X, X, region, kinds, flags, timelimit) end +doc""" +```rst +.. r2r(A, kind [, dims]) + +Performs a multidimensional real-input/real-output (r2r) transform +of type ``kind`` of the array ``A``, as defined in the FFTW manual. +``kind`` specifies either a discrete cosine transform of various types +(``FFTW.REDFT00``, ``FFTW.REDFT01``, ``FFTW.REDFT10``, or +``FFTW.REDFT11``), a discrete sine transform of various types +(``FFTW.RODFT00``, ``FFTW.RODFT01``, ``FFTW.RODFT10``, or +``FFTW.RODFT11``), a real-input DFT with halfcomplex-format output +(``FFTW.R2HC`` and its inverse ``FFTW.HC2R``), or a discrete +Hartley transform (``FFTW.DHT``). The ``kind`` argument may be +an array or tuple in order to specify different transform types +along the different dimensions of ``A``; ``kind[end]`` is used +for any unspecified dimensions. See the FFTW manual for precise +definitions of these transform types, at http://www.fftw.org/doc. + +The optional ``dims`` argument specifies an iterable subset of +dimensions (e.g. an integer, range, tuple, or array) to transform +along. ``kind[i]`` is then the transform type for ``dims[i]``, +with ``kind[end]`` being used for ``i > length(kind)``. + +See also :func:`plan_r2r` to pre-plan optimized r2r transforms. +``` +""" +FFTW.r2r + +doc""" +```rst +.. r2r!(A, kind [, dims]) + +Same as :func:`r2r`, but operates in-place on ``A``, which must be +an array of real or complex floating-point numbers. +``` +""" +FFTW.r2r! + +doc""" +```rst +.. plan_r2r!(A, kind [, dims [, flags [, timelimit]]]) + +Similar to :func:`Base.plan_fft`, but corresponds to :func:`r2r!`. +``` +""" +FFTW.plan_r2r! + +doc""" +```rst +.. plan_r2r(A, kind [, dims [, flags [, timelimit]]]) + +Pre-plan an optimized r2r transform, similar to :func:`Base.plan_fft` +except that the transforms (and the first three arguments) +correspond to :func:`r2r` and :func:`r2r!`, respectively. +``` +""" +FFTW.plan_r2r + # mapping from r2r kind to the corresponding inverse transform const inv_kind = Dict{Int,Int}(R2HC => HC2R, HC2R => R2HC, DHT => DHT, REDFT00 => REDFT00, diff --git a/base/fft/dct.jl b/base/fft/dct.jl index 53c8f6bb65ee7..8d8d3996ef35d 100644 --- a/base/fft/dct.jl +++ b/base/fft/dct.jl @@ -38,6 +38,46 @@ for (pf, pfr, K, inplace) in ((:plan_dct, :plan_r2r, REDFT10, false), end end +doc""" +```rst +.. plan_dct!(A [, dims [, flags [, timelimit]]]) + +Same as :func:`plan_dct`, but operates in-place on ``A``. +``` +""" +plan_dct! + +doc""" +```rst +.. plan_idct(A [, dims [, flags [, timelimit]]]) + +Pre-plan an optimized inverse discrete cosine transform (DCT), similar to +:func:`plan_fft` except producing a function that computes :func:`idct`. +The first two arguments have the same meaning as for :func:`idct`. +``` +""" +plan_idct + +doc""" +```rst +.. plan_dct(A [, dims [, flags [, timelimit]]]) + +Pre-plan an optimized discrete cosine transform (DCT), similar to +:func:`plan_fft` except producing a function that computes :func:`dct`. +The first two arguments have the same meaning as for :func:`dct`. +``` +""" +plan_dct + +doc""" +```rst +.. plan_idct!(A [, dims [, flags [, timelimit]]]) + +Same as :func:`plan_idct`, but operates in-place on ``A``. +``` +""" +plan_idct! + function plan_inv{T,K,inplace}(p::DCTPlan{T,K,inplace}) X = Array(T, p.plan.sz) iK = inv_kind[K] @@ -59,6 +99,57 @@ for f in (:dct, :dct!, :idct, :idct!) end end +doc""" +```rst +.. dct(A [, dims]) + +Performs a multidimensional type-II discrete cosine transform (DCT) +of the array ``A``, using the unitary normalization of the DCT. +The optional ``dims`` argument specifies an iterable subset of +dimensions (e.g. an integer, range, tuple, or array) to transform +along. Most efficient if the size of ``A`` along the transformed +dimensions is a product of small primes; see :func:`nextprod`. See +also :func:`plan_dct` for even greater efficiency. +``` +""" +dct + +doc""" +```rst +.. idct(A [, dims]) + +Computes the multidimensional inverse discrete cosine transform (DCT) +of the array ``A`` (technically, a type-III DCT with the unitary +normalization). +The optional ``dims`` argument specifies an iterable subset of +dimensions (e.g. an integer, range, tuple, or array) to transform +along. Most efficient if the size of ``A`` along the transformed +dimensions is a product of small primes; see :func:`nextprod`. See +also :func:`plan_idct` for even greater efficiency. +``` +""" +idct + +doc""" +```rst +.. dct!(A [, dims]) + +Same as :func:`dct!`, except that it operates in-place +on ``A``, which must be an array of real or complex floating-point +values. +``` +""" +dct! + +doc""" +```rst +.. idct!(A [, dims]) + +Same as :func:`idct!`, but operates in-place on ``A``. +``` +""" +idct! + const sqrthalf = sqrt(0.5) const sqrt2 = sqrt(2.0) const onerange = 1:1 From 1a5e9cff45d5d78bf555d3b43e05fc5dca0137c0 Mon Sep 17 00:00:00 2001 From: nkottary Date: Tue, 15 Sep 2015 14:15:13 +0530 Subject: [PATCH 0163/1938] Moved docstrings for *fft* functions from helpdb.jl to base/dft.jl and base/fft/FFTW.jl --- base/dft.jl | 233 +++++++++++++++++++++++++++++++++++++++++ base/docs/helpdb.jl | 250 -------------------------------------------- base/fft/FFTW.jl | 24 +++++ 3 files changed, 257 insertions(+), 250 deletions(-) diff --git a/base/dft.jl b/base/dft.jl index 706d55c259389..f0eea127eaed5 100644 --- a/base/dft.jl +++ b/base/dft.jl @@ -39,6 +39,170 @@ for f in (:fft, :bfft, :ifft, :fft!, :bfft!, :ifft!, :rfft) end end +doc""" +```rst +.. plan_ifft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Same as :func:`plan_fft`, but produces a plan that performs inverse transforms +:func:`ifft`. +``` +""" +plan_ifft + +doc""" +```rst +.. plan_ifft!(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Same as :func:`plan_ifft`, but operates in-place on ``A``. +``` +""" +plan_ifft! + +doc""" +```rst +.. plan_bfft!(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Same as :func:`plan_bfft`, but operates in-place on ``A``. +``` +""" +plan_bfft! + +doc""" +```rst +.. plan_bfft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Same as :func:`plan_fft`, but produces a plan that performs an unnormalized +backwards transform :func:`bfft`. +``` +""" +plan_bfft + +doc""" +```rst +.. plan_fft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Pre-plan an optimized FFT along given dimensions (``dims``) of arrays +matching the shape and type of ``A``. (The first two arguments have +the same meaning as for :func:`fft`.) Returns an object ``P`` which +represents the linear operator computed by the FFT, and which contains +all of the information needed to compute ``fft(A, dims)`` quickly. + +To apply ``P`` to an array ``A``, use ``P * A``; in general, the +syntax for applying plans is much like that of matrices. (A plan +can only be applied to arrays of the same size as the ``A`` for +which the plan was created.) You can also apply a plan with a +preallocated output array ``�`` by calling ``A_mul_B!(�, plan, +A)``. You can compute the inverse-transform plan by ``inv(P)`` and +apply the inverse plan with ``P \ �`` (the inverse plan is cached +and reused for subsequent calls to ``inv`` or ``\``), and apply the +inverse plan to a pre-allocated output array ``A`` with +``A_ldiv_B!(A, P, �)``. + +The ``flags`` argument is a bitwise-or of FFTW planner flags, defaulting +to ``FFTW.ESTIMATE``. e.g. passing ``FFTW.MEASURE`` or ``FFTW.PATIENT`` +will instead spend several seconds (or more) benchmarking different +possible FFT algorithms and picking the fastest one; see the FFTW manual +for more information on planner flags. The optional ``timelimit`` argument +specifies a rough upper bound on the allowed planning time, in seconds. +Passing ``FFTW.MEASURE`` or ``FFTW.PATIENT`` may cause the input array ``A`` +to be overwritten with zeros during plan creation. + +:func:`plan_fft!` is the same as :func:`plan_fft` but creates a plan +that operates in-place on its argument (which must be an array of +complex floating-point numbers). :func:`plan_ifft` and so on +are similar but produce plans that perform the equivalent of +the inverse transforms :func:`ifft` and so on. +``` +""" +plan_fft + +doc""" +```rst +.. plan_fft!(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Same as :func:`plan_fft`, but operates in-place on ``A``. +``` +""" +plan_fft! + +doc""" +```rst +.. rfft(A [, dims]) + +Multidimensional FFT of a real array ``A``, exploiting the fact that +the transform has conjugate symmetry in order to save roughly half +the computational time and storage costs compared with :func:`fft`. +If ``A`` has size ``(n_1, ..., n_d)``, the result has size +``(div(n_1,2)+1, ..., n_d)``. + +The optional ``dims`` argument specifies an iterable subset of one or +more dimensions of ``A`` to transform, similar to :func:`fft`. Instead +of (roughly) halving the first dimension of ``A`` in the result, the +``dims[1]`` dimension is (roughly) halved in the same way. +``` +""" +rfft + +doc""" +```rst +.. ifft!(A [, dims]) + +Same as :func:`ifft`, but operates in-place on ``A``. +``` +""" +ifft! + +doc""" + ifft(A [, dims]) + +Multidimensional inverse FFT. + +A one-dimensional inverse FFT computes + +$$\operatorname{IDFT}(A)[k] = \frac{1}{\operatorname{length}(A)} +\sum_{n=1}^{\operatorname{length}(A)} \exp\left(+i\frac{2\pi (n-1)(k-1)} +{\operatorname{length}(A)} \right) A[n].$$ + +A multidimensional inverse FFT simply performs this operation along each transformed dimension of `A`. +""" +ifft + +doc""" +```rst +.. fft!(A [, dims]) + +Same as :func:`fft`, but operates in-place on ``A``, +which must be an array of complex floating-point numbers. +``` +""" +fft! + +doc""" +```rst +.. bfft(A [, dims]) + +Similar to :func:`ifft`, but computes an unnormalized inverse (backward) +transform, which must be divided by the product of the sizes of the +transformed dimensions in order to obtain the inverse. (This is slightly +more efficient than :func:`ifft` because it omits a scaling step, which in +some applications can be combined with other computational steps elsewhere.) + +.. math:: + + \operatorname{BDFT}(A)[k] = \operatorname{length}(A) \operatorname{IDFT}(A)[k] +``` +""" +bfft + +doc""" +```rst +.. bfft!(A [, dims]) + +Same as :func:`bfft`, but operates in-place on ``A``. +``` +""" +bfft! + # promote to a complex floating-point type (out-of-place only), # so implementations only need Complex{Float} methods for f in (:fft, :bfft, :ifft) @@ -142,6 +306,36 @@ for f in (:brfft, :irfft) end end +doc""" +```rst +.. irfft(A, d [, dims]) + +Inverse of :func:`rfft`: for a complex array ``A``, gives the +corresponding real array whose FFT yields ``A`` in the first half. +As for :func:`rfft`, ``dims`` is an optional subset of dimensions +to transform, defaulting to ``1:ndims(A)``. + +``d`` is the length of the transformed real array along the ``dims[1]`` +dimension, which must satisfy ``div(d,2)+1 == size(A,dims[1])``. +(This parameter cannot be inferred from ``size(A)`` since both +``2*size(A,dims[1])-2`` as well as ``2*size(A,dims[1])-1`` are valid sizes +for the transformed real array.) +``` +""" +irfft + +doc""" +```rst +.. brfft(A, d [, dims]) + +Similar to :func:`irfft` but computes an unnormalized inverse transform +(similar to :func:`bfft`), which must be divided by the product +of the sizes of the transformed dimensions (of the real output array) +in order to obtain the inverse transform. +``` +""" +brfft + function rfft_output_size(x::AbstractArray, region) d1 = first(region) osize = [size(x)...] @@ -161,26 +355,65 @@ plan_irfft{T}(x::AbstractArray{Complex{T}}, d::Integer, region; kws...) = ScaledPlan(plan_brfft(x, d, region; kws...), normalization(T, brfft_output_size(x, d, region), region)) +doc""" +```rst +.. plan_irfft(A, d [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Pre-plan an optimized inverse real-input FFT, similar to :func:`plan_rfft` +except for :func:`irfft` and :func:`brfft`, respectively. The first +three arguments have the same meaning as for :func:`irfft`. +``` +""" +plan_irfft + ############################################################################## export fftshift, ifftshift fftshift(x) = circshift(x, div([size(x)...],2)) +doc""" + fftshift(x) + +Swap the first and second halves of each dimension of `x`. +""" +fftshift(x) + function fftshift(x,dim) s = zeros(Int,ndims(x)) s[dim] = div(size(x,dim),2) circshift(x, s) end +doc""" + fftshift(x,dim) + +Swap the first and second halves of the given dimension of array `x`. +""" +fftshift(x,dim) + ifftshift(x) = circshift(x, div([size(x)...],-2)) +doc""" + ifftshift(x) + +Undoes the effect of `fftshift`. +""" +ifftshift + function ifftshift(x,dim) s = zeros(Int,ndims(x)) s[dim] = -div(size(x,dim),2) circshift(x, s) end +doc""" + ifftshift(x, [dim]) + +Undoes the effect of `fftshift`. +""" +ifftshift + ############################################################################## # FFTW module (may move to an external package at some point): diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 49f590f590730..48e030f43b403 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -557,127 +557,6 @@ Run the function `f` using the `handler` as the handler. """ Test.with_handler -doc""" -```rst -.. plan_bfft!(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Same as :func:`plan_bfft`, but operates in-place on ``A``. -``` -""" -plan_bfft! - -doc""" -```rst -.. plan_irfft(A, d [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Pre-plan an optimized inverse real-input FFT, similar to :func:`plan_rfft` -except for :func:`irfft` and :func:`brfft`, respectively. The first -three arguments have the same meaning as for :func:`irfft`. -``` -""" -plan_irfft - -doc""" -```rst -.. plan_rfft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Pre-plan an optimized real-input FFT, similar to :func:`plan_fft` -except for :func:`rfft` instead of :func:`fft`. The first two -arguments, and the size of the transformed result, are the same as -for :func:`rfft`. -``` -""" -plan_rfft - -doc""" -```rst -.. plan_fft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Pre-plan an optimized FFT along given dimensions (``dims``) of arrays -matching the shape and type of ``A``. (The first two arguments have -the same meaning as for :func:`fft`.) Returns an object ``P`` which -represents the linear operator computed by the FFT, and which contains -all of the information needed to compute ``fft(A, dims)`` quickly. - -To apply ``P`` to an array ``A``, use ``P * A``; in general, the -syntax for applying plans is much like that of matrices. (A plan -can only be applied to arrays of the same size as the ``A`` for -which the plan was created.) You can also apply a plan with a -preallocated output array ``Â`` by calling ``A_mul_B!(Â, plan, -A)``. You can compute the inverse-transform plan by ``inv(P)`` and -apply the inverse plan with ``P \ Â`` (the inverse plan is cached -and reused for subsequent calls to ``inv`` or ``\``), and apply the -inverse plan to a pre-allocated output array ``A`` with -``A_ldiv_B!(A, P, Â)``. - -The ``flags`` argument is a bitwise-or of FFTW planner flags, defaulting -to ``FFTW.ESTIMATE``. e.g. passing ``FFTW.MEASURE`` or ``FFTW.PATIENT`` -will instead spend several seconds (or more) benchmarking different -possible FFT algorithms and picking the fastest one; see the FFTW manual -for more information on planner flags. The optional ``timelimit`` argument -specifies a rough upper bound on the allowed planning time, in seconds. -Passing ``FFTW.MEASURE`` or ``FFTW.PATIENT`` may cause the input array ``A`` -to be overwritten with zeros during plan creation. - -:func:`plan_fft!` is the same as :func:`plan_fft` but creates a plan -that operates in-place on its argument (which must be an array of -complex floating-point numbers). :func:`plan_ifft` and so on -are similar but produce plans that perform the equivalent of -the inverse transforms :func:`ifft` and so on. -``` -""" -plan_fft - -doc""" -```rst -.. plan_bfft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Same as :func:`plan_fft`, but produces a plan that performs an unnormalized -backwards transform :func:`bfft`. -``` -""" -plan_bfft - -doc""" -```rst -.. plan_fft!(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Same as :func:`plan_fft`, but operates in-place on ``A``. -``` -""" -plan_fft! - -doc""" -```rst -.. plan_ifft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Same as :func:`plan_fft`, but produces a plan that performs inverse transforms -:func:`ifft`. -``` -""" -plan_ifft - -doc""" -```rst -.. plan_brfft(A, d [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Pre-plan an optimized real-input unnormalized transform, similar to -:func:`plan_rfft` except for :func:`brfft` instead of :func:`rfft`. -The first two arguments and the size of the transformed result, are -the same as for :func:`brfft`. -``` -""" -plan_brfft - -doc""" -```rst -.. plan_ifft!(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) - -Same as :func:`plan_ifft`, but operates in-place on ``A``. -``` -""" -plan_ifft! - # Base.Profile doc""" @@ -2981,16 +2860,6 @@ julia> structinfo(StatStruct) """ fieldoffsets -doc""" -```rst -.. fft!(A [, dims]) - -Same as :func:`fft`, but operates in-place on ``A``, -which must be an array of complex floating-point numbers. -``` -""" -fft! - doc""" randn([rng], [dims...]) @@ -3019,13 +2888,6 @@ Modified Bessel function of the first kind of order `nu`, $I_\nu(x)$. """ besseli -doc""" - ifftshift(x, [dim]) - -Undoes the effect of `fftshift`. -""" -ifftshift - doc""" eachmatch(r::Regex, s::AbstractString[, overlap::Bool=false]) @@ -3898,23 +3760,6 @@ Seek a stream relative to the current position. """ skip -doc""" -```rst -.. bfft(A [, dims]) - -Similar to :func:`ifft`, but computes an unnormalized inverse (backward) -transform, which must be divided by the product of the sizes of the -transformed dimensions in order to obtain the inverse. (This is slightly -more efficient than :func:`ifft` because it omits a scaling step, which in -some applications can be combined with other computational steps elsewhere.) - -.. math:: - - \operatorname{BDFT}(A)[k] = \operatorname{length}(A) \operatorname{IDFT}(A)[k] -``` -""" -bfft - doc""" lu(A) -> L, U, p @@ -5032,24 +4877,6 @@ For a given iterable object and iteration state, return the current item and the """ next -doc""" -```rst -.. irfft(A, d [, dims]) - -Inverse of :func:`rfft`: for a complex array ``A``, gives the -corresponding real array whose FFT yields ``A`` in the first half. -As for :func:`rfft`, ``dims`` is an optional subset of dimensions -to transform, defaulting to ``1:ndims(A)``. - -``d`` is the length of the transformed real array along the ``dims[1]`` -dimension, which must satisfy ``div(d,2)+1 == size(A,dims[1])``. -(This parameter cannot be inferred from ``size(A)`` since both -``2*size(A,dims[1])-2`` as well as ``2*size(A,dims[1])-1`` are valid sizes -for the transformed real array.) -``` -""" -irfft - doc""" nnz(A) @@ -5307,15 +5134,6 @@ Removes and returns a value from a `Channel`. Blocks till data is available. """ take!(::Channel) -doc""" -```rst -.. ifft!(A [, dims]) - -Same as :func:`ifft`, but operates in-place on ``A``. -``` -""" -ifft! - doc""" sort!(v, [alg=,] [by=,] [lt=,] [rev=false]) @@ -6745,20 +6563,6 @@ Wake up tasks waiting for a condition, passing them `val`. If `all` is `true` (t """ notify -doc""" - fftshift(x) - -Swap the first and second halves of each dimension of `x`. -""" -fftshift(x) - -doc""" - fftshift(x,dim) - -Swap the first and second halves of the given dimension of array `x`. -""" -fftshift(x,dim) - doc""" unique(itr[, dim]) @@ -7182,15 +6986,6 @@ Construct the symmetric difference of elements in the passed in sets or arrays. """ symdiff -doc""" -```rst -.. bfft!(A [, dims]) - -Same as :func:`bfft`, but operates in-place on ``A``. -``` -""" -bfft! - doc""" histrange(v, n) @@ -7856,21 +7651,6 @@ Send the given value to the last `consume` call, switching to the consumer task. """ produce -doc""" - ifft(A [, dims]) - -Multidimensional inverse FFT. - -A one-dimensional inverse FFT computes - -$$\operatorname{IDFT}(A)[k] = \frac{1}{\operatorname{length}(A)} -\sum_{n=1}^{\operatorname{length}(A)} \exp\left(+i\frac{2\pi (n-1)(k-1)} -{\operatorname{length}(A)} \right) A[n].$$ - -A multidimensional inverse FFT simply performs this operation along each transformed dimension of `A`. -""" -ifft - doc""" StackOverflowError() @@ -8751,18 +8531,6 @@ Compute cosine of `x`, where `x` is in degrees """ cosd -doc""" -```rst -.. brfft(A, d [, dims]) - -Similar to :func:`irfft` but computes an unnormalized inverse transform -(similar to :func:`bfft`), which must be divided by the product -of the sizes of the transformed dimensions (of the real output array) -in order to obtain the inverse transform. -``` -""" -brfft - doc""" cycle(iter) @@ -11502,24 +11270,6 @@ For matrices or vectors $A$ and $B$, calculates $Aᴴ$ \ $Bᴴ$ """ Ac_ldiv_Bc -doc""" -```rst -.. rfft(A [, dims]) - -Multidimensional FFT of a real array ``A``, exploiting the fact that -the transform has conjugate symmetry in order to save roughly half -the computational time and storage costs compared with :func:`fft`. -If ``A`` has size ``(n_1, ..., n_d)``, the result has size -``(div(n_1,2)+1, ..., n_d)``. - -The optional ``dims`` argument specifies an iterable subset of one or -more dimensions of ``A`` to transform, similar to :func:`fft`. Instead -of (roughly) halving the first dimension of ``A`` in the result, the -``dims[1]`` dimension is (roughly) halved in the same way. -``` -""" -rfft - doc""" ```rst .. @enum EnumName EnumValue1[=x] EnumValue2[=y] diff --git a/base/fft/FFTW.jl b/base/fft/FFTW.jl index f6f5e9fdb51ed..e3a6de64f2108 100644 --- a/base/fft/FFTW.jl +++ b/base/fft/FFTW.jl @@ -717,6 +717,30 @@ for (Tr,Tc) in ((:Float32,:Complex64),(:Float64,:Complex128)) end end +doc""" +```rst +.. plan_rfft(A [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Pre-plan an optimized real-input FFT, similar to :func:`plan_fft` +except for :func:`rfft` instead of :func:`fft`. The first two +arguments, and the size of the transformed result, are the same as +for :func:`rfft`. +``` +""" +plan_rfft + +doc""" +```rst +.. plan_brfft(A, d [, dims]; flags=FFTW.ESTIMATE; timelimit=Inf) + +Pre-plan an optimized real-input unnormalized transform, similar to +:func:`plan_rfft` except for :func:`brfft` instead of :func:`rfft`. +The first two arguments and the size of the transformed result, are +the same as for :func:`brfft`. +``` +""" +plan_brfft + # FFTW r2r transforms (low-level interface) for f in (:r2r, :r2r!) From 92aa1a60ab9c3f9772ebe266e9012084d9342173 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 17 Sep 2015 17:10:52 -0400 Subject: [PATCH 0164/1938] if uv_write returns a failure code, free the unused uv_req object --- base/stream.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/stream.jl b/base/stream.jl index 24485279aeea3..56a52d12ddccc 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -935,6 +935,7 @@ function uv_write(s::LibuvStream, p::Ptr, n::UInt) s, p, n, uvw, uv_jl_writecb_task::Ptr{Void}) if err < 0 + Libc.free(uvw) uv_error("write", err) end ct = current_task() From 518aa9ad62bbfb31f86d9de952498b37de876f29 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Fri, 18 Sep 2015 16:57:52 -0400 Subject: [PATCH 0165/1938] fix repl_cmd precompile --- base/precompile.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/precompile.jl b/base/precompile.jl index bf3aed56cf6a7..89be4021a140a 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -348,7 +348,7 @@ precompile(Base.rehash!, (Dict{Any,Any}, Int)) precompile(Base.rehash!, (Dict{UInt8, Any}, Int)) precompile(Base.reinit_stdio, ()) precompile(Base.repeat, (ASCIIString, Int)) -precompile(Base.repl_cmd, (Cmd,)) +precompile(Base.repl_cmd, (Cmd, Base.Terminals.TTYTerminal)) precompile(Base.require, (Symbol,)) precompile(Base.rr2id, (RemoteRef,)) precompile(Base.rsearch, (ASCIIString, Char)) From a64faf893c50d8395a2a942a5ad3c7fe61b88cd6 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Fri, 18 Sep 2015 16:58:05 -0400 Subject: [PATCH 0166/1938] remove unused repl_hook function --- base/client.jl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/base/client.jl b/base/client.jl index 1530242c9787d..85890939fe113 100644 --- a/base/client.jl +++ b/base/client.jl @@ -76,11 +76,6 @@ function repl_cmd(cmd, out) nothing end -function repl_hook(input::AbstractString) - Expr(:call, :(Base.repl_cmd), - macroexpand(Expr(:macrocall,symbol("@cmd"),input))) -end - display_error(er) = display_error(er, []) function display_error(er, bt) with_output_color(:red, STDERR) do io From 0f49b2f063b2c596cbb3a5bf199e24c14da3d967 Mon Sep 17 00:00:00 2001 From: Kiran Pamnany Date: Fri, 18 Sep 2015 14:10:17 -0700 Subject: [PATCH 0167/1938] @nany exported; was already defined --- base/cartesian.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/cartesian.jl b/base/cartesian.jl index 6b4e61fd84295..b6d0f1b21f745 100644 --- a/base/cartesian.jl +++ b/base/cartesian.jl @@ -2,7 +2,7 @@ module Cartesian -export @nloops, @nref, @ncall, @nexprs, @nextract, @nall, @ntuple, @nif +export @nloops, @nref, @ncall, @nexprs, @nextract, @nall, @nany, @ntuple, @nif ### Cartesian-specific macros From 6eab323177601c53c0059299ed3492d156d9d025 Mon Sep 17 00:00:00 2001 From: Kiran Pamnany Date: Fri, 18 Sep 2015 14:12:07 -0700 Subject: [PATCH 0168/1938] Reduce default number of threads to 4. --- src/options.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options.h b/src/options.h index 421a9954c44f7..96b870b15cd9d 100644 --- a/src/options.h +++ b/src/options.h @@ -99,7 +99,7 @@ // defaults for # threads #define NUM_THREADS_NAME "JULIA_NUM_THREADS" -#define DEFAULT_NUM_THREADS 8 +#define DEFAULT_NUM_THREADS 4 // affinitization behavior #define MACHINE_EXCLUSIVE_NAME "JULIA_EXCLUSIVE" From f3402da30706b2e55e8751b5a8460b001db88b74 Mon Sep 17 00:00:00 2001 From: Kiran Pamnany Date: Fri, 18 Sep 2015 14:20:34 -0700 Subject: [PATCH 0169/1938] Refactor threading stuff into Base.Threads The Threads module now includes the threading constructs (@threads), the thread management functions (threadid(), maxthreads(), nthreads()), atomics (Atomic, atomic_cas!, atomic_xchg!, etc.), and locks (SpinLock, RecursiveSpinLock, Mutex, lock! and unlock!). Some test cases are in test/threads.jl. Performance tests are updated in test/perf/threads/. --- base/atomics.jl | 3 - base/locks.jl | 164 ++++++++++++++++++ base/sysimg.jl | 5 +- base/{threading.jl => threadingconstructs.jl} | 2 - base/threads.jl | 33 ++++ test/atomics.jl | 12 -- test/choosetests.jl | 6 +- test/perf/threads/laplace3d/laplace3d.jl | 2 +- test/perf/threads/lbm3d/lbm3d.jl | 2 +- test/perf/threads/stockcorr/pstockcorr.jl | 2 +- test/threads.jl | 104 ++++++++--- 11 files changed, 287 insertions(+), 48 deletions(-) create mode 100644 base/locks.jl rename base/{threading.jl => threadingconstructs.jl} (99%) create mode 100644 base/threads.jl delete mode 100644 test/atomics.jl diff --git a/base/atomics.jl b/base/atomics.jl index f0b4752846a24..916326f407ee0 100644 --- a/base/atomics.jl +++ b/base/atomics.jl @@ -23,8 +23,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -module Atomics - using Base.Intrinsics: llvmcall import Base: setindex!, getindex @@ -82,4 +80,3 @@ for (typ, lt) in atomicintsmap end end -end diff --git a/base/locks.jl b/base/locks.jl new file mode 100644 index 0000000000000..367bc14a0afe0 --- /dev/null +++ b/base/locks.jl @@ -0,0 +1,164 @@ +# Copyright (c) 2015, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include("uv_constants.jl") + +export SpinLock, Mutex, init_lock!, destroy_lock!, lock!, trylock!, unlock! + +abstract Lock + +# Test-and-test-and-set spin locks are quickest up to about 30ish +# contending threads. If you have more contention than that, perhaps +# a lock is the wrong way to synchronize. +type TatasLock <: Lock + handle::Atomic{Int} + TatasLock() = new(Atomic{Int}(0)) +end + +typealias SpinLock TatasLock + +function lock!(l::TatasLock) + while true + if l.handle[] == 0 + p = atomic_xchg!(l.handle, 1) + if p == 0 + return 0 + end + end + # TODO: pause + end +end + +function trylock!(l::TatasLock) + if l.handle[] == 0 + return atomic_xchg!(l.handle, 1) + end + return 1 +end + +function unlock!(l::TatasLock) + l.handle[] = 0 + return 0 +end + + +# Recursive test-and-test-and-set lock. Slower. +type RecursiveTatasLock <: Lock + ownertid::Atomic{Int16} + handle::Atomic{Int} + RecursiveTatasLock() = new(0, Atomic{Int}(0)) +end + +typealias RecursiveSpinLock RecursiveTatasLock + +function lock!(l::RecursiveTatasLock) + if l.ownertid[] == threadid() + return 0 + end + while true + if l.handle[] == 0 + p = atomic_xchg!(l.handle, 1) + if p == 0 + l.ownertid[] = threadid() + return 0 + end + end + # TODO: pause + end +end + +function trylock!(l::RecursiveTatasLock) + if l.ownertid[] == threadid() + return 0 + end + if l.handle[] == 0 + p = atomic_xchg!(l.handle, 1) + if p == 0 + l.ownertid[] = threadid() + end + return p + end + return 1 +end + +function unlock!(l::RecursiveTatasLock) + if l.ownertid[] != threadid() + return 1 + end + l.ownertid[] = 0 + l.handle[] = 0 + return 0 +end + + +# These are mutexes from libuv, which abstract pthread mutexes and +# Windows critical sections. We're doing some error checking (and +# paying for it in overhead), but regardless, in some situations, +# passing a bad parameter will cause an abort. + +# TODO: how defensive to get, and how to turn it off? +# TODO: how to catch an abort? + +# TODO: this size is tested correct for pthreads on Linux and Darwin only +const UV_MUTEX_SIZE = 48 + +type Mutex <: Lock + ownertid::Int16 + handle::Array{Int8} + Mutex() = (m = new(zero(Int16), zeros(Int8, UV_MUTEX_SIZE)); + ccall(:uv_mutex_init, Void, (Ptr{Void},), m.handle); + finalizer(m, (x -> ccall(:uv_mutex_destroy, Void, (Ptr{Void},), x.handle))); + m) +end + +function lock!(m::Mutex) + if m.ownertid == threadid() + return 0 + end + ccall(:uv_mutex_lock, Void, (Ptr{Void},), m.handle) + m.ownertid = threadid() + return 0 +end + +function trylock!(m::Mutex) + if m.ownertid == threadid() + return 0 + end + r = ccall(:uv_mutex_trylock, Cint, (Ptr{Void},), m.handle) + if r == 0 + m.ownertid = threadid() + end + return r +end + +function unlock!(m::Mutex) + if m.ownertid != threadid() + return UV_EPERM + end + m.ownertid = 0 + ccall(:uv_mutex_unlock, Void, (Ptr{Void},), m.handle) + return 0 +end + diff --git a/base/sysimg.jl b/base/sysimg.jl index 50954e94e66cf..c729328338e28 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -299,9 +299,8 @@ include("deprecated.jl") include("docs/helpdb.jl") include("docs/basedocs.jl") -# threading stuff -include("threading.jl") -include("atomics.jl") +# threads +include("threads.jl") function __init__() # Base library init diff --git a/base/threading.jl b/base/threadingconstructs.jl similarity index 99% rename from base/threading.jl rename to base/threadingconstructs.jl index b2e4496a3838d..db6bb4fdc6221 100644 --- a/base/threading.jl +++ b/base/threadingconstructs.jl @@ -23,7 +23,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -module Threading export threadid, maxthreads, nthreads, @threads @@ -112,4 +111,3 @@ macro threads(args...) end end -end # module diff --git a/base/threads.jl b/base/threads.jl new file mode 100644 index 0000000000000..78f7f2a2095e4 --- /dev/null +++ b/base/threads.jl @@ -0,0 +1,33 @@ +# Copyright (c) 2015, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Threads + +include("threadingconstructs.jl") +include("atomics.jl") +include("locks.jl") + +end + diff --git a/test/atomics.jl b/test/atomics.jl deleted file mode 100644 index 9c7d4d511d25e..0000000000000 --- a/test/atomics.jl +++ /dev/null @@ -1,12 +0,0 @@ -using Base.Test -using Base.Threading -using Base.Atomics - -x = Atomic() - -@threads all for i = 1:10000 - atomic_add!(x, 1) -end - -@test x[] == 10000 - diff --git a/test/choosetests.jl b/test/choosetests.jl index 1ffda104bd22a..2a5198567f956 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -30,9 +30,9 @@ function choosetests(choices = []) "euler", "show", "lineedit", "replcompletions", "repl", "replutil", "sets", "test", "goto", "llvmcall", "grisu", "nullable", "meta", "profile", "libgit2", "docs", "markdown", - "threads", "atomics", "base64", "serialize", "functors", "misc", - "enums", "cmdlineargs", "i18n", "workspace", "libdl", "int", - "intset", "floatfuncs", "compile" + "threads", "base64", "serialize", "functors", "misc", "enums", + "cmdlineargs", "i18n", "workspace", "libdl", "int", "intset", + "floatfuncs", "compile" ] if Base.USE_GPL_LIBS diff --git a/test/perf/threads/laplace3d/laplace3d.jl b/test/perf/threads/laplace3d/laplace3d.jl index 9a6e1998f1664..616224e745d48 100644 --- a/test/perf/threads/laplace3d/laplace3d.jl +++ b/test/perf/threads/laplace3d/laplace3d.jl @@ -1,7 +1,7 @@ ## 3D Laplace equation using Base.Cartesian -using Base.Threading +using Base.Threads const sixth = 1.0f0/6.0f0 const error_tol = 0.00001 diff --git a/test/perf/threads/lbm3d/lbm3d.jl b/test/perf/threads/lbm3d/lbm3d.jl index ffaee05cd8eb3..beff7dcb009a8 100644 --- a/test/perf/threads/lbm3d/lbm3d.jl +++ b/test/perf/threads/lbm3d/lbm3d.jl @@ -14,7 +14,7 @@ # commented out in both because of issue #10527. #using PyPlot -using Base.Threading +using Base.Threads include("circshift.jl") diff --git a/test/perf/threads/stockcorr/pstockcorr.jl b/test/perf/threads/stockcorr/pstockcorr.jl index 7f41a1cfbad69..b4530c07f9556 100644 --- a/test/perf/threads/stockcorr/pstockcorr.jl +++ b/test/perf/threads/stockcorr/pstockcorr.jl @@ -2,7 +2,7 @@ ## ## Added threaded implementation (2015-04-06) -using Base.Threading +using Base.Threads #STOCKCORR - The original, unoptimised code that simulates two correlated assets function stockcorr() diff --git a/test/threads.jl b/test/threads.jl index c6969474b7e5d..e58b6f0ba100a 100644 --- a/test/threads.jl +++ b/test/threads.jl @@ -1,47 +1,107 @@ using Base.Test -using Base.Threading +using Base.Threads -expected = [1:nthreads();] +# threading constructs -# test 1 -arr = zeros(Int16, nthreads()) - -function foo(A) +# parallel call form +function threaded_call(A) tid = threadid() A[tid] = tid end -@time @threads all foo(arr) - -@test arr == expected - +function test_threaded_call() + expected = collect(1:nthreads()) + arr = zeros(Int16, nthreads()) + @threads all threaded_call(arr) + @test arr == expected +end -# test 2 -arr = zeros(Int16, nthreads()) +test_threaded_call() -function bar(A) +# parallel loop form +function threaded_loop(A) @threads all for i = 1:nthreads() tid = threadid() A[i] = tid end end -@time bar(arr) - -@test arr == expected - +function test_threaded_loop() + expected = collect(1:nthreads()) + arr = zeros(Int16, nthreads()) + threaded_loop(arr) + @test arr == expected +end -# test 3 -arr = zeros(Int16, nthreads()) +test_threaded_loop() -function baz(A) +# parallel block form +function threaded_block(A) @threads all begin tid = threadid() A[tid] = tid end end -@time baz(arr) +function test_threaded_block() + expected = collect(1:nthreads()) + arr = zeros(Int16, nthreads()) + threaded_block(arr) + @test arr == expected +end + +test_threaded_block() + +# parallel atomic addition +function threaded_atomic_add(x, n) + @threads all for i = 1:n + atomic_add!(x, 1) + end +end + +function test_threaded_atomic_add() + x = Atomic() + threaded_atomic_add(x, 10000) + @test x[] == 10000 +end + +test_threaded_atomic_add() + +# spin locks +function threaded_add_using_spinlock(s, x, n) + @threads all for i = 1:n + lock!(s) + x = x + 1 + unlock!(s) + end + return x +end + +function test_spinlock() + s = SpinLock() + x = 0 + x = threaded_add_using_spinlock(s, x, 10000) + @test x == 10000 +end + +test_spinlock() + +# mutexes +function threaded_add_using_mutex(m, x, n) + @threads all for i = 1:n + lock!(m) + x = x + 1 + unlock!(m) + end + return x +end + +function test_mutex() + m = Mutex() + x = 0 + x = threaded_add_using_mutex(m, x, 10000) + @test x == 10000 +end -@test arr == expected +test_mutex() From eb31d4198c1cdeb5ffb7da586e7bc7ed5d47974b Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Fri, 18 Sep 2015 17:22:53 -0400 Subject: [PATCH 0170/1938] expands discussion of `similar` a bit --- doc/manual/interfaces.rst | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/doc/manual/interfaces.rst b/doc/manual/interfaces.rst index 233dd69baa6cf..15c20ff082d87 100644 --- a/doc/manual/interfaces.rst +++ b/doc/manual/interfaces.rst @@ -173,8 +173,6 @@ A key part in defining an ``AbstractArray`` subtype is :func:`Base.linearindexin This distinction determines which scalar indexing methods the type must define. ``LinearFast()`` arrays are simple: just define :func:`getindex(A::ArrayType, i::Int) `. When the array is subsequently indexed with a multidimensional set of indices, the fallback :func:`getindex(A::AbstractArray, I...)` efficiently converts the indices into one linear index and then calls the above method. ``LinearSlow()`` arrays, on the other hand, require methods to be defined for each supported dimensionality with ``ndims(A)`` ``Int`` indices. For example, the builtin ``SparseMatrix`` type only supports two dimensions, so it just defines :func:`getindex(A::SparseMatrix, i::Int, j::Int)`. The same holds for :func:`setindex!`. -The result of indexing an AbstractArray can itself be an array (for instance when indexing by a ``Range``). The AbstractArray fallback methods use :func:`similar` to create an ``Array`` of the appropriate size and element type, which is filled in using the basic indexing method described above. When implementing an array wrapper you often want the result to be wrapped as well, which can be accomplished by defining ``Base.similar{T}(A::YourArrayType, ::Type{T}, dims::Dims)`` to create the appropriate wrapped array. - Returning to the sequence of squares from above, we could instead define it as a subtype of an ``AbstractArray{Int, 1}``: .. doctest:: @@ -258,7 +256,7 @@ Notice that this is a ``LinearSlow`` array, so we must manually define :func:`ge 2.0 5.0 8.0 3.0 6.0 9.0 -Since the ``SparseArray`` is mutable, we were able to override :func:`similar`. This means that when a base function needs to return an array, it's able to return a new ``SparseArray``: +The result of indexing an AbstractArray can itself be an array (for instance when indexing by a ``Range``). The AbstractArray fallback methods use :func:`similar` to allocate an ``Array`` of the appropriate size and element type, which is filled in using the basic indexing method described above. However, when implementing an array wrapper you often want the result to be wrapped as well: .. doctest:: @@ -267,6 +265,16 @@ Since the ``SparseArray`` is mutable, we were able to override :func:`similar`. 1.0 4.0 7.0 2.0 5.0 8.0 +In this example it is accomplished by defining ``Base.similar{T}(A::SparseArray, ::Type{T}, dims::Dims)`` to create the appropriate wrapped array. For this to work it's important that ``SparseArray`` is mutable (supports ``setindex!``). :func:`similar` is also used to allocate result arrays for arithmetic on `AbstractArray`s, for instance: + +.. doctest:: + + julia> A + 4 + 3x3 SparseArray{Float64,2}: + 5.0 8.0 11.0 + 6.0 9.0 12.0 + 7.0 10.0 13.0 + In addition to all the iterable and indexable methods from above, these types can also interact with each other and use all of the methods defined in the standard library for ``AbstractArrays``: .. doctest:: From e279aea72277dc6ef41ec7ae18627c971d49ba84 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Fri, 18 Sep 2015 18:12:16 -0400 Subject: [PATCH 0171/1938] Char: reduce assumption UInt32(::Char) == reinterpret(UInt32, ::Char) --- base/char.jl | 8 ++++---- base/datafmt.jl | 2 +- base/iostream.jl | 4 ++-- base/unicode/utf16.jl | 2 +- base/unicode/utf32.jl | 5 ++++- base/unicode/utf8.jl | 4 ++-- base/unicode/utf8proc.jl | 7 ++----- test/strings/basic.jl | 10 +++++----- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/base/char.jl b/base/char.jl index 7999f4074b090..25ff9a100beed 100644 --- a/base/char.jl +++ b/base/char.jl @@ -7,8 +7,8 @@ convert{T<:Number}(::Type{T}, x::Char) = convert(T, UInt32(x)) rem{T<:Number}(x::Char, ::Type{T}) = rem(UInt32(x), T) -typemax(::Type{Char}) = Char(typemax(UInt32)) -typemin(::Type{Char}) = Char(typemin(UInt32)) +typemax(::Type{Char}) = reinterpret(Char, typemax(UInt32)) +typemin(::Type{Char}) = reinterpret(Char, typemin(UInt32)) size(c::Char) = () size(c::Char,d) = convert(Int, d) < 1 ? throw(BoundsError()) : 1 @@ -38,8 +38,8 @@ isless(x::Char, y::Integer) = isless(UInt32(x), y) isless(x::Integer, y::Char) = isless(x, UInt32(y)) -(x::Char, y::Char) = Int(x) - Int(y) --(x::Char, y::Integer) = reinterpret(Char, Int32(x) - Int32(y)) -+(x::Char, y::Integer) = reinterpret(Char, Int32(x) + Int32(y)) +-(x::Char, y::Integer) = Char(Int32(x) - Int32(y)) ++(x::Char, y::Integer) = Char(Int32(x) + Int32(y)) +(x::Integer, y::Char) = y + x bswap(x::Char) = Char(bswap(UInt32(x))) diff --git a/base/datafmt.jl b/base/datafmt.jl index a5a9a3581236c..f06f79161c934 100644 --- a/base/datafmt.jl +++ b/base/datafmt.jl @@ -10,7 +10,7 @@ import Base: _default_delims, tryparse_internal export countlines, readdlm, readcsv, writedlm, writecsv -const invalid_dlm = Char(0xfffffffe) +const invalid_dlm = reinterpret(Char, 0xfffffffe) const offs_chunk_size = 5000 countlines(f::AbstractString,eol::Char='\n') = open(io->countlines(io,eol),f)::Int diff --git a/base/iostream.jl b/base/iostream.jl index e3bdd07329baf..b51b7adfaa5ab 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -184,9 +184,9 @@ function write(s::IOStream, c::Char) if !iswritable(s) throw(ArgumentError("write failed, IOStream is not writeable")) end - Int(ccall(:ios_pututf8, Int32, (Ptr{Void}, Char), s.ios, c)) + Int(ccall(:ios_pututf8, Int32, (Ptr{Void}, UInt32), s.ios, c)) end -read(s::IOStream, ::Type{Char}) = ccall(:jl_getutf8, Char, (Ptr{Void},), s.ios) +read(s::IOStream, ::Type{Char}) = Char(ccall(:jl_getutf8, UInt32, (Ptr{Void},), s.ios)) takebuf_string(s::IOStream) = ccall(:jl_takebuf_string, Any, (Ptr{Void},), s.ios)::ByteString diff --git a/base/unicode/utf16.jl b/base/unicode/utf16.jl index 2384177e1128d..712adbb75a896 100644 --- a/base/unicode/utf16.jl +++ b/base/unicode/utf16.jl @@ -259,7 +259,7 @@ function map(fun, str::UTF16String) if !isa(c2, Char) throw(UnicodeError(UTF_ERR_MAP_CHAR, 0, 0)) end - uc = reinterpret(UInt32, c2) + uc = UInt32(c2) if uc < 0x10000 if is_surrogate_codeunit(UInt16(uc)) throw(UnicodeError(UTF_ERR_INVALID_CHAR, 0, uc)) diff --git a/base/unicode/utf32.jl b/base/unicode/utf32.jl index 83e7b60830152..4b9ebeee4f278 100644 --- a/base/unicode/utf32.jl +++ b/base/unicode/utf32.jl @@ -122,9 +122,12 @@ function convert(::Type{UTF32String}, dat::AbstractVector{UInt32}) @inbounds return fast_utf_copy(UTF32String, UInt32, length(dat), dat, true) end -convert{T<:Union{Int32,Char}}(::Type{UTF32String}, data::AbstractVector{T}) = +convert(::Type{UTF32String}, data::AbstractVector{Int32}) = convert(UTF32String, reinterpret(UInt32, convert(Vector{T}, data))) +convert(::Type{UTF32String}, data::AbstractVector{Char}) = + convert(UTF32String, map(UInt32, data)) + convert{T<:AbstractString, S<:Union{UInt32,Char,Int32}}(::Type{T}, v::AbstractVector{S}) = convert(T, utf32(v)) diff --git a/base/unicode/utf8.jl b/base/unicode/utf8.jl index a269ad9486082..5f278c0e18b4b 100644 --- a/base/unicode/utf8.jl +++ b/base/unicode/utf8.jl @@ -79,7 +79,7 @@ function next(s::UTF8String, i::Int) end function first_utf8_byte(ch::Char) - c = reinterpret(UInt32, ch) + c = UInt32(ch) c < 0x80 ? c%UInt8 : c < 0x800 ? ((c>>6) | 0xc0)%UInt8 : c < 0x10000 ? ((c>>12) | 0xe0)%UInt8 : @@ -166,7 +166,7 @@ function string(a::Union{ByteString,Char}...) s = Array(UInt8,0) for d in a if isa(d,Char) - c = reinterpret(UInt32, d::Char) + c = UInt32(d::Char) if c < 0x80 push!(s, c%UInt8) elseif c < 0x800 diff --git a/base/unicode/utf8proc.jl b/base/unicode/utf8proc.jl index 55ed682974a5c..7eee57ea64fa4 100644 --- a/base/unicode/utf8proc.jl +++ b/base/unicode/utf8proc.jl @@ -118,11 +118,8 @@ end charwidth(c::Char) = Int(ccall(:utf8proc_charwidth, Cint, (UInt32,), c)) -# faster x+y that does no overflow checking -fastplus(x::Char, y::UInt32) = reinterpret(Char, reinterpret(UInt32, x) + y) - -lowercase(c::Char) = isascii(c) ? ('A' <= c <= 'Z' ? fastplus(c,0x00000020) : c) : ccall(:utf8proc_tolower, Char, (UInt32,), c) -uppercase(c::Char) = isascii(c) ? ('a' <= c <= 'z' ? fastplus(c,0xffffffe0) : c) : ccall(:utf8proc_toupper, Char, (UInt32,), c) +lowercase(c::Char) = isascii(c) ? ('A' <= c <= 'Z' ? c + 0x20 : c) : Char(ccall(:utf8proc_tolower, UInt32, (UInt32,), c)) +uppercase(c::Char) = isascii(c) ? ('a' <= c <= 'z' ? c - 0x20 : c) : Char(ccall(:utf8proc_toupper, UInt32, (UInt32,), c)) ############################################################################ diff --git a/test/strings/basic.jl b/test/strings/basic.jl index 2b19745579275..a6198e670f91a 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -261,14 +261,14 @@ end # issue # 11389: Vector{UInt32} was copied with UTF32String, unlike Vector{Char} a = UInt32[48,0] b = UTF32String(a) -@test b=="0" +@test b == "0" a[1] = 65 -@test b=="A" +@test b == "A" c = Char['0','\0'] d = UTF32String(c) -@test d=="0" +@test d == "0" c[1] = 'A' -@test d=="A" +@test d == "A" # iteration @test [c for c in "ḟøøƀäṙ"] == ['ḟ', 'ø', 'ø', 'ƀ', 'ä', 'ṙ'] @@ -480,7 +480,7 @@ str = "abcdef\uff\uffff\u10ffffABCDEF" foomap(ch) = (ch > 65) foobar(ch) = Char(0xd800) -foobaz(ch) = Char(0x200000) +foobaz(ch) = reinterpret(Char, typemax(UInt32)) @test_throws UnicodeError map(foomap, utf16(str)) @test_throws UnicodeError map(foobar, utf16(str)) @test_throws UnicodeError map(foobaz, utf16(str)) From 266e5139bb0c066b8677da3c11fc6ca78617a789 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Fri, 18 Sep 2015 15:53:55 -0700 Subject: [PATCH 0172/1938] Avoid using gnu extension ?: syntax also fix dsfmt build with msvc for new srccache location --- deps/Makefile | 2 +- src/intrinsics.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index a7a66437b612a..1bbc057e3062a 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -955,7 +955,7 @@ ifneq ($(USEMSVC), 1) DSFMT_CFLAGS += -O3 -finline-functions -fomit-frame-pointer -fno-strict-aliasing \ --param max-inline-insns-single=1800 -Wmissing-prototypes -Wall -std=c99 -shared else -DSFMT_CFLAGS += -Wl,-dll,-def:../libdSFMT.def +DSFMT_CFLAGS += -Wl,-dll,-def:../../libdSFMT.def endif ifeq ($(ARCH), x86_64) DSFMT_CFLAGS += -msse2 -DHAVE_SSE2 diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 4f82c6437687d..87b789a9bed8a 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -1010,7 +1010,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, if (!newtyp && r->getType() != x->getType()) // cast back to the exact original type (e.g. float vs. int) before remarking as a julia type r = builder.CreateBitCast(r, x->getType()); - return mark_julia_type(r, newtyp ?: xinfo.typ); + return mark_julia_type(r, newtyp ? newtyp : xinfo.typ); } } assert(0); From 3ed181f022eb9c80f951b83ed88bac507b729675 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Fri, 18 Sep 2015 18:21:45 -0400 Subject: [PATCH 0173/1938] test/spawn.jl: fix some odd usage of let in tests --- test/spawn.jl | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/test/spawn.jl b/test/spawn.jl index 5707e28a917f1..fa6b51a9bf3cc 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -308,24 +308,24 @@ let fname = tempname() rm(fname) end -let cmd = AbstractString[] - # Ensure that quoting works - @test Base.shell_split("foo bar baz") == ["foo", "bar", "baz"] - @test Base.shell_split("foo\\ bar baz") == ["foo bar", "baz"] - @test Base.shell_split("'foo bar' baz") == ["foo bar", "baz"] - @test Base.shell_split("\"foo bar\" baz") == ["foo bar", "baz"] - - # "Over quoted" - @test Base.shell_split("'foo\\ bar' baz") == ["foo\\ bar", "baz"] - @test Base.shell_split("\"foo\\ bar\" baz") == ["foo\\ bar", "baz"] - - # Ensure that shell_split handles quoted spaces - cmd = ["/Volumes/External HD/program", "-a"] +# Ensure that quoting works +@test Base.shell_split("foo bar baz") == ["foo", "bar", "baz"] +@test Base.shell_split("foo\\ bar baz") == ["foo bar", "baz"] +@test Base.shell_split("'foo bar' baz") == ["foo bar", "baz"] +@test Base.shell_split("\"foo bar\" baz") == ["foo bar", "baz"] + +# "Over quoted" +@test Base.shell_split("'foo\\ bar' baz") == ["foo\\ bar", "baz"] +@test Base.shell_split("\"foo\\ bar\" baz") == ["foo\\ bar", "baz"] + +# Ensure that shell_split handles quoted spaces +let cmd = ["/Volumes/External HD/program", "-a"] @test Base.shell_split("/Volumes/External\\ HD/program -a") == cmd @test Base.shell_split("'/Volumes/External HD/program' -a") == cmd @test Base.shell_split("\"/Volumes/External HD/program\" -a") == cmd +end - # Backticks should automatically quote where necessary - cmd = ["foo bar", "baz"] +# Backticks should automatically quote where necessary +let cmd = ["foo bar", "baz"] @test string(`$cmd`) == "`'foo bar' baz`" end From cd5922102189ffc329765dd60f038bf3109742b3 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 18 Sep 2015 18:18:21 -0400 Subject: [PATCH 0174/1938] fix bug in shell_parse and shell_split --- base/shell.jl | 2 +- test/spawn.jl | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/base/shell.jl b/base/shell.jl index cfd7eb26ee690..eb091b7922de7 100644 --- a/base/shell.jl +++ b/base/shell.jl @@ -81,7 +81,7 @@ function shell_parse(raw::AbstractString, interp::Bool) if done(s,k) error("unterminated double quote") end - if s[k] == '"' || s[k] == '$' + if s[k] == '"' || s[k] == '$' || s[k] == '\\' update_arg(s[i:j-1]); i = k c, k = next(s,k) end diff --git a/test/spawn.jl b/test/spawn.jl index fa6b51a9bf3cc..9d3d006fd587b 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -329,3 +329,5 @@ end let cmd = ["foo bar", "baz"] @test string(`$cmd`) == "`'foo bar' baz`" end + +@test Base.shell_split("\"\\\\\"") == ["\\"] From 9d8571f57e496350d2229a4517cc270439c0067f Mon Sep 17 00:00:00 2001 From: catawbasam Date: Sat, 19 Sep 2015 00:28:33 +0000 Subject: [PATCH 0175/1938] basedocs #12438 part 4 white space add --- base/docs/basedocs.jl | 113 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/base/docs/basedocs.jl b/base/docs/basedocs.jl index cd2a536c3dc01..859cba1ac73b5 100644 --- a/base/docs/basedocs.jl +++ b/base/docs/basedocs.jl @@ -125,6 +125,119 @@ keywords[:macro] = doc""" This macro takes one argument: `name`. When `@sayhello` is encountered, the quoted expression is expanded to interpolate the value of the argument into the final expression. """ +keywords[:importall] = doc""" + `importall` imports all names exported by the specified module, as if `import` were used individually on all of them. For example: + + importall Distributions + + As with `import`, functions imported by `importall` can be extended. + """ + +keywords[:local] = doc""" + `local` introduces a new local variable. For example: + + function foo(n) + x = 0 + for i = 1:n + local x + x = i + end + x + end + + julia> foo(10) + 0 + + Here `local x` introduces a separate `x` inside the loop, so the function returns `0`. + """ + +keywords[:global] = doc""" + `global x` makes `x` in the current scope and its inner scopes refer to the global variable of that name. In the example below, `global` is needed so the function can modify the global variable `z`: + + z=3 + function foo() + global z=6 + end + + julia> foo() + 6 + julia> z + 6 + + Without the `global` declaration in `foo()`, a new local variable would have been created inside foo(), and the `z` in the global scope would have remained equal to `3`. + """ + +keywords[:let] = doc""" + `let` statements allocate new variable bindings each time they run. Whereas an assignment modifies an existing value location, `let` creates new locations. This difference is only detectable in the case of variables that outlive their scope via closures. + The `let` syntax accepts a comma-separated series of assignments and variable names: + + let var1 = value1, var2, var3 = value3 + code + end + + The assignments are evaluated in order, with each right-hand side evaluated in the scope before the new variable on the left-hand side has been introduced. Therefore it makes sense to write something like `let x = x`, since the two `x` variables are distinct and have separate storage. + """ + +keywords[:quote] = doc""" + `quote` creates multiple expression objects in a block without using the explicit `Expr` constructor. For example: + + ex = quote + x = 1 + y = 2 + x + y + end + + Unlike the other means of quoting, `:( ... )`, this form introduces `QuoteNode` elements to the expression tree, which must be considered when directly manipulating the tree. For other purposes, `:( ... )` and `quote .. end` blocks are treated identically. + """ + +keywords[symbol("'")] = doc""" + `'` is the conjugate transposition operator: + + > A = reshape(1:4, 2,2) + 2x2 Array{Int64,2}: + 1 3 + 2 4 + + > A' + 2x2 Array{Int64,2}: + 1 2 + 3 4 + + > B = A + im + 2x2 Array{Complex{Int64},2}: + 1+1im 3+1im + 2+1im 4+1im + + > B' + 2x2 Array{Complex{Int64},2}: + 1-1im 2-1im + 3-1im 4-1im + """ + +keywords[symbol(".'")] = doc""" + `.'` is the transposition operator: + + > A = reshape(1:4, 2,2) + 2x2 Array{Int64,2}: + 1 3 + 2 4 + + > A.' + 2x2 Array{Int64,2}: + 1 2 + 3 4 + + > B = A + im + 2x2 Array{Complex{Int64},2}: + 1+1im 3+1im + 2+1im 4+1im + + > B.' + 2x2 Array{Complex{Int64},2}: + 1+1im 2+1im + 3+1im 4+1im + """ + keywords[:const] = doc""" `const` is used to declare global variables which are also constant. In almost all code (and particularly performance sensitive code) From 00c7f1c81375fc16b57773db95201e5ff4eb8285 Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Fri, 18 Sep 2015 22:05:17 -0400 Subject: [PATCH 0176/1938] adds some missing backticks --- doc/manual/interfaces.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/interfaces.rst b/doc/manual/interfaces.rst index 15c20ff082d87..cd5f74e3006d8 100644 --- a/doc/manual/interfaces.rst +++ b/doc/manual/interfaces.rst @@ -256,7 +256,7 @@ Notice that this is a ``LinearSlow`` array, so we must manually define :func:`ge 2.0 5.0 8.0 3.0 6.0 9.0 -The result of indexing an AbstractArray can itself be an array (for instance when indexing by a ``Range``). The AbstractArray fallback methods use :func:`similar` to allocate an ``Array`` of the appropriate size and element type, which is filled in using the basic indexing method described above. However, when implementing an array wrapper you often want the result to be wrapped as well: +The result of indexing an ``AbstractArray`` can itself be an array (for instance when indexing by a ``Range``). The ``AbstractArray`` fallback methods use :func:`similar` to allocate an ``Array`` of the appropriate size and element type, which is filled in using the basic indexing method described above. However, when implementing an array wrapper you often want the result to be wrapped as well: .. doctest:: @@ -265,7 +265,7 @@ The result of indexing an AbstractArray can itself be an array (for instance whe 1.0 4.0 7.0 2.0 5.0 8.0 -In this example it is accomplished by defining ``Base.similar{T}(A::SparseArray, ::Type{T}, dims::Dims)`` to create the appropriate wrapped array. For this to work it's important that ``SparseArray`` is mutable (supports ``setindex!``). :func:`similar` is also used to allocate result arrays for arithmetic on `AbstractArray`s, for instance: +In this example it is accomplished by defining ``Base.similar{T}(A::SparseArray, ::Type{T}, dims::Dims)`` to create the appropriate wrapped array. For this to work it's important that ``SparseArray`` is mutable (supports ``setindex!``). :func:`similar` is also used to allocate result arrays for arithmetic on ``AbstractArray``s, for instance: .. doctest:: From c89037500e017b30df660c68ffad2fc17419545f Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Fri, 18 Sep 2015 15:32:13 -0400 Subject: [PATCH 0177/1938] adds some details to at-async and tweaks at-schedule doc --- base/docs/helpdb.jl | 4 ++-- doc/stdlib/parallel.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 7f0bcabe660bf..23e67dbb6253f 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -2384,7 +2384,7 @@ promote doc""" @schedule -Wrap an expression in a `Task` and add it to the scheduler's queue. +Wrap an expression in a `Task` and add it to the local machine's scheduler queue. """ :@schedule @@ -4672,7 +4672,7 @@ erfinv doc""" @async -Wraps an expression in a closure and schedules it to run on the local machine. Also adds it to the set of items that the nearest enclosing `@sync` waits for. +Like `@schedule`, `@async` wraps an expression in a `Task` and adds it to the local machine's scheduler queue. Additionally it adds the task to the set of items that the nearest enclosing `@sync` waits for. `@async` also wraps the expression in a `let x=x, y=y, ...` block to create a new scope with copies of all variables referenced in the expression. """ :@async diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 6075736808aa8..bd54410e8b835 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -97,7 +97,7 @@ Tasks .. Docstring generated from Julia source - Wrap an expression in a ``Task`` and add it to the scheduler's queue. + Wrap an expression in a ``Task`` and add it to the local machine's scheduler queue. .. function:: @task @@ -401,7 +401,7 @@ General Parallel Computing Support .. Docstring generated from Julia source - Wraps an expression in a closure and schedules it to run on the local machine. Also adds it to the set of items that the nearest enclosing ``@sync`` waits for. + Like ``@schedule``\ , ``@async`` wraps an expression in a ``Task`` and adds it to the local machine's scheduler queue. Additionally it adds the task to the set of items that the nearest enclosing ``@sync`` waits for. ``@async`` also wraps the expression in a ``let x=x, y=y, ...`` block to create a new scope with copies of all variables referenced in the expression. .. function:: @sync From 392460c2046ee9c03b25a3bdf927fd62bc602105 Mon Sep 17 00:00:00 2001 From: Darwin Darakananda Date: Fri, 18 Sep 2015 22:21:08 -0700 Subject: [PATCH 0178/1938] Make `invalid_char` type-dependent --- base/datafmt.jl | 13 ++++++++----- test/readdlm.jl | 3 +++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/base/datafmt.jl b/base/datafmt.jl index f06f79161c934..2ef80e8e1d1c6 100644 --- a/base/datafmt.jl +++ b/base/datafmt.jl @@ -9,8 +9,11 @@ import Base: _default_delims, tryparse_internal export countlines, readdlm, readcsv, writedlm, writecsv +invalid_dlm(::Type{Char}) = reinterpret(Char, 0xfffffffe) +invalid_dlm(::Type{UInt8}) = 0xfe +invalid_dlm(::Type{UInt16}) = 0xfffe +invalid_dlm(::Type{UInt32}) = 0xfffffffe -const invalid_dlm = reinterpret(Char, 0xfffffffe) const offs_chunk_size = 5000 countlines(f::AbstractString,eol::Char='\n') = open(io->countlines(io,eol),f)::Int @@ -27,10 +30,10 @@ function countlines(io::IO, eol::Char='\n') nl end -readdlm(input, T::Type; opts...) = readdlm(input, invalid_dlm, T, '\n'; opts...) +readdlm(input, T::Type; opts...) = readdlm(input, invalid_dlm(Char), T, '\n'; opts...) readdlm(input, dlm::Char, T::Type; opts...) = readdlm(input, dlm, T, '\n'; opts...) -readdlm(input; opts...) = readdlm(input, invalid_dlm, '\n'; opts...) +readdlm(input; opts...) = readdlm(input, invalid_dlm(Char), '\n'; opts...) readdlm(input, dlm::Char; opts...) = readdlm(input, dlm, '\n'; opts...) readdlm(input, dlm::Char, eol::Char; opts...) = readdlm_auto(input, dlm, Float64, eol, true; opts...) @@ -231,7 +234,7 @@ end function readdlm_string(sbuff::ByteString, dlm::Char, T::Type, eol::Char, auto::Bool, optsd::Dict) - ign_empty = (dlm == invalid_dlm) + ign_empty = (dlm == invalid_dlm(Char)) quotes = get(optsd, :quotes, true) comments = get(optsd, :comments, true) comment_char = get(optsd, :comment_char, '#') @@ -365,7 +368,7 @@ function dlm_parse{T,D}(dbuff::T, eol::D, dlm::D, qchar::D, cchar::D, ign_adj_dl all_ascii = (D <: UInt8) || (isascii(eol) && isascii(dlm) && (!allow_quote || isascii(qchar)) && (!allow_comments || isascii(cchar))) (T <: UTF8String) && all_ascii && (return dlm_parse(dbuff.data, eol%UInt8, dlm%UInt8, qchar%UInt8, cchar%UInt8, ign_adj_dlm, allow_quote, allow_comments, skipstart, skipblanks, dh)) ncols = nrows = col = 0 - is_default_dlm = (dlm == UInt32(invalid_dlm) % D) + is_default_dlm = (dlm == invalid_dlm(D)) error_str = "" # 0: begin field, 1: quoted field, 2: unquoted field, 3: second quote (could either be end of field or escape character), 4: comment, 5: skipstart state = (skipstart > 0) ? 5 : 0 diff --git a/test/readdlm.jl b/test/readdlm.jl index d584f985abd8b..2957910792b42 100644 --- a/test/readdlm.jl +++ b/test/readdlm.jl @@ -225,3 +225,6 @@ for data in ["A B C", "A B C\n"] @test hdr == AbstractString["A" "B" "C"] @test data == Array(Float64, 0, 3) end + +# fix #13179 parsing unicode lines with default delmiters +@test isequaldlm(readdlm(IOBuffer("# Should ignore this π\n1\tα\n2\tβ\n")), Any[1 "α"; 2 "β"], Any) From 9723ec07bf08bf55c60192b8942d2efe1a94bf57 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sun, 20 Sep 2015 01:22:22 +1000 Subject: [PATCH 0179/1938] Workaround incorrect Uint deprecation message The Uint deprecation warning suggested a system dependent type (UInt64 or UInt32) rather than UInt, due to this being a type alias. Hardcode a warning message for this rather common type, pending a more powerful deprecation warning system. --- src/module.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index ec21d7f7f7ac5..3f0d3f4efc1c6 100644 --- a/src/module.c +++ b/src/module.c @@ -489,7 +489,15 @@ void jl_binding_deprecation_warning(jl_binding_t *b) jl_value_t *v = b->value; if (v && (jl_is_type(v) || (jl_is_function(v) && jl_is_gf(v)))) { jl_printf(JL_STDERR, ", use "); - jl_static_show(JL_STDERR, v); + if (b->owner && strcmp(b->owner->name->name, "Base") == 0 && + strcmp(b->name->name, "Uint") == 0) { + // TODO: Suggesting type b->value is wrong for typealiases. + // Uncommon in Base, hardcoded here for now, see #13221 + jl_printf(JL_STDERR, "UInt"); + } + else { + jl_static_show(JL_STDERR, v); + } jl_printf(JL_STDERR, " instead"); } jl_printf(JL_STDERR, ".\n"); From 9a5c3f047840ba7f7a8106a339becfc33dce7167 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Sat, 19 Sep 2015 13:33:22 -0400 Subject: [PATCH 0180/1938] doc: pipe -> pipeline --- doc/manual/running-external-programs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/running-external-programs.rst b/doc/manual/running-external-programs.rst index dbbe88bbbbb08..56d436a72cf2c 100644 --- a/doc/manual/running-external-programs.rst +++ b/doc/manual/running-external-programs.rst @@ -257,7 +257,7 @@ This expression invokes the ``echo`` command with three words as arguments: "hello", "\|", and "sort". The result is that a single line is printed: "hello \| sort". Inside of backticks, a "\|" is just a literal pipe character. How, then, does one construct a pipeline? -Instead of using "\|" inside of backticks, one uses :func:`pipe`: +Instead of using "\|" inside of backticks, one uses :func:`pipeline`: .. doctest:: From e37936c77fabc805dd75e71219e7aeb77d079d6e Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Sat, 19 Sep 2015 13:36:19 -0400 Subject: [PATCH 0181/1938] remove unused unmark_symbols code --- src/alloc.c | 11 ----------- src/gc.c | 8 -------- 2 files changed, 19 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index a098f230933aa..1903755c884ea 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -418,17 +418,6 @@ static jl_sym_t *mk_symbol(const char *str, size_t len) return sym; } -static void unmark_symbols_(jl_sym_t *root) -{ - while (root != NULL) { - jl_set_typeof(root, jl_sym_type); - unmark_symbols_(root->left); - root = root->right; - } -} - -void jl_unmark_symbols(void) { unmark_symbols_(symtab); } - static jl_sym_t **symtab_lookup(jl_sym_t **ptree, const char *str, size_t len, jl_sym_t **parent) { int x; diff --git a/src/gc.c b/src/gc.c index d05dd4dc11d50..44a88d8854e98 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1316,8 +1316,6 @@ static gcval_t** sweep_page(pool_t* p, gcpage_t* pg, gcval_t **pfl, int sweep_ma return pfl; } -//extern void jl_unmark_symbols(void); - static void gc_sweep_once(int sweep_mask) { #ifdef GC_TIME @@ -1336,12 +1334,6 @@ static void gc_sweep_once(int sweep_mask) sweep_big(sweep_mask); #ifdef GC_TIME jl_printf(JL_STDOUT, "GC sweep big %.2f (freed %d/%d with %d rst)\n", (clock_now() - t0)*1000, big_freed, big_total, big_reset); - t0 = clock_now(); -#endif - //if (sweep_mask == GC_MARKED) - // jl_unmark_symbols(); -#ifdef GC_TIME - jl_printf(JL_STDOUT, "GC sweep symbols %.2f\n", (clock_now() - t0)*1000); #endif } From 08061be840f05865215ed1e9b7d06c15f7197edc Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 18 Sep 2015 14:03:57 -0400 Subject: [PATCH 0182/1938] remove jl_alloca; now unused --- base/inference.jl | 3 +-- src/intrinsics.cpp | 9 +-------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 9609ad741fa1a..2611f17c8fa5d 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1989,8 +1989,7 @@ function is_pure_builtin(f) if !(f === Intrinsics.pointerref || # this one is volatile f === Intrinsics.pointerset || # this one is never effect-free f === Intrinsics.ccall || # this one is never effect-free - f === Intrinsics.llvmcall || # this one is never effect-free - f === Intrinsics.jl_alloca) + f === Intrinsics.llvmcall) # this one is never effect-free return true end end diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 87b789a9bed8a..d6231b77e5a76 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -43,7 +43,7 @@ namespace JL_I { // pointer access pointerref, pointerset, // c interface - ccall, cglobal, jl_alloca, llvmcall + ccall, cglobal, llvmcall }; }; @@ -1345,12 +1345,6 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, Value *tmp = builder.CreateAShr(fy, ConstantInt::get(intt,((IntegerType*)intt)->getBitWidth()-1)); return builder.CreateXor(builder.CreateAdd(x,tmp),tmp); } - HANDLE(jl_alloca,1) { - *newtyp = jl_voidpointer_type; - AllocaInst *AI = builder.CreateAlloca(T_int8, JL_INT(x)); - AI->setAlignment(16); - return AI; - } HANDLE(ceil_llvm,1) { x = FP(x); return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::ceil, @@ -1503,6 +1497,5 @@ extern "C" void jl_init_intrinsic_functions(void) ADD_I(check_top_bit); ADD_I(nan_dom_err); ADD_I(ccall); ADD_I(cglobal); - ADD_I(jl_alloca); ADD_I(llvmcall); } From 425fa466e97bcc08b69f5f6c7f409e0286b91a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1ll=20Haraldsson?= Date: Sat, 19 Sep 2015 16:32:09 -0400 Subject: [PATCH 0183/1938] add `countlines` change to NEWS Missing from #11947 Closes #12844 --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index 135fda38422bd..b989233f52dd7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -278,6 +278,8 @@ Library improvements * Capture groups in regular expressions can now be named using PCRE syntax, `(?P...)`. Capture group matches can be accessed by name by indexing a `Match` object with the name of the group ([#11566]). + * `countlines()` now counts all lines, not just non-empty ([#11947]). + * Array and AbstractArray improvements * New multidimensional iterators and index types for efficient iteration over `AbstractArray`s. Array iteration should generally be written as `for i in eachindex(A) ... end` rather than `for i = 1:length(A) ... end` ([#8432]). From a01f41aba0895d51c92163d87f3317350a500f74 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Sat, 19 Sep 2015 20:16:02 -0400 Subject: [PATCH 0184/1938] test/strings/io.jl: pull array values into local variables --- test/strings/io.jl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/strings/io.jl b/test/strings/io.jl index dc315fb3bb62e..c2a1b7924d9bf 100644 --- a/test/strings/io.jl +++ b/test/strings/io.jl @@ -54,13 +54,14 @@ cx = Any[ ] for i = 1:size(cx,1) - @test cx[i,1] == convert(UInt32, cx[i,2]) - @test string(cx[i,2]) == unescape_string(cx[i,3]) - if isascii(cx[i,2]) || !isprint(cx[i,2]) - @test cx[i,3] == escape_string(string(cx[i,2])) + cp, ch, st = cx[i,:] + @test cp == convert(UInt32, ch) + @test string(ch) == unescape_string(st) + if isascii(ch) || !isprint(ch) + @test st == escape_string(string(ch)) end for j = 1:size(cx,1) - str = string(cx[i,2], cx[j,2]) + str = string(ch, cx[j,2]) @test str == unescape_string(escape_string(str)) end end From a3f13abb6be3d081e3dc5b1a962871b31f9a8490 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Sat, 19 Sep 2015 20:33:20 -0400 Subject: [PATCH 0185/1938] add tests for show(::IO, ::Char) --- test/strings/io.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/strings/io.jl b/test/strings/io.jl index c2a1b7924d9bf..78939ab43405e 100644 --- a/test/strings/io.jl +++ b/test/strings/io.jl @@ -64,6 +64,7 @@ for i = 1:size(cx,1) str = string(ch, cx[j,2]) @test str == unescape_string(escape_string(str)) end + @test repr(ch) == "'$(isprint(ch) ? ch : st)'" end for i = 0:0x7f, p = ["","\0","x","xxx","\x7f","\uFF","\uFFF", From d506475beaa408407d7ef3afb26f3856659cf7e3 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Sat, 19 Sep 2015 19:03:31 -0400 Subject: [PATCH 0186/1938] implement show(::IO, ::Char) directly, not via UTF32String printing --- base/char.jl | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/base/char.jl b/base/char.jl index 25ff9a100beed..12a4bc0ae504c 100644 --- a/base/char.jl +++ b/base/char.jl @@ -45,8 +45,37 @@ isless(x::Integer, y::Char) = isless(x, UInt32(y)) bswap(x::Char) = Char(bswap(UInt32(x))) print(io::IO, c::Char) = (write(io, c); nothing) + +const hex_chars = UInt8['0':'9';'a':'z'] + function show(io::IO, c::Char) - print(io, '\'') - print_escaped(io, utf32(c), "'") - print(io, '\'') + if c <= '\\' + b = c == '\0' ? 0x30 : + c == '\a' ? 0x61 : + c == '\b' ? 0x62 : + c == '\t' ? 0x74 : + c == '\n' ? 0x6e : + c == '\v' ? 0x76 : + c == '\f' ? 0x66 : + c == '\r' ? 0x72 : + c == '\e' ? 0x65 : + c == '\'' ? 0x27 : + c == '\\' ? 0x5c : 0xff + if b != 0xff + write(io, 0x27, 0x5c, b, 0x27) + return + end + end + if isprint(c) + write(io, 0x27, c, 0x27) + else + u = UInt32(c) + write(io, 0x27, 0x5c, c <= '\x7f' ? 0x78 : c <= '\uffff' ? 0x75 : 0x55) + d = max(2, 8 - (leading_zeros(u) >> 2)) + while 0 < d + write(io, hex_chars[((u >> ((d -= 1) << 2)) & 0xf) + 1]) + end + write(io, 0x27) + end + return end From 76f3fe55b860791a136291a8435676e53319e3c7 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sun, 20 Sep 2015 10:59:03 +1000 Subject: [PATCH 0187/1938] Add reminder to remove Uint deprecation hack --- base/deprecated.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/deprecated.jl b/base/deprecated.jl index cdcb77fa60522..f72e9ff61e46d 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -160,6 +160,7 @@ end @deprecate_binding String AbstractString +# 13221 - when removing Uint deprecation, remove hack in jl_binding_deprecation_warning @deprecate_binding Uint UInt @deprecate_binding Uint8 UInt8 @deprecate_binding Uint16 UInt16 From 092825822cf36f057148de7740a4b5a50ccfd93c Mon Sep 17 00:00:00 2001 From: kshyatt Date: Sat, 19 Sep 2015 21:22:05 -0700 Subject: [PATCH 0188/1938] Extrema tests for log and log1p --- test/math.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/math.jl b/test/math.jl index 6adeed1d83626..c7781daa89e25 100644 --- a/test/math.jl +++ b/test/math.jl @@ -157,6 +157,14 @@ for T in (Float32, Float64) @test_approx_eq sinh(x) (exp(x)-exp(-x))/2 @test_approx_eq tan(x) sin(x)/cos(x) @test_approx_eq tanh(x) sinh(x)/cosh(x) + + #Edge cases + @test isinf(log(zero(T))) + @test isnan(log(convert(T,NaN))) + @test_throws DomainError log(-one(T)) + @test isinf(log1p(-one(T))) + @test isnan(log1p(convert(T,NaN))) + @test_throws DomainError log1p(convert(T,-2.0)) end for T in (Int, Float64, BigFloat) From 8681313fbab1dcd88b8a9fe9b56f91edd0b75f9b Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Sun, 20 Sep 2015 11:18:53 -0400 Subject: [PATCH 0189/1938] ui: remove unused '-b' option from getopt list The '-b/--build' option was removed in #11640 a703799d8827c9a7a14fb9a070be909e625afea0 --- ui/repl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/repl.c b/ui/repl.c index 5ef39003f7f6c..1e59fe76044f7 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -117,7 +117,7 @@ void parse_opts(int *argcp, char ***argvp) opt_use_precompiled, opt_incremental }; - static char* shortopts = "+vhqFfH:e:E:P:L:J:C:ip:Ob:"; + static char* shortopts = "+vhqFfH:e:E:P:L:J:C:ip:O"; static struct option longopts[] = { // exposed command line options // NOTE: This set of required arguments need to be kept in sync From f700a90d0288382bc0e5410f86af94bb5bc4599b Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Fri, 28 Aug 2015 12:36:32 -0400 Subject: [PATCH 0190/1938] Add documentation for givens, ldltfact!, and eigvals!. Fixes #10526 [ci skip] --- base/docs/helpdb.jl | 24 +++++++++++------------- base/linalg/eigen.jl | 7 +++++++ base/linalg/givens.jl | 25 +++++++++++++++++++++++++ base/linalg/ldlt.jl | 6 ++++++ doc/stdlib/linalg.rst | 38 +++++++++++++++++++++++++++----------- 5 files changed, 76 insertions(+), 24 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 7f0bcabe660bf..380268012cfff 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -10133,21 +10133,19 @@ Computes `x*y+z` without rounding the intermediate result `x*y`. On some systems fma doc""" -```rst -.. eigvals(A,[irange,][vl,][vu]) -Returns the eigenvalues of ``A``. If ``A`` is :class:`Symmetric`, -:class:`Hermitian` or :class:`SymTridiagonal`, it is possible to calculate -only a subset of the eigenvalues by specifying either a :class:`UnitRange` -``irange`` covering indices of the sorted eigenvalues, or a pair ``vl`` and -``vu`` for the lower and upper boundaries of the eigenvalues. + eigvals(A,[irange,][vl,][vu]) -> values -For general non-symmetric matrices it is possible to specify how the matrix -is balanced before the eigenvector calculation. The option ``permute=true`` -permutes the matrix to become closer to upper triangular, and ``scale=true`` -scales the matrix by its diagonal elements to make rows and columns more -equal in norm. The default is ``true`` for both options. -``` +Returns the eigenvalues of `A`. If `A` is `Symmetric`, `Hermitian` or `SymTridiagonal`, +it is possible to calculate only a subset of the eigenvalues by specifying either a +`UnitRange` `irange` covering indices of the sorted eigenvalues, or a pair `vl` and `vu` +for the lower and upper boundaries of the eigenvalues. + +For general non-symmetric matrices it is possible to specify how the matrix is balanced +before the eigenvector calculation. The option `permute=true` permutes the matrix to +become closer to upper triangular, and `scale=true` scales the matrix by its diagonal +elements to make rows and columns moreequal in norm. The default is `true` for both +options. """ eigvals diff --git a/base/linalg/eigen.jl b/base/linalg/eigen.jl index 2a412c7bb285d..611d8050af737 100644 --- a/base/linalg/eigen.jl +++ b/base/linalg/eigen.jl @@ -71,6 +71,13 @@ eigvecs(A::Union{Number, AbstractMatrix}, args...; kwargs...) = eigvecs(eigfact( eigvecs{T,V,S,U}(F::Union{Eigen{T,V,S,U}, GeneralizedEigen{T,V,S,U}}) = F[:vectors]::S eigvals{T,V,S,U}(F::Union{Eigen{T,V,S,U}, GeneralizedEigen{T,V,S,U}}) = F[:values]::U + +doc""" + + eigvals!(A,[irange,][vl,][vu]) -> values + +Same as `eigvals`, but saves space by overwriting the input `A` (and `B`), instead of creating a copy. +""" function eigvals!{T<:BlasReal}(A::StridedMatrix{T}; permute::Bool=true, scale::Bool=true) issym(A) && return eigvals!(Symmetric(A)) _, valsre, valsim, _ = LAPACK.geevx!(permute ? (scale ? 'B' : 'P') : (scale ? 'S' : 'N'), 'N', 'N', 'N', A) diff --git a/base/linalg/givens.jl b/base/linalg/givens.jl index 7b63429404a16..4f934f906a3ad 100644 --- a/base/linalg/givens.jl +++ b/base/linalg/givens.jl @@ -220,6 +220,19 @@ function givensAlgorithm{T<:AbstractFloat}(f::Complex{T}, g::Complex{T}) return cs, sn, r end +doc""" + + givens{T}(::T, ::T, ::Integer, ::Integer) -> {Givens, T} + +Computes the tuple `(G, r) = givens(f, g, i1, i2)` where `G` is a Givens rotation and `r` +is a scalar such that `G*x=y` with `x[i1]=f`, `x[i2]=g`, `y[i1]=r`, and `y[i2]=0`. The +cosine and sine of the rotation angle can be extracted from the `Givens` type with `G.c` +and `G.s` respectively. The arguments `f` and `g` can be either `Float32`, `Float64`, +`Complex{Float32}`, or `Complex{Float64}`. The `Givens` type supports left multiplication +`G*A` and conjugated transpose right multiplication `A*G'`. The type doesn't have a `size` +and can therefore be multiplied with matrices of arbitrary size as long as `i2<=size(A,2)` +for `G*A` or `i2<=size(A,1)` for `A*G'`. +""" function givens{T}(f::T, g::T, i1::Integer, i2::Integer) if i1 >= i2 throw(ArgumentError("second index must be larger than the first")) @@ -227,7 +240,19 @@ function givens{T}(f::T, g::T, i1::Integer, i2::Integer) c, s, r = givensAlgorithm(f, g) Givens(i1, i2, convert(T, c), convert(T, s)), r end +""" + + givens{T}(::AbstractArray{T}, ::Integer, ::Integer, ::Integer) -> {Givens, T} +Computes the tuple `(G, r) = givens(A, i1, i2, col)` where `G` is Givens rotation and `r` +is a scalar such that `G*A[:,col]=y` with `y[i1]=r`, and `y[i2]=0`. The cosine and sine of +the rotation angle can be extracted from the `Givens` type with `G.c` and `G.s` +respectively. The element type of `A` can be either `Float32`, `Float64`, +`Complex{Float32}`, or `Complex{Float64}`. The `Givens` type supports left multiplication +`G*A` and conjugated transpose right multiplication `A*G'`. The type doesn't have a `size` +and can therefore be multiplied with matrices of arbitrary size as long as `i2<=size(A,2)` +for `G*A` or `i2<=size(A,1)` for `A*G'`. +""" function givens{T}(A::AbstractMatrix{T}, i1::Integer, i2::Integer, col::Integer) if i1 >= i2 throw(ArgumentError("second index must be larger than the first")) diff --git a/base/linalg/ldlt.jl b/base/linalg/ldlt.jl index 1bc5272cf5e15..9c47ab1d101c7 100644 --- a/base/linalg/ldlt.jl +++ b/base/linalg/ldlt.jl @@ -15,6 +15,12 @@ convert{T,S,U<:AbstractMatrix}(::Type{LDLt{T}}, F::LDLt{S,U}) = convert(LDLt{T,U convert{T,S,U}(::Type{Factorization{T}}, F::LDLt{S,U}) = convert(LDLt{T,U}, F) # SymTridiagonal +doc""" + + ldltfact!(::SymTridiagonal) -> LDLt + +Same as `ldltfact`, but saves space by overwriting the input `A`, instead of creating a copy. +""" function ldltfact!{T<:Real}(S::SymTridiagonal{T}) n = size(S,1) d = S.dv diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 5f388022cd153..126e13dd99cae 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -155,6 +155,12 @@ Linear algebra functions in Julia are largely implemented by calling functions f The function calls the C library CHOLMOD and many other functions from the library are wrapped but not exported. +.. function:: ldltfact!(::SymTridiagonal) -> LDLt + + .. Docstring generated from Julia source + + Same as ``ldltfact``\ , but saves space by overwriting the input ``A``\ , instead of creating a copy. + .. function:: qr(A [,pivot=Val{false}][;thin=true]) -> Q, R, [p] .. Docstring generated from Julia source @@ -289,21 +295,19 @@ Linear algebra functions in Julia are largely implemented by calling functions f factorization to a tuple; where possible, using :func:`eigfact` is recommended. -.. function:: eigvals(A,[irange,][vl,][vu]) +.. function:: eigvals(A,[irange,][vl,][vu]) -> values .. Docstring generated from Julia source - Returns the eigenvalues of ``A``. If ``A`` is :class:`Symmetric`, - :class:`Hermitian` or :class:`SymTridiagonal`, it is possible to calculate - only a subset of the eigenvalues by specifying either a :class:`UnitRange` - ``irange`` covering indices of the sorted eigenvalues, or a pair ``vl`` and - ``vu`` for the lower and upper boundaries of the eigenvalues. + Returns the eigenvalues of ``A``\ . If ``A`` is ``Symmetric``\ , ``Hermitian`` or ``SymTridiagonal``\ , it is possible to calculate only a subset of the eigenvalues by specifying either a ``UnitRange`` ``irange`` covering indices of the sorted eigenvalues, or a pair ``vl`` and ``vu`` for the lower and upper boundaries of the eigenvalues. - For general non-symmetric matrices it is possible to specify how the matrix - is balanced before the eigenvector calculation. The option ``permute=true`` - permutes the matrix to become closer to upper triangular, and ``scale=true`` - scales the matrix by its diagonal elements to make rows and columns more - equal in norm. The default is ``true`` for both options. + For general non-symmetric matrices it is possible to specify how the matrix is balanced before the eigenvector calculation. The option ``permute=true`` permutes the matrix to become closer to upper triangular, and ``scale=true`` scales the matrix by its diagonal elements to make rows and columns moreequal in norm. The default is ``true`` for both options. + +.. function:: eigvals!(A,[irange,][vl,][vu]) -> values + + .. Docstring generated from Julia source + + Same as ``eigvals``\ , but saves space by overwriting the input ``A`` (and ``B``\ ), instead of creating a copy. .. function:: eigmax(A) @@ -508,6 +512,18 @@ Linear algebra functions in Julia are largely implemented by calling functions f Return only the singular values from the generalized singular value decomposition of ``A`` and ``B``\ . +.. function:: givens{T}(::T, ::T, ::Integer, ::Integer) -> {Givens, T} + + .. Docstring generated from Julia source + + Computes the tuple ``(G, r) = givens(f, g, i1, i2)`` where ``G`` is a Givens rotation and ``r`` is a scalar such that ``G*x=y`` with ``x[i1]=f``\ , ``x[i2]=g``\ , ``y[i1]=r``\ , and ``y[i2]=0``\ . The cosine and sine of the rotation angle can be extracted from the ``Givens`` type with ``G.c`` and ``G.s`` respectively. The arguments ``f`` and ``g`` can be either ``Float32``\ , ``Float64``\ , ``Complex{Float32}``\ , or ``Complex{Float64}``\ . The ``Givens`` type supports left multiplication ``G*A`` and conjugated transpose right multiplication ``A*G'``\ . The type doesn't have a ``size`` and can therefore be multiplied with matrices of arbitrary size as long as ``i2<=size(A,2)`` for ``G*A`` or ``i2<=size(A,1)`` for ``A*G'``\ . + +.. function:: givens{T}(::AbstractArray{T}, ::Integer, ::Integer, ::Integer) -> {Givens, T} + + .. Docstring generated from Julia source + + Computes the tuple ``(G, r) = givens(A, i1, i2, col)`` where ``G`` is Givens rotation and ``r`` is a scalar such that ``G*A[:,col]=y`` with ``y[i1]=r``\ , and ``y[i2]=0``\ . The cosine and sine of the rotation angle can be extracted from the ``Givens`` type with ``G.c`` and ``G.s`` respectively. The element type of ``A`` can be either ``Float32``\ , ``Float64``\ , ``Complex{Float32}``\ , or ``Complex{Float64}``\ . The ``Givens`` type supports left multiplication ``G*A`` and conjugated transpose right multiplication ``A*G'``\ . The type doesn't have a ``size`` and can therefore be multiplied with matrices of arbitrary size as long as ``i2<=size(A,2)`` for ``G*A`` or ``i2<=size(A,1)`` for ``A*G'``\ . + .. function:: triu(M) .. Docstring generated from Julia source From 7d022e8f39bd5d9573114cb1d9a50e99a33c28fa Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Sun, 20 Sep 2015 12:13:22 -0400 Subject: [PATCH 0191/1938] readdlm: reinterpret Char delims to UInt32 rather than convert More decoupling of Char from its current representation. --- base/datafmt.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/datafmt.jl b/base/datafmt.jl index 2ef80e8e1d1c6..61fce42fdc323 100644 --- a/base/datafmt.jl +++ b/base/datafmt.jl @@ -360,7 +360,7 @@ colval{T<:Char, S<:ByteString}(sbuff::S, startpos::Int, endpos::Int, cells::Arra colval{S<:ByteString}(sbuff::S, startpos::Int, endpos::Int, cells::Array, row::Int, col::Int) = true dlm_parse(s::ASCIIString, eol::Char, dlm::Char, qchar::Char, cchar::Char, ign_adj_dlm::Bool, allow_quote::Bool, allow_comments::Bool, skipstart::Int, skipblanks::Bool, dh::DLMHandler) = begin - dlm_parse(s.data, UInt32(eol)%UInt8, UInt32(dlm)%UInt8, UInt32(qchar)%UInt8, UInt32(cchar)%UInt8, + dlm_parse(s.data, reinterpret(UInt32,eol)%UInt8, reinterpret(UInt32,dlm)%UInt8, reinterpret(UInt32,qchar)%UInt8, reinterpret(UInt32,cchar)%UInt8, ign_adj_dlm, allow_quote, allow_comments, skipstart, skipblanks, dh) end From dff1c901bbb3ac00a48612213b4536584fb2f2af Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Sun, 20 Sep 2015 15:02:52 -0400 Subject: [PATCH 0192/1938] better cmdline option err handling (fixes #12679) Moves all the cmdline option error handling code from client.jl to repl.c. --- base/client.jl | 165 ++++++++++++++++++-------------------------- test/cmdlineargs.jl | 5 ++ ui/repl.c | 30 +++++--- 3 files changed, 92 insertions(+), 108 deletions(-) diff --git a/base/client.jl b/base/client.jl index 85890939fe113..8303bbdb6a204 100644 --- a/base/client.jl +++ b/base/client.jl @@ -207,110 +207,77 @@ function init_bind_addr() LPROC.bind_port = UInt16(bind_port) end -# NOTE: This set of required arguments need to be kept in sync with the required arguments defined in ui/repl.c -let reqarg = Set(UTF8String["--home", "-H", - "--eval", "-e", - "--print", "-E", - "--post-boot", "-P", - "--load", "-L", - "--sysimage", "-J", - "--cpu-target", "-C", - "--procs", "-p", - "--machinefile", - "--color", - "--history-file", - "--startup-file", - "--compile", - "--check-bounds", - "--depwarn", - "--inline", - "--output-o", - "--output-ji", - "--output-bc", - "--bind-to", - "--precompiled"]) - global process_options - function process_options(opts::JLOptions, args::Vector{UTF8String}) - if !isempty(args) - arg = first(args) - if !isempty(arg) && arg[1] == '-' && in(arg, reqarg) - println(STDERR, "julia: option `$arg` is missing an argument") - exit(1) - end - idxs = find(x -> x == "--", args) - if length(idxs) > 1 - println(STDERR, "julia: redundant option terminator `--`") - exit(1) - end - deleteat!(ARGS, idxs) +function process_options(opts::JLOptions, args::Vector{UTF8String}) + if !isempty(args) + arg = first(args) + idxs = find(x -> x == "--", args) + if length(idxs) > 1 + println(STDERR, "julia: redundant option terminator `--`") + exit(1) end - repl = true - startup = (opts.startupfile != 2) - history_file = (opts.historyfile != 0) - quiet = (opts.quiet != 0) - color_set = (opts.color != 0) - global have_color = (opts.color == 1) - global is_interactive = (opts.isinteractive != 0) - while true - # load ~/.juliarc file - startup && load_juliarc() - - # startup worker - if opts.worker != 0 - start_worker() # does not return - end - # add processors - if opts.nprocs > 0 - addprocs(opts.nprocs) - end - # load processes from machine file - if opts.machinefile != C_NULL - addprocs(load_machine_file(bytestring(opts.machinefile))) - end - # load file immediately on all processors - if opts.load != C_NULL - @sync for p in procs() - @async remotecall_fetch(p, include, bytestring(opts.load)) - end - end - # eval expression - if opts.eval != C_NULL - repl = false - eval(Main, parse_input_line(bytestring(opts.eval))) - break - end - # eval expression and show result - if opts.print != C_NULL - repl = false - show(eval(Main, parse_input_line(bytestring(opts.print)))) - println() - break - end - # eval expression but don't disable interactive mode - if opts.postboot != C_NULL - eval(Main, parse_input_line(bytestring(opts.postboot))) - end - # load file - if !isempty(args) - if !isempty(args[1]) && args[1][1] != '-' - # program - repl = false - # remove filename from ARGS - shift!(ARGS) - if !is_interactive - ccall(:jl_exit_on_sigint, Void, (Cint,), 1) - end - include(args[1]) - else - println(STDERR, "julia: unknown option `$(args[1])`") - exit(1) - end + deleteat!(ARGS, idxs) + end + repl = true + startup = (opts.startupfile != 2) + history_file = (opts.historyfile != 0) + quiet = (opts.quiet != 0) + color_set = (opts.color != 0) + global have_color = (opts.color == 1) + global is_interactive = (opts.isinteractive != 0) + while true + # load ~/.juliarc file + startup && load_juliarc() + + # startup worker + if opts.worker != 0 + start_worker() # does not return + end + # add processors + if opts.nprocs > 0 + addprocs(opts.nprocs) + end + # load processes from machine file + if opts.machinefile != C_NULL + addprocs(load_machine_file(bytestring(opts.machinefile))) + end + # load file immediately on all processors + if opts.load != C_NULL + @sync for p in procs() + @async remotecall_fetch(p, include, bytestring(opts.load)) end + end + # eval expression + if opts.eval != C_NULL + repl = false + eval(Main, parse_input_line(bytestring(opts.eval))) + break + end + # eval expression and show result + if opts.print != C_NULL + repl = false + show(eval(Main, parse_input_line(bytestring(opts.print)))) + println() break end - repl |= is_interactive - return (quiet,repl,startup,color_set,history_file) + # eval expression but don't disable interactive mode + if opts.postboot != C_NULL + eval(Main, parse_input_line(bytestring(opts.postboot))) + end + # load file + if !isempty(args) && !isempty(args[1]) + # program + repl = false + # remove filename from ARGS + shift!(ARGS) + if !is_interactive + ccall(:jl_exit_on_sigint, Void, (Cint,), 1) + end + include(args[1]) + end + break end + repl |= is_interactive + return (quiet,repl,startup,color_set,history_file) end const roottask = current_task() diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index ab1736524519c..664c25c4a0b88 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -193,4 +193,9 @@ let exename = `$(joinpath(JULIA_HOME, Base.julia_exename())) --precompiled=yes` # issue #10562 @test readchomp(`$exename -e 'println(ARGS);' ''`) == "UTF8String[\"\"]" + # issue #12679 + @test readchomp(pipeline(ignorestatus(`$exename -f --compile=yes -foo`),stderr=`cat`)) == "ERROR: unknown option `-o`" + @test readchomp(pipeline(ignorestatus(`$exename -f -p`),stderr=`cat`)) == "ERROR: option `-p/--procs` is missing an argument" + @test readchomp(pipeline(ignorestatus(`$exename -f --inline`),stderr=`cat`)) == "ERROR: option `--inline` is missing an argument" + @test readchomp(pipeline(ignorestatus(`$exename -f -e "@show ARGS" -now -- julia RUN.jl`),stderr=`cat`)) == "ERROR: unknown option `-n`" end diff --git a/ui/repl.c b/ui/repl.c index 1e59fe76044f7..ea9cab2f85440 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -160,27 +160,40 @@ void parse_opts(int *argcp, char ***argvp) { 0, 0, 0, 0 } }; // getopt handles argument parsing up to -- delineator - int lastind = optind; int argc = *argcp; + char **argv = *argvp; if (argc > 0) { for (int i=0; i < argc; i++) { - if (!strcmp((*argvp)[i], "--")) { + if (!strcmp(argv[i], "--")) { argc = i; break; } } } - int c; char *endptr; - opterr = 0; - int skip = 0; - while ((c = getopt_long(argc,*argvp,shortopts,longopts,0)) != -1) { + opterr = 0; // suppress getopt warning messages + while (1) { + int lastind = optind; + int c = getopt_long(argc, argv, shortopts, longopts, 0); + if (c == -1) break; switch (c) { case 0: break; case '?': - if (optind != lastind) skip++; - lastind = optind; + case ':': + if (optopt) { + for (struct option *o = longopts; o->val; o++) { + if (optopt == o->val) { + if (strchr(shortopts, o->val)) + jl_errorf("option `-%c/--%s` is missing an argument", o->val, o->name); + else + jl_errorf("option `--%s` is missing an argument", o->name); + } + } + jl_errorf("unknown option `-%c`", optopt); + } else { + jl_errorf("unknown option `%s`", argv[lastind]); + } break; case 'v': // version jl_printf(JL_STDOUT, "julia version %s\n", JULIA_VERSION_STRING); @@ -391,7 +404,6 @@ void parse_opts(int *argcp, char ***argvp) } jl_options.code_coverage = codecov; jl_options.malloc_log = malloclog; - optind -= skip; *argvp += optind; *argcp -= optind; } From a203e721d5a703f5663a55a6ac207e0c70a2022e Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Sun, 20 Sep 2015 21:29:52 -0400 Subject: [PATCH 0193/1938] Adds description of triple-quoting behavior to manual --- doc/manual/strings.rst | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index aea78005454c7..926cc9cda9620 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -439,6 +439,49 @@ backslash: julia> print("I have \$100 in my account.\n") I have $100 in my account. +Triple-Quoted Strings Literals +------------------------------ + +When strings are created using triple-quotes (``"""..."""``) they have some +special behavior that can be useful for creating longer blocks of text. First, +if the opening ``"""`` is followed by a newline, the newline is stripped from +the resulting string. + +:: + + """hello""" + +is equivalent to + +:: + + """ + hello""" + +but + +:: + + """ + + hello""" + +will contain a literal newline at the beginning. Trailing whitespace is left +unaltered. They can contain ``"`` symbols without escaping. Triple-quoted strings +are also dedented to the level of the least-indented line. This is useful for +defining strings within code that is indented. For example: + +.. doctest:: + + julia> str = """ + Hello, + world. + """ + " Hello,\n world.\n" + +In this case the final (empty) line before the closing ``"""`` sets the +indentation level. + Common Operations ----------------- From dc4f1798fe2fa7429e2551caba529cf0b022150d Mon Sep 17 00:00:00 2001 From: Iain Dunning Date: Thu, 10 Sep 2015 22:08:48 -0400 Subject: [PATCH 0194/1938] Add new Base.Test --- base/docs/helpdb.jl | 37 --- base/test.jl | 644 +++++++++++++++++++++++++++++++++++++++----- test/test.jl | 134 ++++----- 3 files changed, 643 insertions(+), 172 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 48e030f43b403..6aa8599c3899b 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -520,43 +520,6 @@ Return `true` iff an array is heap-ordered according to the given order. """ Collections.isheap -# Base.Test - -doc""" - @test_throws(extype, ex) - -Test that the expression `ex` throws an exception of type `extype` and calls the current handler to handle the result. -""" -:(Test.@test_throws) - -doc""" - @test_approx_eq_eps(a, b, tol) - -Test two floating point numbers `a` and `b` for equality taking in account a margin of tolerance given by `tol`. -""" -:(Test.@test_approx_eq_eps) - -doc""" - @test(ex) - -Test the expression `ex` and calls the current handler to handle the result. -""" -:(Test.@test) - -doc""" - @test_approx_eq(a, b) - -Test two floating point numbers `a` and `b` for equality taking in account small numerical errors. -""" -:(Test.@test_approx_eq) - -doc""" - with_handler(f, handler) - -Run the function `f` using the `handler` as the handler. -""" -Test.with_handler - # Base.Profile doc""" diff --git a/base/test.jl b/base/test.jl index a9172998fa513..e74e695c2601d 100644 --- a/base/test.jl +++ b/base/test.jl @@ -1,111 +1,605 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +""" +Simple unit testing functionality: + +* `@test` +* `@test_throws` + +All tests belong to a *test set*. There is a default, task-level +test set that throws on the first failure. Users can choose to wrap +their tests in (possibly nested) test sets that will store results +and summarize them at the end of the test set. See: + +* `@testset` +* `@testloop` + +for more information. +""" module Test -export @test, @test_throws, @test_approx_eq, @test_approx_eq_eps, @inferred +export @test, @test_throws +export @testset, @testloop +# Legacy approximate testing functions, yet to be included +export @test_approx_eq, @test_approx_eq_eps, @inferred +#----------------------------------------------------------------------- + +""" + Result + +All tests produce a result object. This object may or may not be +'stored', depending on whether the test is part of a test set. +""" abstract Result -type Success <: Result + +""" + Pass + +The test condition was true, i.e. the expression evaluated to true or +the correct exception was thrown. +""" +immutable Pass <: Result + test_type::Symbol + orig_expr expr - resultexpr - res - Success(expr, resultexpr=nothing, res=nothing) = new(expr, resultexpr, res) + value +end +function Base.show(io::IO, t::Pass) + print_with_color(:green, io, "Test Passed\n") + print(io, " Expression: ", t.orig_expr) + if !isa(t.expr, Expr) + # Maybe just a constant, like true + print(io, "\n Evaluated: ", t.expr) + elseif t.test_type == :test && t.expr.head == :comparison + # The test was an expression, so display the term-by-term + # evaluated version as well + print(io, "\n Evaluated: ", t.expr) + elseif t.test_type == :test_throws + # The correct type of exception was thrown + print(io, "\n Thrown: ", typeof(t.value)) + end end -type Failure <: Result + +""" + Pass + +The test condition was false, i.e. the expression evaluated to false or +the correct exception was not thrown. +""" +type Fail <: Result + test_type::Symbol + orig_expr expr - resultexpr + value +end +function Base.show(io::IO, t::Fail) + print_with_color(:red, io, "Test Failed\n") + print(io, " Expression: ", t.orig_expr) + if t.test_type == :test_throws + # Either no exception, or wrong exception + print(io, "\n Expected: ", t.expr) + print(io, "\n Thrown: ", typeof(t.value)) + elseif !isa(t.expr, Expr) + # Maybe just a constant, like false + print(io, "\n Evaluated: ", t.expr) + elseif t.test_type == :test && t.expr.head == :comparison + # The test was an expression, so display the term-by-term + # evaluated version as well + print(io, "\n Evaluated: ", t.expr) + end end -Failure(expr) = Failure(expr, nothing) + +""" + Error + +The test condition couldn't be evaluated due to an exception, or +it evaluated to something other than a `Bool`. +""" type Error <: Result - expr - err + test_type::Symbol + orig_expr + value backtrace end - -default_handler(r::Success) = r.res -function default_handler(r::Failure) - if r.resultexpr !== nothing - error("test failed: $(r.resultexpr)\n in expression: $(r.expr)") - else - error("test failed in expression: $(r.expr)") +function Base.show(io::IO, t::Error) + print_with_color(:red, io, "Error During Test\n") + if t.test_type == :test_nonbool + println(io, " Expression evaluated to non-Boolean") + println(io, " Expression: ", t.orig_expr) + print( io, " Value: ", t.value) + elseif t.test_type == :test_error + println(io, " Test threw an exception of type ", typeof(t.value)) + println(io, " Expression: ", t.orig_expr) + # Capture error message and indent to match + errmsg = sprint(showerror, t.value, t.backtrace) + print(io, join(map(line->string(" ",line), + split(errmsg, "\n")), "\n")) end end -default_handler(r::Error) = rethrow(r) -handler() = get(task_local_storage(), :TEST_HANDLER, default_handler) -with_handler(f::Function, handler) = - task_local_storage(f, :TEST_HANDLER, handler) +#----------------------------------------------------------------------- -import Base.showerror +# @test - check if the expression evaluates to true +# In the special case of a comparison, e.g. x == 5, generate code to +# evaluate each term in the comparison individually so the results +# can be displayed nicely. +""" + @test ex -showerror(io::IO, r::Error) = showerror(io, r, []) -function showerror(io::IO, r::Error, bt) - println(io, "test error in expression: $(r.expr)") - showerror(io, r.err, r.backtrace) +Tests that the expression `ex` evaluates to `true`. +Returns a `Pass` `Result` if it does, a `Fail` `Result` if it is +`false`, and an `Error` `Result` if it could not be evaluated. +""" +macro test(ex) + # If the test is a comparison + if typeof(ex) == Expr && ex.head == :comparison + # Generate a temporary for every term in the expression + n = length(ex.args) + terms = [gensym() for i in 1:n] + # Create a new block that evaluates each term in the + # comparison indivudally + comp_block = Expr(:block) + comp_block.args = [:( + $(terms[i]) = $(esc(ex.args[i])) + ) for i in 1:n] + # The block should then evaluate whether the comparison + # evaluates to true by splicing in the new terms into the + # original comparsion. The block returns + # - an expression with the values of terms spliced in + # - the result of the comparison itself + push!(comp_block.args, Expr(:return, + :( Expr(:comparison, $(terms...)), # Terms spliced in + $(Expr(:comparison, terms...)) # Comparison itself + ))) + # Return code that calls do_test with an anonymous function + # that calls the comparison block + :(do_test(()->($comp_block), $(Expr(:quote,ex)))) + else + # Something else, perhaps just a single value + # Return code that calls do_test with an anonymous function + # that returns the expression and its value + :(do_test(()->($(Expr(:quote,ex)), $(esc(ex))), $(Expr(:quote,ex)))) + end end -function do_test(body,qex) - handler()(try - rex, val = body() - val ? Success(qex, rex) : Failure(qex,rex) +# An internal function, called by the code generated by the @test +# macro to actually perform the evaluation and manage the result. +function do_test(predicate, orig_expr) + # get_testset() returns the most recently added tests set + # We then call record() with this test set and the test result + record(get_testset(), + try + # expr, in the case of a comparison, will contain the + # comparison with evaluated values of each term spliced in. + # For anything else, just contains the test expression. + # value is the evaluated value of the whole test expression. + # Ideally it is true, but it may be false or non-Boolean. + expr, value = predicate() + if isa(value, Bool) + value ? Pass(:test, orig_expr, expr, value) : + Fail(:test, orig_expr, expr, value) + else + # If the result is non-Boolean, this counts as an Error + Error(:test_nonbool, orig_expr, value, nothing) + end catch err - Error(qex,err,catch_backtrace()) + # The predicate couldn't be evaluated without throwing an + # exception, so that is an Error and not a Fail + Error(:test_error, orig_expr, err, catch_backtrace()) end) end -function do_test_throws(body, qex, bt, extype) - handler()(try - body() - Failure(qex, "$qex did not throw $(extype === nothing ? "anything" : extype)") +#----------------------------------------------------------------------- + +""" + @test_throws extype ex + +Tests that the expression `ex` throws an exception of type `extype`. +""" +macro test_throws(extype, ex) + :(do_test_throws( ()->($(esc(ex))), $(Expr(:quote,ex)), + backtrace(), $(esc(extype)) )) +end + +# An internal function, called by the code generated by @test_throws +# to evaluate and catch the thrown exception - if it exists +function do_test_throws(predicate, orig_expr, bt, extype) + record(get_testset(), + try + predicate() + # If we hit this line, no exception was thrown. We treat + # this as equivalent to the wrong exception being thrown. + Fail(:test_throws, orig_expr, extype, nothing) catch err - if extype === nothing - Base.warn(""" - @test_throws without an exception type is deprecated; - Use `@test_throws $(typeof(err)) $(qex)` instead. - """, bt = bt) - Success(qex, nothing, err) + # Check the right type of exception was thrown + if isa(err, extype) + Pass(:test_throws, orig_expr, extype, err) else - if isa(err, extype) - Success(qex, nothing, err) - else - if isa(err,Type) - Failure(qex, "the type $err was thrown instead of an instance of $extype") - else - Failure(qex, "$err was thrown instead of $extype") - end - end + Fail(:test_throws, orig_expr, extype, err) end end) end -macro test(ex) - if typeof(ex) == Expr && ex.head == :comparison - syms = [gensym() for i = 1:length(ex.args)] - func_block = Expr(:block) - # insert assignment into a block - func_block.args = [:($(syms[i]) = $(esc(ex.args[i]))) for i = 1:length(ex.args)] - # finish the block with a return - push!(func_block.args, Expr(:return, :(Expr(:comparison, $(syms...)), $(Expr(:comparison, syms...))))) - :(do_test(()->($func_block), $(Expr(:quote,ex)))) +#----------------------------------------------------------------------- + +# The AbstractTestSet interface is defined by two methods: +# record(AbstractTestSet, Result) +# Called by do_test after a test is evaluated +# finish(AbstractTestSet) +# Called after the test set has been popped from the test set stack +abstract AbstractTestSet + +""" + TestSetException + +Thrown when a test set finishes and not all tests passed. +""" +type TestSetException <: Exception + pass::Int + fail::Int + error::Int +end +function Base.show(io::IO, ex::TestSetException) + print(io, "Some tests did not pass: ") + print(io, ex.pass, " passed, ") + print(io, ex.fail, " failed, ") + print(io, ex.error, " errored.") +end + +#----------------------------------------------------------------------- + +""" + FallbackTestSet + +A simple fallback test set that throws immediately on a failure. +""" +immutable FallbackTestSet <: AbstractTestSet +end +fallback_testset = FallbackTestSet() + +# Records nothing, and throws an error immediately whenever a Fail or +# Error occurs. Takes no action in the event of a Pass result +record(ts::FallbackTestSet, t::Pass) = t +function record(ts::FallbackTestSet, t::Union{Fail,Error}) + println(t) + error("There was an error during testing") + t +end +# We don't need to do anything as we don't record anything +finish(ts::FallbackTestSet) = nothing + +#----------------------------------------------------------------------- + +""" + DefaultTestSet + +If using the DefaultTestSet, the test results will be recorded. If there +are any `Fail`s or `Error`s, an exception will be thrown only at the end, +along with a summary of the test results. +""" +type DefaultTestSet <: AbstractTestSet + description::AbstractString + results::Vector + anynonpass::Bool +end +DefaultTestSet(desc) = DefaultTestSet(desc, [], false) + +# For a passing result, simply store the result +record(ts::DefaultTestSet, t::Pass) = (push!(ts.results, t); t) +# For the other result types, immediately print the error message +# but do not terminate. Print a backtrace. +function record(ts::DefaultTestSet, t::Union{Fail,Error}) + print_with_color(:white, ts.description, ": ") + print(t) + Base.show_backtrace(STDOUT, backtrace()) + println() + push!(ts.results, t) + t +end + +# When a DefaultTestSet finishes, it records itself to its parent +# testset, if there is one. This allows for recursive printing of +# the results at the end of the tests +record(ts::DefaultTestSet, t::AbstractTestSet) = push!(ts.results, t) + +# Called at the end of a @testset, behaviour depends on whether +# this is a child of another testset, or the "root" testset +function finish(ts::DefaultTestSet) + # If we are a nested test set, do not print a full summary + # now - let the parent test set do the printing + if get_testset_depth() != 0 + # Attach this test set to the parent test set + parent_ts = get_testset() + record(parent_ts, ts) + return + end + # Calculate the overall number for each type so each of + # the test result types are aligned + passes, fails, errors, c_passes, c_fails, c_errors = get_test_counts(ts) + total_pass = passes + c_passes + total_fail = fails + c_fails + total_error = errors + c_errors + dig_pass = total_pass > 0 ? ndigits(total_pass) : 0 + dig_fail = total_fail > 0 ? ndigits(total_fail) : 0 + dig_error = total_error > 0 ? ndigits(total_error) : 0 + total = total_pass + total_fail + total_error + dig_total = total > 0 ? ndigits(total) : 0 + # For each category, take max of digits and header width if there are + # tests of that type + pass_width = dig_pass > 0 ? max(length("Pass"), dig_pass) : 0 + fail_width = dig_fail > 0 ? max(length("Fail"), dig_fail) : 0 + error_width = dig_error > 0 ? max(length("Error"), dig_error) : 0 + total_width = dig_total > 0 ? max(length("Total"), dig_total) : 0 + # Calculate the alignment of the test result counts by + # recursively walking the tree of test sets + align = max(get_alignment(ts, 0), length("Test Summary:")) + # Print the outer test set header once + print_with_color(:white, rpad("Test Summary:",align," ")) + print(" | ") + if pass_width > 0 + print_with_color(:green, lpad("Pass",pass_width," ")) + print(" ") + end + if fail_width > 0 + print_with_color(:red, lpad("Fail",fail_width," ")) + print(" ") + end + if error_width > 0 + print_with_color(:red, lpad("Error",error_width," ")) + print(" ") + end + if total_width > 0 + print_with_color(:blue, lpad("Total",total_width," ")) + end + println() + # Recursively print a summary at every level + print_counts(ts, 0, align, pass_width, fail_width, error_width, total_width) + # Finally throw an error as we are the outermost test set + if total != total_pass + throw(TestSetException(total_pass,total_fail,total_error)) + end +end + +# Recursive function that finds the column that the result counts +# can begin at by taking into account the width of the descriptions +# and the amount of indentation. If a test set had no failures, and +# no failures in child test sets, there is no need to include those +# in calculating the alignment +function get_alignment(ts::DefaultTestSet, depth::Int) + # The minimum width at this depth is... + ts_width = 2*depth + length(ts.description) + # If all passing, no need to look at children + !ts.anynonpass && return ts_width + # Return the maximum of this width and the minimum width + # for all children (if they exist) + length(ts.results) == 0 && return ts_width + child_widths = map(t->get_alignment(t, depth+1), ts.results) + return max(ts_width, maximum(child_widths)) +end +get_alignment(ts, depth::Int) = 0 + +# Recursive function that counts the number of test results of each +# type directly in the testset, and totals across the child testsets +function get_test_counts(ts::DefaultTestSet) + passes, fails, errors = 0, 0, 0 + c_passes, c_fails, c_errors = 0, 0, 0 + for t in ts.results + isa(t, Pass) && (passes += 1) + isa(t, Fail) && (fails += 1) + isa(t, Error) && (errors += 1) + if isa(t, DefaultTestSet) + np, nf, ne, ncp, ncf, nce = get_test_counts(t) + c_passes += np + ncp + c_fails += nf + ncf + c_errors += ne + nce + end + end + ts.anynonpass = (fails + errors + c_fails + c_errors > 0) + return passes, fails, errors, c_passes, c_fails, c_errors +end + +# Recursive function that prints out the results at each level of +# the tree of test sets +function print_counts(ts::DefaultTestSet, depth, align, + pass_width, fail_width, error_width, total_width) + # Count results by each type at this level, and recursively + # through and child test sets + passes, fails, errors, c_passes, c_fails, c_errors = get_test_counts(ts) + subtotal = passes + fails + errors + c_passes + c_fails + c_errors + + # Print test set header, with an alignment that ensures all + # the test results appear above each other + print(rpad(string(" "^depth, ts.description), align, " "), " | ") + + np = passes + c_passes + if np > 0 + print_with_color(:green, lpad(string(np), pass_width, " "), " ") + elseif pass_width > 0 + # No passes at this level, but some at another level + print(" "^pass_width, " ") + end + + nf = fails + c_fails + if nf > 0 + print_with_color(:red, lpad(string(nf), fail_width, " "), " ") + elseif fail_width > 0 + # No fails at this level, but some at another level + print(" "^fail_width, " ") + end + + ne = errors + c_errors + if ne > 0 + print_with_color(:red, lpad(string(ne), error_width, " "), " ") + elseif error_width > 0 + # No errors at this level, but some at another level + print(" "^error_width, " ") + end + + if np == 0 && nf == 0 && ne == 0 + print_with_color(:blue, "No tests") else - :(do_test(()->($(Expr(:quote,ex)), $(esc(ex))), $(Expr(:quote,ex)))) + print_with_color(:blue, lpad(string(subtotal), total_width, " ")) + end + println() + + # Only print results at lower levels if we had failures + if np != subtotal + for t in ts.results + if isa(t, DefaultTestSet) + print_counts(t, depth + 1, align, + pass_width, fail_width, error_width, total_width) + end + end + end +end + +#----------------------------------------------------------------------- + +""" + @testset "description" begin ... end + @testset begin ... end + +Starts a new test set. The test results will be recorded, and if there +are any `Fail`s or `Error`s, an exception will be thrown only at the end, +along with a summary of the test results. +""" +macro testset(args...) + # Parse arguments to do determine if any options passed in + if length(args) == 2 + # Looks like description format + desc, tests = args + !isa(desc, AbstractString) && error("Unexpected argument to @testset") + elseif length(args) == 1 + # No description provided + desc, tests = "test set", args[1] + elseif length(args) >= 3 + error("Too many arguments to @testset") + else + error("Too few arguments to @testset") + end + # Generate a block of code that initializes a new testset, adds + # it to the task local storage, evaluates the test(s), before + # finally removing the testset and giving it a change to take + # action (such as reporting the results) + ts = gensym() + quote + $ts = DefaultTestSet($desc) + add_testset($ts) + $(esc(tests)) + pop_testset() + finish($ts) end end -macro test_throws(args...) - ex = nothing - extype = nothing - # Users should pass (ExceptionType, Expression) but we give a warning to users that only pass (Expression) - if length(args) == 1 - ex = args[1] - elseif length(args) == 2 - ex = args[2] - extype = args[1] + +""" + @testloop "description \$v" for v in (...) ... end + @testloop for x in (...), y in (...) ... end + +Starts a new test set for each iteration of the loop. The description +string accepts interpolation from the loop indices. If no description +is provided, one is constructed based on the variables. +""" +macro testloop(args...) + # Parse arguments to do determine if any options passed in + if length(args) == 2 + # Looks like description format + desc, testloop = args + isa(desc,AbstractString) || (isa(desc,Expr) && desc.head == :string) || error("Unexpected argument to @testloop") + isa(testloop,Expr) && testloop.head == :for || error("Unexpected argument to @testloop") + + elseif length(args) == 1 + # No description provided + testloop = args[1] + isa(testloop,Expr) && testloop.head == :for || error("Unexpected argument to @testloop") + loopvars = testloop.args[1] + if loopvars.head == :(=) + # 1 variable + v = loopvars.args[1] + desc = Expr(:string,"$v = ",v) + else + # multiple variables + v = loopvars.args[1].args[1] + desc = Expr(:string,"$v = ",v) # first variable + for l = loopvars.args[2:end] + v = l.args[1] + push!(desc.args,", $v = ") + push!(desc.args,v) + end + end + elseif length(args) >= 3 + error("Too many arguments to @testloop") + else + error("Too few arguments to @testloop") + end + + # Uses a similar block as for `@testset`, except that it is + # wrapped in the outer loop provided by the user + ts = gensym() + tests = testloop.args[2] + blk = quote + $ts = DefaultTestSet($(esc(desc))) + add_testset($ts) + $(esc(tests)) + pop_testset() + finish($ts) end - :(do_test_throws(()->($(esc(ex))),$(Expr(:quote,ex)),backtrace(),$(esc(extype)))) + Expr(:for,esc(testloop.args[1]),blk) +end + +#----------------------------------------------------------------------- +# Various helper methods for test sets + +""" + get_testset() + +Retrieve the active test set from the task's local storage. If no +test set is active, use the fallback default test set. +""" +function get_testset() + testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) + return length(testsets) == 0 ? fallback_testset : testsets[end] +end + +""" + add_testset(ts::AbstractTestSet) + +Adds the test set to the task_local_storage. +""" +function add_testset(ts::AbstractTestSet) + testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) + push!(testsets, ts) + setindex!(task_local_storage(), testsets, :__BASETESTNEXT__) +end + +""" + pop_testset() + +Pops the last test set added to the task_local_storage. If there are no +active test sets, returns the default test set. +""" +function pop_testset() + testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) + ret = length(testsets) == 0 ? fallback_testset : pop!(testsets) + setindex!(task_local_storage(), testsets, :__BASETESTNEXT__) + return ret +end + +""" + get_testset_depth() + +Returns the number of active test sets, not including the defaut test set +""" +function get_testset_depth() + testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) + return length(testsets) end +#----------------------------------------------------------------------- +# Legacy approximate testing functions, yet to be included + approx_full(x::AbstractArray) = x approx_full(x::Number) = x approx_full(x) = full(x) @@ -145,10 +639,20 @@ array_eps(a) = eps(float(maximum(x->(isfinite(x) ? abs(x) : oftype(x,NaN)), a))) test_approx_eq(va, vb, astr, bstr) = test_approx_eq(va, vb, 1E4*length(va)*max(array_eps(va), array_eps(vb)), astr, bstr) +""" + @test_approx_eq_eps(a, b, tol) + +Test two floating point numbers `a` and `b` for equality taking in account a margin of tolerance given by `tol`. +""" macro test_approx_eq_eps(a, b, c) :(test_approx_eq($(esc(a)), $(esc(b)), $(esc(c)), $(string(a)), $(string(b)))) end +""" + @test_approx_eq(a, b) + +Test two floating point numbers `a` and `b` for equality taking in account small numerical errors. +""" macro test_approx_eq(a, b) :(test_approx_eq($(esc(a)), $(esc(b)), $(string(a)), $(string(b)))) end diff --git a/test/test.jl b/test/test.jl index bbd8adde09ed0..bfd2aba90dcfc 100644 --- a/test/test.jl +++ b/test/test.jl @@ -1,5 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +using Base.Test + # test file to test testing # Test @test @@ -9,9 +11,6 @@ @test strip("\t hi \n") == "hi" @test strip("\t this should fail \n") != "hi" -scary = Base.Test.Error("hi",DimensionMismatch,[]) -@test sprint(showerror,scary) == "test error in expression: hi\nDimensionMismatch" - a = Array(Float64, 2, 2, 2, 2, 2) a[1,1,1,1,1] = 10 @test a[1,1,1,1,1] == 10 @@ -19,74 +18,79 @@ a[1,1,1,1,1] = 10 @test rand() != rand() +sprint(show, @test true) +sprint(show, @test 10 == 2*5) +sprint(show, @test !false) -# Test with_handler -successflag = false -failureflag = false -errorflag = false -test_handler(r::Test.Success) = !successflag -test_handler(r::Test.Failure) = !failureflag -test_handler(r::Test.Error) = !errorflag +OLD_STDOUT = STDOUT +catch_out = IOStream("") +rd, wr = redirect_stdout() -Test.with_handler(test_handler) do +@testset "no errors" begin @test true - @test successflag - @test !failureflag - @test !errorflag - successflag = false - @test false - @test !successflag - @test failureflag - @test !errorflag - failureflag = false - @test error("throw error") - @test !successflag - @test !failureflag - @test errorflag -end - -# Test evaluation of comparison tests -i7586_1() = 1 -i7586_2() = 7 -i7586_3() = 9 - -comparison_flags_s = [false,false,false] -comparison_flags_f = [false,false,false] -function test_handler2(r::Test.Success) - comparison_flags_s[1] = (r.resultexpr.args[1] == 1) - comparison_flags_s[2] = (r.resultexpr.args[3] == 7) - comparison_flags_s[3] = (r.resultexpr.args[5] == 9) + @test 1 == 1 end -function test_handler2(r::Test.Failure) - comparison_flags_f[1] = (r.resultexpr.args[1] == 1) - comparison_flags_f[2] = (r.resultexpr.args[3] == 7) - comparison_flags_f[3] = (r.resultexpr.args[5] == 10) +try + +@testset "outer" begin + @testset "inner1" begin + @test true + @test false + @test 1 == 1 + @test 2 == :foo + @test 3 == 3 + @testset "d" begin + @test 4 == 4 + end + @testset begin + @test :blank != :notblank + end + end + @testset "inner1" begin + @test 1 == 1 + @test 2 == 2 + @test 3 == :bar + @test 4 == 4 + @test_throws ErrorException 1+1 + @test_throws ErrorException error() + @testset "errrrr" begin + @test "not bool" + @test error() + end + end + + @testset "loop with desc" begin + @testloop "loop1 $T" for T in (Float32, Float64) + @test 1 == T(1) + end + end + @testset "loops without desc" begin + @testloop for T in (Float32, Float64) + @test 1 == T(1) + end + @testloop for T in (Float32, Float64), S in (Int32,Int64) + @test S(1) == T(1) + end + end + srand(123) + @testset "some loops fail" begin + @testloop for i in 1:5 + @test i <= rand(1:10) + end + end end - -Test.with_handler(test_handler2) do - @test i7586_1() <= i7586_2() <= i7586_3() - @test i7586_1() >= i7586_2() >= 10 + # These lines shouldn't be called + redirect_stdout(OLD_STDOUT) + error("No exception was thrown!") +catch ex + redirect_stdout(OLD_STDOUT) + + @test isa(ex, Test.TestSetException) + @test ex.pass == 21 + @test ex.fail == 5 + @test ex.error == 2 end -@test all(comparison_flags_s) -@test all(comparison_flags_f) - -# Test @test_throws -domainerror_thrower() = throw(DomainError()) -boundserror_thrower() = throw(BoundsError()) -error_thrower() = error("An error happened") -@test_throws DomainError domainerror_thrower() -@test_throws BoundsError boundserror_thrower() - -failureflag = false -successflag = false -Test.with_handler(test_handler) do - @test_throws DomainError boundserror_thrower() - @test failureflag - @test_throws DomainError domainerror_thrower() - @test successflag -end - # Test @test_approx_eq # TODO @@ -97,4 +101,4 @@ end @test_throws ErrorException Test.test_approx_eq(ones(10),zeros(10),1e-8,"a","b") # Test @test_approx_eq_eps -# TODO +# TODO \ No newline at end of file From e385bbcbd7483751e47c74d9dc99d83f939fbd55 Mon Sep 17 00:00:00 2001 From: Iain Dunning Date: Thu, 17 Sep 2015 23:45:24 -0400 Subject: [PATCH 0195/1938] New manual for tests --- doc/stdlib/test.rst | 279 +++++++++++++++++++++++++++----------------- 1 file changed, 171 insertions(+), 108 deletions(-) diff --git a/doc/stdlib/test.rst b/doc/stdlib/test.rst index cfd9bf56358da..142d2efd3e2cd 100644 --- a/doc/stdlib/test.rst +++ b/doc/stdlib/test.rst @@ -1,6 +1,6 @@ -***************************** - Unit and Functional Testing -***************************** +************** + Unit Testing +************** Testing Base Julia ------------------ @@ -20,64 +20,185 @@ binary install, you can run the test suite using ``Base.runtests()``. .. module:: Base.Test -Test Framework --------------- +Basic Unit Tests +---------------- -The ``Test`` module contains macros and functions related to testing. -A default handler is provided to run the tests, and a custom one can be -provided by the user by using the :func:`registerhandler` function. +The ``Base.Test`` module provides simple *unit testing* functionality. +Unit testing is a way to see if your code is correct by checking that +the results are what you expect. It can be helpful to ensure your code +still works after you make changes, and can be used when developing as +a way of specifying the behaviors your code should have when complete. -To use the default handler, the macro :func:`@test` can be used directly:: +Simple unit testing can be performed with the :func:`@test` and +:func:`@test_throws` macros: - julia> using Base.Test +.. function:: @test ex - julia> @test 1 == 1 + .. Docstring generated from Julia source - julia> @test 1 == 0 - ERROR: test failed: 1 == 0 - in error at error.jl:21 - in default_handler at test.jl:19 - in do_test at test.jl:39 + Tests that the expression ``ex`` evaluates to ``true``\ . Returns a ``Pass`` ``Result`` if it does, a ``Fail`` ``Result`` if it is ``false``\ , and an ``Error`` ``Result`` if it could not be evaluated. - julia> @test error("This is what happens when a test fails") - ERROR: test error during error("This is what happens when a test fails") - This is what happens when a test fails - in error at error.jl:21 - in anonymous at test.jl:62 - in do_test at test.jl:37 +.. function:: @test_throws extype ex -As seen in the examples above, failures or errors will print the abstract -syntax tree of the expression in question. + .. Docstring generated from Julia source -Another macro is provided to check if the given expression throws an exception of type ``extype``, -:func:`@test_throws`:: + Tests that the expression ``ex`` throws an exception of type ``extype``\ . - julia> @test_throws ErrorException error("An error") - ErrorException("An error") +For example, suppose we want to check our new function ``foo(x)`` works +as expected:: - julia> @test_throws BoundsError error("An error") - ERROR: test failed: error("An error") - in error at error.jl:21 - in default_handler at test.jl:19 - in do_test_throws at test.jl:55 + julia> using Base.Test - julia> @test_throws DomainError throw(DomainError()) - DomainError() + julia> foo(x) = length(x)^2 + foo (generic function with 1 method) - julia> @test_throws DomainError throw(EOFError()) - ERROR: test failed: throw(EOFError()) - in error at error.jl:21 - in default_handler at test.jl:19 - in do_test_throws at test.jl:55 +If the condition is true, a ``Pass`` is returned:: + + julia> @test foo("bar") == 9 + Test Passed + Expression: foo("bar") == 9 + Evaluated: 9 == 9 + + julia> @test foo("fizz") >= 10 + Test Passed + Expression: foo("fizz") >= 10 + Evaluated: 16 >= 10 + +If the condition is false, then a ``Fail`` is returned and an +exception is thrown:: + + julia> @test foo("f") == 20 + Test Failed + Expression: foo("f") == 20 + Evaluated: 1 == 20 + ERROR: There was an error during testing + in record at test.jl:268 + in do_test at test.jl:191 + +If the condition could not be evaluated because an exception was thrown, +which occurs in this case because :func:`length` is not defined for +symbols, an ``Error`` object is returned and an exception is thrown:: + + julia> @test foo(:cat) == 1 + Error During Test + Test threw an exception of type MethodError + Expression: foo(:cat) == 1 + MethodError: `length` has no method matching length(::Symbol) + in foo at none:1 + in anonymous at test.jl:159 + in do_test at test.jl:180 + ERROR: There was an error during testing + in record at test.jl:268 + in do_test at test.jl:191 + +If we expect that evaluating an expression *should* throw an exception, +then we can use :func:`@test_throws` to check this occurs:: + julia> @test_throws MethodError foo(:cat) + Test Passed + Expression: foo(:cat) + Evaluated: MethodError + + +Working with Test Sets +---------------------- + +Typically a large of number of tests are used to make sure functions +work correctly over a range of inputs. In the event a test fails, the +default behavior is to throw an exception immediately. However, it is +normally preferrable to run the rest of the tests first to get a +better picture of how many errors there are in the code being tested. + +The :func:`@testset` and :func:`@testloop` macros can be used to +group tests into *sets*. All the tests in a test set will be run, +and at the end of the test set a summary will be printed. If any of +the tests failed, or could not be evaluated due to an error, the +test set will then throw a ``TestSetException``. + + +.. function:: @testset "description" begin ... end + @testset begin ... end + + .. Docstring generated from Julia source + + Starts a new test set. The test results will be recorded, and if there are any ``Fail``\ s or ``Error``\ s, an exception will be thrown only at the end, along with a summary of the test results. + +.. function:: @testloop "description $v" for v in (...) ... end + @testloop for x in (...), y in (...) ... end + + .. Docstring generated from Julia source -As floating-point values can be imprecise, you can perform approximate -equality checks using either ``@test a ≈ b`` (where ``≈``, typed via -tab completion of ``\approx``, is the ``isapprox`` function) or use -the macros ``@test_approx_eq`` macro (which differs from ``isapprox`` -in that it treats NaN values as equal and has a smaller default -tolerance) or ``@test_approx_eq_eps`` (which takes an extra argument -indicating the relative tolerance):: + Starts a new test set for each iteration of the loop. The description string accepts interpolation from the loop indices. If no description is provided, one is constructed based on the variables. + +We can put our tests for the ``foo(x)`` function in a test set:: + + julia> @testset "Foo Tests" begin + @test foo("a") == 1 + @test foo("ab") == 4 + @test foo("abc") == 9 + end + Test Summary: | Pass Total + Foo Tests | 3 3 + +Test sets can all also be nested:: + + julia> @testset "Foo Tests" begin + @testset "Animals" begin + @test foo("cat") == 9 + @test foo("dog") == foo("cat") + end + @testloop "Arrays $i" for i in 1:3 + @test foo(zeros(i)) == i^2 + @test foo(ones(i)) == i^2 + end + end + Test Summary: | Pass Total + Foo Tests | 8 8 + +In the event that a nested test set has no failures, as happened here, +it will be hidden in the summary. If we do have a test failure, only +the details for the failed test sets will be shown:: + + julia> @testset "Foo Tests" begin + @testset "Animals" begin + @testset "Felines" begin + @test foo("cat") == 9 + end + @testset "Canines" begin + @test foo("dog") == 9 + end + end + @testset "Arrays" begin + @test foo(zeros(2)) == 4 + @test foo(ones(4)) == 15 + end + end + + Arrays: Test Failed + Expression: foo(ones(4)) == 15 + Evaluated: 16 == 15 + in record at test.jl:297 + in do_test at test.jl:191 + Test Summary: | Pass Fail Total + Foo Tests | 3 1 4 + Animals | 2 2 + Arrays | 1 1 2 + ERROR: Some tests did not pass: 3 passed, 1 failed, 0 errored. + in finish at test.jl:362 + + +Other Test Macros +----------------- + +As calculations on floating-point values can be imprecise, you can +perform approximate equality checks using either ``@test a ≈ b`` +(where ``≈``, typed via tab completion of ``\approx``, +is the :func:`isapprox` function) or use :func:`isapprox` directly. + +An alternative is the ``@test_approx_eq`` macro (which differs from +``isapprox`` in that it treats NaN values as equal and has a smaller +default tolerance) or ``@test_approx_eq_eps`` (which takes an extra +argument indicating the relative tolerance):: julia> @test 1 ≈ 0.999999999 @@ -107,58 +228,9 @@ indicating the relative tolerance):: in error at error.jl:22 in test_approx_eq at test.jl:68 -Handlers --------- - -A handler is a function defined for three kinds of arguments: ``Success``, ``Failure``, ``Error``:: - - # An example definition of a test handler - test_handler(r::Success) = nothing - test_handler(r::Failure) = error("test failed: $(r.expr)") - test_handler(r::Error) = rethrow(r) - -A different handler can be used for a block (with :func:`with_handler`):: - - julia> using Base.Test - - julia> custom_handler(r::Test.Success) = println("Success on $(r.expr)") - custom_handler (generic function with 1 method) - - julia> custom_handler(r::Test.Failure) = error("Error on custom handler: $(r.expr)") - custom_handler (generic function with 2 methods) - - julia> custom_handler(r::Test.Error) = rethrow(r) - custom_handler (generic function with 3 methods) - - julia> Test.with_handler(custom_handler) do - @test 1 == 1 - @test 1 != 1 - end - Success on :((1==1)) - ERROR: Error on custom handler: :((1!=1)) - in error at error.jl:21 - in custom_handler at none:1 - in do_test at test.jl:39 - in anonymous at no file:3 - in task_local_storage at task.jl:28 - in with_handler at test.jl:24 - -The ``Success`` and ``Failure`` types include an additonal field, ``resultexpr``, which is a partially evaluated expression. For example, in a comparison it will contain an expression with the left and right sides evaluated. - -Macros ------- - -.. function:: @test(ex) - - .. Docstring generated from Julia source - - Test the expression ``ex`` and calls the current handler to handle the result. - -.. function:: @test_throws(extype, ex) - - .. Docstring generated from Julia source - - Test that the expression ``ex`` throws an exception of type ``extype`` and calls the current handler to handle the result. +Note that these macros will fail immediately, and are not compatible +with :func:`@testset`, so using `@test isapprox` is encouraged when +writing new tests. .. function:: @test_approx_eq(a, b) @@ -172,12 +244,3 @@ Macros Test two floating point numbers ``a`` and ``b`` for equality taking in account a margin of tolerance given by ``tol``\ . -Functions ---------- - -.. function:: with_handler(f, handler) - - .. Docstring generated from Julia source - - Run the function ``f`` using the ``handler`` as the handler. - From cc12b633ac2666a5573389ddcf37cea2763c9cb1 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sun, 20 Sep 2015 12:10:59 -0400 Subject: [PATCH 0196/1938] set LLVM_VER from LLVM_CONFIG when using system LLVM --- Make.inc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Make.inc b/Make.inc index b5d1d8c0a2548..d11567b3100bd 100644 --- a/Make.inc +++ b/Make.inc @@ -634,10 +634,6 @@ endif endif endif -ifeq ($(USE_SYSTEM_LLVM), 1) -JCPPFLAGS+=-DSYSTEM_LLVM -endif - ifeq ($(origin LLVM_CONFIG), undefined) ifeq ($(USE_SYSTEM_LLVM), 1) LLVM_CONFIG := llvm-config$(EXE) @@ -646,6 +642,11 @@ LLVM_CONFIG := $(build_bindir)/llvm-config$(EXE) endif endif # LLVM_CONFIG undefined +ifeq ($(USE_SYSTEM_LLVM), 1) +JCPPFLAGS+=-DSYSTEM_LLVM +LLVM_VER := $(shell $(LLVM_CONFIG) --version) +endif # SYSTEM_LLVM + ifeq ($(BUILD_OS),$(OS)) LLVM_CONFIG_HOST := $(LLVM_CONFIG) else @@ -657,7 +658,7 @@ ifeq ($(shell $(LLVM_CONFIG_HOST) --version),3.3) LLVM_CONFIG_HOST = $(call spawn,$(LLVM_CONFIG)) endif endif -endif # SYSTEM_LLVM +endif ifeq ($(USE_SYSTEM_PCRE), 1) PCRE_CONFIG := pcre2-config From 2a2de21a84fa562fe73fec09c45368142d61d811 Mon Sep 17 00:00:00 2001 From: Tomas Lycken Date: Mon, 21 Sep 2015 15:19:55 +0200 Subject: [PATCH 0197/1938] Name Start Menu shortcuts more descriptively This is the first step towards #13239. It would be nice to also include exact commit number and architecture, but I don't know how to get at them. --- contrib/windows/build-installer.nsi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/windows/build-installer.nsi b/contrib/windows/build-installer.nsi index ba236623e32d1..f4718688bd136 100644 --- a/contrib/windows/build-installer.nsi +++ b/contrib/windows/build-installer.nsi @@ -148,8 +148,8 @@ SectionEnd # Helper function to create Start Menu folder and shortcuts Function AddToStartMenu CreateDirectory "$SMPROGRAMS\${JuliaStartMenuFolder}" - CreateShortcut "$SMPROGRAMS\${JuliaStartMenuFolder}\julia.lnk" "$INSTDIR\julia.lnk" "" "" "" "" "" "The Julia Language" - CreateShortcut "$SMPROGRAMS\${JuliaStartMenuFolder}\Uninstall.lnk" "$instdir\Uninstall.exe" + CreateShortcut "$SMPROGRAMS\${JuliaStartMenuFolder}\julia-${Version}.lnk" "$INSTDIR\julia.lnk" "" "" "" "" "" "The Julia Language" + CreateShortcut "$SMPROGRAMS\${JuliaStartMenuFolder}\Uninstall-Julia-${Version}.lnk" "$instdir\Uninstall.exe" FunctionEnd # Opens the installation folder From c8a5d0974b7dfbb7fc3f87117d8dfd794d44fa80 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 21 Sep 2015 15:25:17 -0400 Subject: [PATCH 0198/1938] using memset in fill! is overly complicated these methods (added in 058ad764dead5cc1306a467de02584c7921a2d23) show no difference in code generated or performance differences on modern julia revert #9142, close #13038 --- base/array.jl | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/base/array.jl b/base/array.jl index f1f8a3958acc0..a1a17d57e24d2 100644 --- a/base/array.jl +++ b/base/array.jl @@ -183,19 +183,9 @@ function fill!(a::Union{Array{UInt8}, Array{Int8}}, x::Integer) end function fill!{T<:Union{Integer,AbstractFloat}}(a::Array{T}, x) - # note: checking bit pattern - xT = convert(T,x) - if isbits(T) && nfields(T)==0 && - ((sizeof(T)==1 && reinterpret(UInt8, xT) == 0) || - (sizeof(T)==2 && reinterpret(UInt16, xT) == 0) || - (sizeof(T)==4 && reinterpret(UInt32, xT) == 0) || - (sizeof(T)==8 && reinterpret(UInt64, xT) == 0)) - ccall(:memset, Ptr{Void}, (Ptr{Void}, Cint, Csize_t), - a, 0, length(a)*sizeof(T)) - else - for i in eachindex(a) - @inbounds a[i] = xT - end + xT = convert(T, x) + for i in eachindex(a) + @inbounds a[i] = xT end return a end From 184c56b1eb8bf1d6a062a5dba49a278ad10a5ecc Mon Sep 17 00:00:00 2001 From: jake bolewski Date: Mon, 21 Sep 2015 18:44:13 -0400 Subject: [PATCH 0199/1938] don't throw a warning for deprecated bindings when --depwarn=no --- src/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index ec21d7f7f7ac5..fb5817f0f66e4 100644 --- a/src/module.c +++ b/src/module.c @@ -481,7 +481,7 @@ DLLEXPORT int jl_is_binding_deprecated(jl_module_t *m, jl_sym_t *var) void jl_binding_deprecation_warning(jl_binding_t *b) { - if (b->deprecated) { + if (b->deprecated && jl_options.depwarn) { if (b->owner) jl_printf(JL_STDERR, "WARNING: %s.%s is deprecated", b->owner->name->name, b->name->name); else From f7786157f06901b57aa3f860daa63607d08d3608 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Tue, 15 Sep 2015 21:59:11 -0700 Subject: [PATCH 0200/1938] Run doc/genstdlib.jl after bootstrap, take 2 this time with a JULIA_ENABLE_DOCBUILD flag to allow disabling this step on the buildbots where #11727 causes trouble on windows ref #13069 --- Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 69770e3d5986d..4bc4791e0914c 100644 --- a/Makefile +++ b/Makefile @@ -98,7 +98,13 @@ julia-sysimg-release : julia-inference julia-ui-release julia-sysimg-debug : julia-inference julia-ui-debug @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) JULIA_BUILD_MODE=debug -julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest +JULIA_ENABLE_DOCBUILD ?= 1 +julia-genstdlib : julia-sysimg-$(JULIA_BUILD_MODE) +ifeq ($(JULIA_ENABLE_DOCBUILD), 1) + @$(call PRINT_JULIA, $(JULIA_EXECUTABLE) doc/genstdlib.jl) +endif + +julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest julia-genstdlib debug release : % : julia-% From d09545be63296a96b0b7507366dfb646b8b6066c Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Mon, 21 Sep 2015 23:03:05 -0400 Subject: [PATCH 0201/1938] Use Base's getindex in eval_user_input hack To avoid breakage when getindex is overwritten --- base/REPL.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/REPL.jl b/base/REPL.jl index 1616ca8f85e24..af165fec45532 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -57,7 +57,7 @@ function eval_user_input(ast::ANY, backend::REPLBackend) ans = backend.ans # note: value wrapped in a non-syntax value to avoid evaluating # possibly-invalid syntax (issue #6763). - eval(Main, :(ans = $(Any[ans])[1])) + eval(Main, :(ans = $(getindex)($(Any[ans]), 1))) backend.in_eval = true value = eval(Main, ast) backend.in_eval = false From f6665b12455d18232fcbf895529d3e64ac53ce11 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 16 Sep 2015 14:46:56 -0400 Subject: [PATCH 0202/1938] more robust git-externals implementation --- deps/Makefile | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index 1bbc057e3062a..5b89213a10e0b 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -263,12 +263,12 @@ include $(SRCDIR)/$1.version ifneq ($(NO_GIT),1) $2_SRC_DIR := $1 $2_SRC_FILE := $$(SRCDIR)/srccache/$1.git -$$($2_SRC_FILE): | $$(SRCDIR)/srccache - git clone -q --mirror --branch $$($2_BRANCH) $$($2_GIT_URL) $$@ -$5/$1: | $$($2_SRC_FILE) +$$($2_SRC_FILE)/HEAD: | $$(SRCDIR)/srccache + git clone -q --mirror --branch $$($2_BRANCH) $$($2_GIT_URL) $$(dir $$@) +$5/$1/.git/HEAD: | $$($2_SRC_FILE)/HEAD # try to update the cache, if that fails, attempt to continue anyways (the ref might already be local) - -cd $$(SRCDIR)/srccache/$1.git && git fetch -q $$($2_GIT_URL) $$($2_BRANCH):remotes/origin/$$($2_BRANCH) - git clone -q --depth=10 --branch $$($2_BRANCH) $$(SRCDIR)/srccache/$1.git $$@ + -cd $$($2_SRC_FILE) && git fetch -q $$($2_GIT_URL) $$($2_BRANCH):remotes/origin/$$($2_BRANCH) + git clone -q --depth=10 --branch $$($2_BRANCH) $$($2_SRC_FILE) $5/$1 cd $5/$1 && git remote set-url origin $$($2_GIT_URL) touch -c $5/$1/$3 ifneq ($5,$$(BUILDDIR)) @@ -276,25 +276,14 @@ $$(BUILDDIR)/$1: mkdir -p $$@ $5/$1/$3: | $$(BUILDDIR)/$1 endif -$5/$1/$3: $$(SRCDIR)/$1.version | $5/$1 +$5/$1/$3: $$(SRCDIR)/$1.version | $5/$1/.git/HEAD # try to update the cache, if that fails, attempt to continue anyways (the ref might already be local) -cd $$(SRCDIR)/srccache/$1.git && git fetch -q $$($2_GIT_URL) $$($2_BRANCH):remotes/origin/$$($2_BRANCH) cd $5/$1 && git fetch -q $$(SRCDIR)/srccache/$1.git $$($2_BRANCH):remotes/origin/$$($2_BRANCH) cd $5/$1 && git checkout -q --detach $$($2_SHA1) @[ '$$($2_SHA1)' = "$$$$(cd $5/$1 && git show -s --format='%H' HEAD)" ] || echo $$(WARNCOLOR)'==> warning: SHA1 hash did not match $1.version file'$$(ENDCOLOR) touch -c $$@ -ifeq ($5,.) -ifeq (exists, $$(shell [ -d $5/$1/.git ] && echo exists )) -$5/$1/$4: $5/$1/.git/HEAD -endif -ifeq (exists, $$(shell [ -d $$(JULIAHOME)/.git/modules/deps/$1 ] && echo exists )) -# also allow backwards compatibility with an existing git-submodule: -$5/$1/$4: $$(JULIAHOME)/.git/modules/deps/$1/HEAD -endif -else #unconditionally add dependency on expected location $5/$1/$4: $5/$1/.git/HEAD -$5/$1/.git/HEAD: | $5/$1 -endif else # NO_GIT From 3b189b9a4e0ebd2578ae3df0ffd29b8d5d1f35bd Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 22 Sep 2015 10:28:43 -0500 Subject: [PATCH 0203/1938] Eliminate format warning for dump_compiles_stream (fix #13263) --- src/codegen.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 89351610e50c0..1db2fb5f47b4b 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -831,7 +831,7 @@ static Function *to_function(jl_lambda_info_t *li) JL_SIGATOMIC_END(); if (dump_compiles_stream != NULL) { uint64_t this_time = jl_hrtime(); - jl_printf(dump_compiles_stream, "%lu\t\"", this_time-last_time); + jl_printf(dump_compiles_stream, "%llu\t\"", (unsigned long long)(this_time-last_time)); jl_static_show(dump_compiles_stream, (jl_value_t*)li); jl_printf(dump_compiles_stream, "\"\n"); last_time = this_time; From adfd25c1dc859f29f7e1fa7d74de1bd7003a279a Mon Sep 17 00:00:00 2001 From: Keith Mason Date: Tue, 8 Sep 2015 10:34:02 -0500 Subject: [PATCH 0204/1938] add ability to write Rationals to a stream; return correct number of bytes written when writing Complex to a stream --- base/complex.jl | 3 +-- base/rational.jl | 9 +++++++++ test/iobuffer.jl | 2 ++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/base/complex.jl b/base/complex.jl index 28eb4a97704f9..99cc4d2cd29fd 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -75,8 +75,7 @@ function read{T<:Real}(s::IO, ::Type{Complex{T}}) Complex{T}(r,i) end function write(s::IO, z::Complex) - write(s,real(z)) - write(s,imag(z)) + write(s,real(z),imag(z)) end ## equality and hashing of complex numbers ## diff --git a/base/rational.jl b/base/rational.jl index f52906b6d84f8..0d044798ee213 100644 --- a/base/rational.jl +++ b/base/rational.jl @@ -57,6 +57,15 @@ function show(io::IO, x::Rational) show(io, den(x)) end +function read{T<:Integer}(s::IO, ::Type{Rational{T}}) + r = read(s,T) + i = read(s,T) + r//i +end +function write(s::IO, z::Rational) + write(s,num(z),den(z)) +end + convert{T<:Integer}(::Type{Rational{T}}, x::Rational) = Rational{T}(convert(T,x.num),convert(T,x.den)) convert{T<:Integer}(::Type{Rational{T}}, x::Integer) = Rational{T}(convert(T,x), convert(T,1)) diff --git a/test/iobuffer.jl b/test/iobuffer.jl index 3b801288f0966..36add0105c485 100644 --- a/test/iobuffer.jl +++ b/test/iobuffer.jl @@ -36,6 +36,8 @@ truncate(io, 0) @test write(io,"boston\ncambridge\n") > 0 @test takebuf_string(io) == "boston\ncambridge\n" @test takebuf_string(io) == "" +@test write(io, Complex{Float64}(0)) == 16 +@test write(io, Rational{Int64}(1//2)) == 16 close(io) @test_throws ArgumentError write(io,UInt8[0]) @test_throws ArgumentError seek(io,0) From f75170d9950bfaac737bff7d963139479ac6184a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiahao=20Chen=20=28=E9=99=88=E5=AE=B6=E8=B1=AA=29?= Date: Sat, 19 Sep 2015 00:15:59 -0400 Subject: [PATCH 0205/1938] perf: automate installation of go dependencies [ci skip] --- test/perf/micro/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/perf/micro/Makefile b/test/perf/micro/Makefile index ee9db271a0153..6b555c1c2c43e 100644 --- a/test/perf/micro/Makefile +++ b/test/perf/micro/Makefile @@ -78,6 +78,10 @@ benchmarks/fortran%.csv: bin/fperf% for t in 1 2 3 4 5; do $<; done >$@ benchmarks/go.csv: perf.go + go get github.com/gonum/blas + CGO_LDFLAGS="$(LIBBLAS) -lm" go install github.com/gonum/blas/cgo + go get github.com/gonum/matrix/mat64 + go get github.com/gonum/stat for t in 1 2 3 4 5; do go run $<; done >$@ benchmarks/julia.csv: perf.jl From 43d56965921fdd0c644246ee11e01a0f53b48ea7 Mon Sep 17 00:00:00 2001 From: Brendan Tracey Date: Mon, 14 Sep 2015 22:04:46 -0600 Subject: [PATCH 0206/1938] Update Go benchmarks to use the gonum suite Cherry-picked and squashed from commits: - 992e61d2d4f7f28b9a860b59a1d158325659f568 - 92d6879dfc31e68c724117f28ab6aeb637e85ade - a2b66acc2ae671556cc012728b9b55a68a178af2 Ref: #13133 Thanks @btracey --- test/perf/micro/perf.go | 174 ++++++++++++++++++++++++---------------- 1 file changed, 103 insertions(+), 71 deletions(-) diff --git a/test/perf/micro/perf.go b/test/perf/micro/perf.go index bde15e46d1a74..13414a6795838 100644 --- a/test/perf/micro/perf.go +++ b/test/perf/micro/perf.go @@ -1,24 +1,45 @@ -//The go.matrix and GoStats packages must be installed -//To install the go.matrix package, run: -// go get github.com/skelterjohn/go.matrix -// go install github.com/skelterjohn/go.matrix -//To install the GoStats package, run: -// go get github.com/GaryBoone/GoStats/stats -// go install github.com/GaryBoone/GoStats/stats - +// Implementation of the Julia benchmark suite in Go. +// +// Three gonum packages must be installed, and then an additional environmental +// variable must be set to use the BLAS installation. +// To install the gonum packages, run: +// go get github.com/gonum/blas +// go get github.com/gonum/matrix/mat64 +// go get github.com/gonum/stat +// The cgo ldflags must then be set to use the BLAS implementation. As an example, +// download OpenBLAS to ~/software +// git clone https://github.com/xianyi/OpenBLAS +// cd OpenBLAS +// make +// Then edit the enivorment variable to have +// export CGO_LDFLAGS="-L/$HOME/software/OpenBLAS -lopenblas" package main import ( "fmt" - matrix "github.com/skelterjohn/go.matrix" - stats "github.com/GaryBoone/GoStats/stats" "math" "math/cmplx" "math/rand" "strconv" "time" + + "github.com/gonum/blas/blas64" + "github.com/gonum/blas/cgo" + "github.com/gonum/matrix/mat64" + "github.com/gonum/stat" ) +func init() { + // Use the BLAS implementation specified in CGO_LDFLAGS. This line can be + // commented out to use the native Go BLAS implementation found in + // github.com/gonum/blas/native. + blas64.Use(cgo.Implementation{}) + + // These are here so that toggling the BLAS implementation does not make imports unused + _ = cgo.Implementation{} + _ = blas64.General{} +} + // fibonacci func fib(n int) int { @@ -61,63 +82,68 @@ func qsort_kernel(a []float64, lo, hi int) []float64 { func randmatstat(t int) (float64, float64) { n := 5 - var v stats.Stats - var w stats.Stats - for i :=0; i= 0) + assert(c.At(0, 0) >= 0) d := float64(time.Since(t).Seconds()) if d < tmin { tmin = d From dd6afa2e5567e1befbcb8322c124c4a3f74e8415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiahao=20Chen=20=28=E9=99=88=E5=AE=B6=E8=B1=AA=29?= Date: Sat, 19 Sep 2015 01:12:37 -0400 Subject: [PATCH 0207/1938] mandel: Force gcc 4.9.2 to not optimize away the loop at -O1 -O2 (HT @jameson) [ci skip] --- test/perf/micro/perf.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/perf/micro/perf.c b/test/perf/micro/perf.c index e9ad4a14bd35c..7d764fccd13ce 100644 --- a/test/perf/micro/perf.c +++ b/test/perf/micro/perf.c @@ -69,7 +69,11 @@ int mandel(double complex z) { } int mandelperf() { - int mandel_sum = 0; + /* The initialization on the next two lines is deliberately written to + * prevent gcc from optimizing away the entire loop. + * (First observed in gcc 4.9.2) */ + static volatile int mandel_sum_init = 0; + int mandel_sum = mandel_sum_init; for (int re=-20; re<=5; re+=1) { for (int im=-10; im<=10; im+=1) { int m = mandel(re/10.0+I*im/10.0); From 117f4673b3aed16f7b81056cc8d5c5008b3b7c97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiahao=20Chen=20=28=E9=99=88=E5=AE=B6=E8=B1=AA=29?= Date: Sat, 19 Sep 2015 01:51:06 -0400 Subject: [PATCH 0208/1938] Fix version detection logic for Julia and Fortran --- test/perf/micro/bin/table.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/perf/micro/bin/table.pl b/test/perf/micro/bin/table.pl index 930e29f0c2cde..d5c3ba075de9a 100755 --- a/test/perf/micro/bin/table.pl +++ b/test/perf/micro/bin/table.pl @@ -19,8 +19,8 @@ rand_mat_mul ); -our $julia_ver = `julia -v | cut -f3 -d" "`; -our $fortran_ver = `gfortran-4.8 -v 2>&1 | grep "gcc version" | cut -f3 -d" "`; +our $julia_ver = `../../../julia -v | cut -f3 -d" "`; +our $fortran_ver = `gfortran -v 2>&1 | grep "gcc version" | cut -f3 -d" "`; our $python_ver = `python -V 2>&1 | cut -f2 -d" "`; our $matlab_ver = `matlab -nodisplay -nojvm -nosplash -r "version -release, quit" | tail -n 3 | head -n 1`; our $R_ver = `R --version | grep "R version" | cut -f3 -d" "`; From 4e83f6e63b5145bb34d15e3715de2e20f0ab0f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiahao=20Chen=20=28=E9=99=88=E5=AE=B6=E8=B1=AA=29?= Date: Sat, 19 Sep 2015 22:32:07 -0400 Subject: [PATCH 0209/1938] Matlab perf: Modifications for R2015b 1. Matlab has completely revamped their JIT for R2015b. To avoid unfairly counting JIT time, run perf twice but record only the second set of times. 2. Setting maxNumCompThreads has been deprecated for awhile and does not seem to be respected consistently by Matlab. Instead, pass the -singleCompThread command line option. [ci skip] --- test/perf/micro/Makefile | 2 +- test/perf/micro/perf.m | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/test/perf/micro/Makefile b/test/perf/micro/Makefile index 6b555c1c2c43e..10070e0f517da 100644 --- a/test/perf/micro/Makefile +++ b/test/perf/micro/Makefile @@ -91,7 +91,7 @@ benchmarks/python.csv: perf.py for t in 1 2 3 4 5; do python $<; done >$@ benchmarks/matlab.csv: perf.m - for t in 1 2 3 4 5; do matlab -nosplash -nodesktop -nojvm -r 'perf;exit' 2>/dev/null | grep '^matlab,'; done >$@ + for t in 1 2 3 4 5; do matlab -nojvm -singleCompThread -r 'perf; perf; exit' | grep ^matlab | tail -8; done >$@ benchmarks/octave.csv: perf.m for t in 1 2 3 4 5; do octave -q --eval perf 2>/dev/null; done >$@ diff --git a/test/perf/micro/perf.m b/test/perf/micro/perf.m index 6c6a846f7b931..884f08cf24f88 100644 --- a/test/perf/micro/perf.m +++ b/test/perf/micro/perf.m @@ -6,9 +6,6 @@ function perf() warning off; - if exist('OCTAVE_VERSION') == 0 - maxNumCompThreads(1); - end f = fib(20); assert(f == 6765) From 13a385c0a7ce6c5e98106540d7c094d92cba3381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiahao=20Chen=20=28=E9=99=88=E5=AE=B6=E8=B1=AA=29?= Date: Sun, 20 Sep 2015 02:04:52 -0400 Subject: [PATCH 0210/1938] perf: add Python 3 [ci skip] --- test/perf/micro/Makefile | 7 +++++-- test/perf/micro/bin/table.pl | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/test/perf/micro/Makefile b/test/perf/micro/Makefile index 10070e0f517da..5de0873390c83 100644 --- a/test/perf/micro/Makefile +++ b/test/perf/micro/Makefile @@ -19,10 +19,9 @@ LIBBLAS=$(BLASDIR)libopenblas.a endif FFLAGS=-fexternal-blas +#gfortran cannot multiply matrices using 64-bit external BLAS. ifeq ($(findstring gfortran, $(FC)), gfortran) ifeq ($(USE_BLAS64), 1) -#gfortran cannot multiply matrices using 64-bit external BLAS. -#External BLAS usage is turned OFF. FFLAGS= endif FFLAGS+= -static-libgfortran @@ -90,6 +89,9 @@ benchmarks/julia.csv: perf.jl benchmarks/python.csv: perf.py for t in 1 2 3 4 5; do python $<; done >$@ +benchmarks/python3.csv: perf.py + for t in 1 2 3 4 5; do python3 $<; done | sed 's/^python/python3/g' >$@ + benchmarks/matlab.csv: perf.m for t in 1 2 3 4 5; do matlab -nojvm -singleCompThread -r 'perf; perf; exit' | grep ^matlab | tail -8; done >$@ @@ -120,6 +122,7 @@ BENCHMARKS = \ benchmarks/go.csv \ benchmarks/julia.csv \ benchmarks/python.csv \ + benchmarks/python3.csv \ benchmarks/matlab.csv \ benchmarks/octave.csv \ benchmarks/r.csv \ diff --git a/test/perf/micro/bin/table.pl b/test/perf/micro/bin/table.pl index d5c3ba075de9a..476fdd45f933d 100755 --- a/test/perf/micro/bin/table.pl +++ b/test/perf/micro/bin/table.pl @@ -22,6 +22,7 @@ our $julia_ver = `../../../julia -v | cut -f3 -d" "`; our $fortran_ver = `gfortran -v 2>&1 | grep "gcc version" | cut -f3 -d" "`; our $python_ver = `python -V 2>&1 | cut -f2 -d" "`; +our $python3_ver = `python3 -V 2>&1 | cut -f2 -d" "`; our $matlab_ver = `matlab -nodisplay -nojvm -nosplash -r "version -release, quit" | tail -n 3 | head -n 1`; our $R_ver = `R --version | grep "R version" | cut -f3 -d" "`; our $octave_ver = `octave -v | grep version | cut -f4 -d" "`; @@ -36,6 +37,7 @@ "fortran" => ["Fortran" , "gcc $fortran_ver" ], "julia" => ["Julia" , $julia_ver ], "python" => ["Python" , $python_ver ], + "python3" => ["Python" , $python3_ver], "matlab" => ["Matlab" , "R$matlab_ver" ], "octave" => ["Octave" , $octave_ver ], "r" => ["R" , $R_ver ], @@ -47,7 +49,7 @@ "java" => ["Java" , $java_ver ], ); -our @systems = qw(fortran julia python r matlab octave mathematica javascript go lua java); +our @systems = qw(fortran julia python python3 r matlab octave mathematica javascript go lua java); print qq[\n]; print qq[\n]; From 1f5a746a65b703acfdce7dcc0dcb99c8bf530537 Mon Sep 17 00:00:00 2001 From: Brendan Tracey Date: Mon, 21 Sep 2015 06:57:16 -0600 Subject: [PATCH 0211/1938] Update Go benchmarks to use the Go testing harness --- test/perf/micro/perf.go | 212 +++++++++++++++++++++------------------- 1 file changed, 109 insertions(+), 103 deletions(-) diff --git a/test/perf/micro/perf.go b/test/perf/micro/perf.go index 13414a6795838..69b002a950205 100644 --- a/test/perf/micro/perf.go +++ b/test/perf/micro/perf.go @@ -21,7 +21,7 @@ import ( "math/cmplx" "math/rand" "strconv" - "time" + "testing" "github.com/gonum/blas/blas64" "github.com/gonum/blas/cgo" @@ -78,6 +78,8 @@ func qsort_kernel(a []float64, lo, hi int) []float64 { return a } +var rnd = rand.New(rand.NewSource(1)) + // randmatstat func randmatstat(t int) (float64, float64) { @@ -94,10 +96,10 @@ func randmatstat(t int) (float64, float64) { qTmp := mat64.NewDense(2*n, 2*n, nil) for i := 0; i < t; i++ { for i := range ad { - ad[i] = rand.NormFloat64() - bd[i] = rand.NormFloat64() - cd[i] = rand.NormFloat64() - dd[i] = rand.NormFloat64() + ad[i] = rnd.NormFloat64() + bd[i] = rnd.NormFloat64() + cd[i] = rnd.NormFloat64() + dd[i] = rnd.NormFloat64() } a := mat64.NewDense(n, n, ad) b := mat64.NewDense(n, n, bd) @@ -132,13 +134,13 @@ func randmatstat(t int) (float64, float64) { func randmatmul(n int) *mat64.Dense { aData := make([]float64, n*n) for i := range aData { - aData[i] = rand.Float64() + aData[i] = rnd.Float64() } a := mat64.NewDense(n, n, aData) bData := make([]float64, n*n) for i := range bData { - bData[i] = rand.Float64() + bData[i] = rnd.Float64() } b := mat64.NewDense(n, n, bData) var c mat64.Dense @@ -194,106 +196,110 @@ func print_perf(name string, time float64) { // run tests -func assert(b bool) { - if b != true { - panic("assert failed") +func assert(b *testing.B, t bool) { + if t != true { + b.Fatal("assert failed") } } func main() { - assert(fib(20) == 6765) - tmin := float64(math.MaxFloat64) - for i := 0; i < 5; i++ { - t := time.Now() - _ = fib(20) - d := float64(time.Since(t).Seconds()) - if d < tmin { - tmin = d - } - } - print_perf("fib", tmin) - - tmin = float64(math.MaxFloat64) - for i := 0; i < 5; i++ { - t := time.Now() - var m uint64 - var n uint32 - for k := 0; k < 1000; k++ { - n = rand.Uint32() - s := fmt.Sprintf("%x", n) - m, _ = strconv.ParseUint(s, 16, 32) - } - assert(uint32(m) == n) - d := float64(time.Since(t).Seconds()) - if d < tmin { - tmin = d - } - } - print_perf("parse_int", tmin) - - assert(mandelperf() == 14791) - tmin = float64(math.MaxFloat64) - for i := 0; i < 5; i++ { - t := time.Now() - _ = mandelperf() - d := float64(time.Since(t).Seconds()) - if d < tmin { - tmin = d - } + for _, bm := range benchmarks { + print_perf(bm.name, runBenchmarkFor(bm.fn)) } - print_perf("mandel", tmin) +} - tmin = float64(math.MaxFloat64) - for i := 0; i < 5; i++ { - lst := make([]float64, 5000) - for k := 0; k < len(lst); k++ { - lst[k] = rand.Float64() - } - t := time.Now() - qsort_kernel(lst, 0, len(lst)-1) - d := float64(time.Since(t).Seconds()) - if d < tmin { - tmin = d - } - } - print_perf("quicksort", tmin) - - assert(math.Abs(pisum()-1.644834071848065) < 1e-6) - tmin = float64(math.MaxFloat64) - for i := 0; i < 5; i++ { - t := time.Now() - pisum() - d := float64(time.Since(t).Seconds()) - if d < tmin { - tmin = d - } - } - print_perf("pi_sum", tmin) - - tmin = float64(math.MaxFloat64) - for i := 0; i < 5; i++ { - t := time.Now() - c1, c2 := randmatstat(1000) - assert(0.5 < c1) - assert(c1 < 1.0) - assert(0.5 < c2) - assert(c2 < 1.0) - d := float64(time.Since(t).Seconds()) - if d < tmin { - tmin = d - } - } - print_perf("rand_mat_stat", tmin) - - tmin = float64(math.MaxFloat64) - for i := 0; i < 5; i++ { - t := time.Now() - c := randmatmul(1000) - assert(c.At(0, 0) >= 0) - d := float64(time.Since(t).Seconds()) - if d < tmin { - tmin = d - } - } - print_perf("rand_mat_mul", tmin) +func runBenchmarkFor(fn func(*testing.B)) (seconds float64) { + bm := testing.Benchmark(fn) + return bm.T.Seconds() / float64(bm.N) +} + +var benchmarks = []struct { + name string + fn func(*testing.B) +}{ + { + name: "fib", + fn: func(b *testing.B) { + for i := 0; i < b.N; i++ { + if fib(20) != 6765 { + b.Fatal("unexpected value for fib(20)") + } + } + }, + }, + + { + name: "parse_int", + fn: func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + for k := 0; k < 1000; k++ { + n := rnd.Uint32() + m, _ := strconv.ParseUint(fmt.Sprintf("%x", n), 16, 32) + if uint32(m) != n { + b.Fatal("incorrect value for m") + } + } + } + }, + }, + + { + name: "mandel", + fn: func(b *testing.B) { + for i := 0; i < b.N; i++ { + if mandelperf() != 14791 { + b.Fatal("unexpected value for mandelperf") + } + } + }, + }, + + { + name: "quicksort", + fn: func(b *testing.B) { + lst := make([]float64, 5000) + for k := 0; k < len(lst); k++ { + lst[k] = rand.Float64() + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + qsort_kernel(lst, 0, len(lst)-1) + } + }, + }, + + { + name: "pi_sum", + fn: func(b *testing.B) { + for i := 0; i < b.N; i++ { + if math.Abs(pisum()-1.644834071848065) >= 1e-6 { + b.Fatal("pi_sum out of range") + } + } + }, + }, + + { + name: "rand_mat_stat", + fn: func(b *testing.B) { + for i := 0; i < b.N; i++ { + c1, c2 := randmatstat(1000) + assert(b, 0.5 < c1) + assert(b, c1 < 1.0) + assert(b, 0.5 < c2) + assert(b, c2 < 1.0) + } + }, + }, + + { + name: "rand_mat_mul", + fn: func(b *testing.B) { + for i := 0; i < b.N; i++ { + c := randmatmul(1000) + assert(b, c.At(0, 0) >= 0) + } + }, + }, } From dfe6893b9c54ef24613c93e20aad6ad69c73cac5 Mon Sep 17 00:00:00 2001 From: Brendan Tracey Date: Mon, 21 Sep 2015 08:44:09 -0600 Subject: [PATCH 0212/1938] Move random number generation inside quicksort loop --- test/perf/micro/perf.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/perf/micro/perf.go b/test/perf/micro/perf.go index 69b002a950205..ab65097653e4e 100644 --- a/test/perf/micro/perf.go +++ b/test/perf/micro/perf.go @@ -259,11 +259,11 @@ var benchmarks = []struct { name: "quicksort", fn: func(b *testing.B) { lst := make([]float64, 5000) - for k := 0; k < len(lst); k++ { - lst[k] = rand.Float64() - } b.ResetTimer() for i := 0; i < b.N; i++ { + for k := range lst { + lst[k] = rnd.Float64() + } qsort_kernel(lst, 0, len(lst)-1) } }, From 2e6bd10e77b603254b1f57b318b7082b2411d6a1 Mon Sep 17 00:00:00 2001 From: Brendan Tracey Date: Mon, 21 Sep 2015 09:31:16 -0600 Subject: [PATCH 0213/1938] Delete unnecessary ResetTimer call --- test/perf/micro/perf.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/perf/micro/perf.go b/test/perf/micro/perf.go index ab65097653e4e..104f8ad526bf5 100644 --- a/test/perf/micro/perf.go +++ b/test/perf/micro/perf.go @@ -231,7 +231,6 @@ var benchmarks = []struct { { name: "parse_int", fn: func(b *testing.B) { - b.ResetTimer() for i := 0; i < b.N; i++ { for k := 0; k < 1000; k++ { n := rnd.Uint32() From 1f85b0b03a1e358cdfa4603f881bf202c726e6f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiahao=20Chen=20=28=E9=99=88=E5=AE=B6=E8=B1=AA=29?= Date: Sun, 20 Sep 2015 02:04:52 -0400 Subject: [PATCH 0214/1938] perf: Switch to Python 3 default [ci skip] --- test/perf/micro/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/perf/micro/Makefile b/test/perf/micro/Makefile index 5de0873390c83..d60ab034bfa97 100644 --- a/test/perf/micro/Makefile +++ b/test/perf/micro/Makefile @@ -4,6 +4,9 @@ include $(JULIAHOME)/deps/Versions.make NODEJSBIN = nodejs +#Use python2 for Python 2.x +PYTHON = python3 + ifeq ($(OS), WINNT) MATHEMATICABIN = MathKernel else ifeq ($(OS), Darwin) @@ -87,7 +90,7 @@ benchmarks/julia.csv: perf.jl for t in 1 2 3 4 5; do ../../../julia $<; done >$@ benchmarks/python.csv: perf.py - for t in 1 2 3 4 5; do python $<; done >$@ + for t in 1 2 3 4 5; do $(PYTHON) $<; done >$@ benchmarks/python3.csv: perf.py for t in 1 2 3 4 5; do python3 $<; done | sed 's/^python/python3/g' >$@ From 7d19b52d3bb35c05d43d9fcaffe2103629e55805 Mon Sep 17 00:00:00 2001 From: kortschak Date: Tue, 22 Sep 2015 16:18:28 +0930 Subject: [PATCH 0215/1938] micro: improve bench, fix cgo build and isolate * Use .so instead of .a, * Use local GOPATH to prevent contaminating client GOPATH, * Use strconv.FormatUint instead of fmt.Sprintf, * Make noise for bench errors. --- test/perf/micro/Makefile | 6 +++--- test/perf/micro/gopath/.gitignore | 2 ++ test/perf/micro/perf.go | 17 +++++++++++++---- 3 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 test/perf/micro/gopath/.gitignore diff --git a/test/perf/micro/Makefile b/test/perf/micro/Makefile index d60ab034bfa97..66075a182135f 100644 --- a/test/perf/micro/Makefile +++ b/test/perf/micro/Makefile @@ -79,9 +79,9 @@ benchmarks/c%.csv: bin/perf% benchmarks/fortran%.csv: bin/fperf% for t in 1 2 3 4 5; do $<; done >$@ +benchmarks/go.csv: export GOPATH=$(abspath gopath) benchmarks/go.csv: perf.go - go get github.com/gonum/blas - CGO_LDFLAGS="$(LIBBLAS) -lm" go install github.com/gonum/blas/cgo + CGO_LDFLAGS="-L$(JULIAHOME)/deps -lopenblas" go get github.com/gonum/blas/cgo go get github.com/gonum/matrix/mat64 go get github.com/gonum/stat for t in 1 2 3 4 5; do go run $<; done >$@ @@ -141,6 +141,6 @@ benchmarks.html: bin/table.pl benchmarks.csv @$(call PRINT_PERL, $^ >$@) clean: - @rm -rf perf.h bin/perf* bin/fperf* benchmarks/*.csv benchmarks.csv mods *~ octave-core perf.log + @rm -rf perf.h bin/perf* bin/fperf* benchmarks/*.csv benchmarks.csv mods *~ octave-core perf.log gopath/* .PHONY: all perf clean diff --git a/test/perf/micro/gopath/.gitignore b/test/perf/micro/gopath/.gitignore new file mode 100644 index 0000000000000..3815753530477 --- /dev/null +++ b/test/perf/micro/gopath/.gitignore @@ -0,0 +1,2 @@ +/pkg +/src \ No newline at end of file diff --git a/test/perf/micro/perf.go b/test/perf/micro/perf.go index 104f8ad526bf5..201fb8b12a5ee 100644 --- a/test/perf/micro/perf.go +++ b/test/perf/micro/perf.go @@ -16,7 +16,9 @@ package main import ( + "errors" "fmt" + "log" "math" "math/cmplx" "math/rand" @@ -204,13 +206,20 @@ func assert(b *testing.B, t bool) { func main() { for _, bm := range benchmarks { - print_perf(bm.name, runBenchmarkFor(bm.fn)) + seconds, err := runBenchmarkFor(bm.fn) + if err != nil { + log.Fatalf("%s %s", bm.name, err) + } + print_perf(bm.name, seconds) } } -func runBenchmarkFor(fn func(*testing.B)) (seconds float64) { +func runBenchmarkFor(fn func(*testing.B)) (seconds float64, err error) { bm := testing.Benchmark(fn) - return bm.T.Seconds() / float64(bm.N) + if (bm == testing.BenchmarkResult{}) { + return 0, errors.New("failed") + } + return bm.T.Seconds() / float64(bm.N), nil } var benchmarks = []struct { @@ -234,7 +243,7 @@ var benchmarks = []struct { for i := 0; i < b.N; i++ { for k := 0; k < 1000; k++ { n := rnd.Uint32() - m, _ := strconv.ParseUint(fmt.Sprintf("%x", n), 16, 32) + m, _ := strconv.ParseUint(strconv.FormatUint(uint64(n), 16), 16, 32) if uint32(m) != n { b.Fatal("incorrect value for m") } From 97d147bcff8a57501e3769a757d406adede223f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiahao=20Chen=20=28=E9=99=88=E5=AE=B6=E8=B1=AA=29?= Date: Sun, 20 Sep 2015 02:04:52 -0400 Subject: [PATCH 0216/1938] Cleanup after merging Mainly reapplies perf: Switch to Python 3 default [ci skip] --- test/perf/micro/Makefile | 4 ---- test/perf/micro/bin/table.pl | 6 ++---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/test/perf/micro/Makefile b/test/perf/micro/Makefile index 66075a182135f..76a2f8137bb99 100644 --- a/test/perf/micro/Makefile +++ b/test/perf/micro/Makefile @@ -92,9 +92,6 @@ benchmarks/julia.csv: perf.jl benchmarks/python.csv: perf.py for t in 1 2 3 4 5; do $(PYTHON) $<; done >$@ -benchmarks/python3.csv: perf.py - for t in 1 2 3 4 5; do python3 $<; done | sed 's/^python/python3/g' >$@ - benchmarks/matlab.csv: perf.m for t in 1 2 3 4 5; do matlab -nojvm -singleCompThread -r 'perf; perf; exit' | grep ^matlab | tail -8; done >$@ @@ -125,7 +122,6 @@ BENCHMARKS = \ benchmarks/go.csv \ benchmarks/julia.csv \ benchmarks/python.csv \ - benchmarks/python3.csv \ benchmarks/matlab.csv \ benchmarks/octave.csv \ benchmarks/r.csv \ diff --git a/test/perf/micro/bin/table.pl b/test/perf/micro/bin/table.pl index 476fdd45f933d..e3ba93b1799ff 100755 --- a/test/perf/micro/bin/table.pl +++ b/test/perf/micro/bin/table.pl @@ -21,8 +21,7 @@ our $julia_ver = `../../../julia -v | cut -f3 -d" "`; our $fortran_ver = `gfortran -v 2>&1 | grep "gcc version" | cut -f3 -d" "`; -our $python_ver = `python -V 2>&1 | cut -f2 -d" "`; -our $python3_ver = `python3 -V 2>&1 | cut -f2 -d" "`; +our $python_ver = `python3 -V 2>&1 | cut -f2 -d" "`; our $matlab_ver = `matlab -nodisplay -nojvm -nosplash -r "version -release, quit" | tail -n 3 | head -n 1`; our $R_ver = `R --version | grep "R version" | cut -f3 -d" "`; our $octave_ver = `octave -v | grep version | cut -f4 -d" "`; @@ -37,7 +36,6 @@ "fortran" => ["Fortran" , "gcc $fortran_ver" ], "julia" => ["Julia" , $julia_ver ], "python" => ["Python" , $python_ver ], - "python3" => ["Python" , $python3_ver], "matlab" => ["Matlab" , "R$matlab_ver" ], "octave" => ["Octave" , $octave_ver ], "r" => ["R" , $R_ver ], @@ -49,7 +47,7 @@ "java" => ["Java" , $java_ver ], ); -our @systems = qw(fortran julia python python3 r matlab octave mathematica javascript go lua java); +our @systems = qw(fortran julia python r matlab octave mathematica javascript go lua java); print qq[
\n]; print qq[\n]; From d7d3c1c78e40979d2ee9549fecb061279aa3279f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiahao=20Chen=20=28=E9=99=88=E5=AE=B6=E8=B1=AA=29?= Date: Fri, 18 Sep 2015 23:26:54 -0400 Subject: [PATCH 0217/1938] microbenchmarks: Fix new location of dependencies --- test/perf/micro/Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/perf/micro/Makefile b/test/perf/micro/Makefile index 76a2f8137bb99..3147e23a89350 100644 --- a/test/perf/micro/Makefile +++ b/test/perf/micro/Makefile @@ -17,7 +17,7 @@ endif #Which BLAS library am I using? ifeq ($(USE_SYSTEM_BLAS), 0) -BLASDIR=$(JULIAHOME)/deps/openblas-$(OPENBLAS_VER)/ +BLASDIR=$(JULIAHOME)/deps/build/openblas/ LIBBLAS=$(BLASDIR)libopenblas.a endif @@ -31,15 +31,15 @@ FFLAGS+= -static-libgfortran endif #Which libm library am I using? -LIBMDIR = $(JULIAHOME)/deps/openlibm/ +LIBMDIR = $(JULIAHOME)/deps/build/openlibm/ ifeq ($(USE_SYSTEM_LIBM), 0) ifeq ($(USE_SYSTEM_OPENLIBM), 0) LIBM = $(LIBMDIR)libopenlibm.a endif endif -DSFMTDIR = $(JULIAHOME)/deps/dsfmt-$(DSFMT_VER) -RMATHDIR = $(JULIAHOME)/deps/Rmath-julia-$(RMATH_JULIA_VER) +DSFMTDIR = $(JULIAHOME)/deps/build/dsfmt-$(DSFMT_VER) +RMATHDIR = $(JULIAHOME)/deps/build/Rmath-julia-$(RMATH_JULIA_VER) default: benchmarks.html @@ -81,7 +81,7 @@ benchmarks/fortran%.csv: bin/fperf% benchmarks/go.csv: export GOPATH=$(abspath gopath) benchmarks/go.csv: perf.go - CGO_LDFLAGS="-L$(JULIAHOME)/deps -lopenblas" go get github.com/gonum/blas/cgo + CGO_LDFLAGS="-L$(BLASDIR) -lopenblas" go get github.com/gonum/blas/cgo go get github.com/gonum/matrix/mat64 go get github.com/gonum/stat for t in 1 2 3 4 5; do go run $<; done >$@ From be103e0174c477fe0709bc11450761a9d32a86ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiahao=20Chen=20=28=E9=99=88=E5=AE=B6=E8=B1=AA=29?= Date: Tue, 22 Sep 2015 17:48:07 -0400 Subject: [PATCH 0218/1938] Fix up Go benchmark dependencies Revert to statically linked libraries Make sure that the locally checked out gopath does not persist after `make clean`. Add ./gopath to local .gitignore --- test/perf/micro/.gitignore | 1 + test/perf/micro/Makefile | 2 +- test/perf/micro/gopath/.gitignore | 2 -- 3 files changed, 2 insertions(+), 3 deletions(-) delete mode 100644 test/perf/micro/gopath/.gitignore diff --git a/test/perf/micro/.gitignore b/test/perf/micro/.gitignore index bae59a651e9bd..19727ca0afb5a 100644 --- a/test/perf/micro/.gitignore +++ b/test/perf/micro/.gitignore @@ -2,4 +2,5 @@ /benchmarks.csv /benchmarks.txt /benchmarks.html +gopath mods/* diff --git a/test/perf/micro/Makefile b/test/perf/micro/Makefile index 3147e23a89350..ae7b240e8529a 100644 --- a/test/perf/micro/Makefile +++ b/test/perf/micro/Makefile @@ -81,7 +81,7 @@ benchmarks/fortran%.csv: bin/fperf% benchmarks/go.csv: export GOPATH=$(abspath gopath) benchmarks/go.csv: perf.go - CGO_LDFLAGS="-L$(BLASDIR) -lopenblas" go get github.com/gonum/blas/cgo + CGO_LDFLAGS="$(LIBBLAS) $(LIBM)" go get github.com/gonum/blas/cgo go get github.com/gonum/matrix/mat64 go get github.com/gonum/stat for t in 1 2 3 4 5; do go run $<; done >$@ diff --git a/test/perf/micro/gopath/.gitignore b/test/perf/micro/gopath/.gitignore deleted file mode 100644 index 3815753530477..0000000000000 --- a/test/perf/micro/gopath/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/pkg -/src \ No newline at end of file From ec8b8891bd9d32eca01b8d89620177ec492c42b0 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 22 Sep 2015 19:23:49 -0400 Subject: [PATCH 0219/1938] Fix get-libuv and get-osxunwind targets --- deps/Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index 5b89213a10e0b..aebfd9d74456b 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -284,6 +284,8 @@ $5/$1/$3: $$(SRCDIR)/$1.version | $5/$1/.git/HEAD @[ '$$($2_SHA1)' = "$$$$(cd $5/$1 && git show -s --format='%H' HEAD)" ] || echo $$(WARNCOLOR)'==> warning: SHA1 hash did not match $1.version file'$$(ENDCOLOR) touch -c $$@ $5/$1/$4: $5/$1/.git/HEAD +$$($2_SRC_FILE): | $$($2_SRC_FILE)/HEAD + touch -c $$@ else # NO_GIT @@ -1764,8 +1766,8 @@ distclean-osxunwind: $(BUILDDIR)/libosxunwind-$(OSXUNWIND_VER) -get-osxunwind: libosxunwind-$(OSXUNWIND_VER)/Makefile -configure-osxunwind: get-osxunwind +get-osxunwind: $(SRCDIR)/srccache/libosxunwind-$(OSXUNWIND_VER).tar.gz +configure-osxunwind: $(BUILDDIR)/libosxunwind-$(OSXUNWIND_VER)/Makefile compile-osxunwind: $(OSXUNWIND_OBJ_SOURCE) check-osxunwind: compile-osxunwind install-osxunwind: $(OSXUNWIND_OBJ_TARGET) From b939592f2a4da9275ac7db9a83fc4a49ae3449af Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Tue, 22 Sep 2015 18:52:06 -0700 Subject: [PATCH 0220/1938] 4-space indent in basedocs Replace a few tabs with spaces found via: grep -n -P '\t' */*.c* */*.h* src/*/*.c* src/*/*.h* */*.jl */*/*.jl */*/*/*.jl --- base/docs/basedocs.jl | 12 ++++++------ base/docs/helpdb.jl | 4 ++-- src/cgutils.cpp | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/base/docs/basedocs.jl b/base/docs/basedocs.jl index 859cba1ac73b5..61bb24616e753 100644 --- a/base/docs/basedocs.jl +++ b/base/docs/basedocs.jl @@ -137,12 +137,12 @@ keywords[:local] = doc""" `local` introduces a new local variable. For example: function foo(n) - x = 0 - for i = 1:n - local x - x = i - end - x + x = 0 + for i = 1:n + local x + x = i + end + x end julia> foo(10) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index f7319bd2669e7..e562757f4a91f 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -8030,8 +8030,8 @@ Determine whether the given generic function has a method matching the given :ob .. doctest:: - julia> method_exists(length, Tuple{Array}) - true + julia> method_exists(length, Tuple{Array}) + true ``` """ method_exists diff --git a/src/cgutils.cpp b/src/cgutils.cpp index a01bb9cd1df09..ee71e8bda810d 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1606,7 +1606,7 @@ static jl_value_t *static_constant_instance(Constant *constant, jl_value_t *jt) assert(constant != NULL); if (isa(constant)) - return NULL; + return NULL; ConstantInt *cint = dyn_cast(constant); if (cint != NULL) { From efa12d71db25071a1b6f14d23375295b4b91487c Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Tue, 22 Sep 2015 19:14:33 -0400 Subject: [PATCH 0221/1938] adds try...catch in testset and reports errors instead of bailing --- base/test.jl | 14 ++++++++++++-- test/test.jl | 7 +++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/base/test.jl b/base/test.jl index e74e695c2601d..77b1927eb39f6 100644 --- a/base/test.jl +++ b/base/test.jl @@ -110,7 +110,11 @@ function Base.show(io::IO, t::Error) print( io, " Value: ", t.value) elseif t.test_type == :test_error println(io, " Test threw an exception of type ", typeof(t.value)) - println(io, " Expression: ", t.orig_expr) + # don't print an empty expression. Probably from an error thrown in + # a @testset and we don't have the expression for it + if t.orig_expr != :() + println(io, " Expression: ", t.orig_expr) + end # Capture error message and indent to match errmsg = sprint(showerror, t.value, t.backtrace) print(io, join(map(line->string(" ",line), @@ -487,7 +491,13 @@ macro testset(args...) quote $ts = DefaultTestSet($desc) add_testset($ts) - $(esc(tests)) + try + $(esc(tests)) + catch err + # something in the test block threw an error. Count that as an + # error in this test set + record($ts, Error(:test_error, :(), err, catch_backtrace())) + end pop_testset() finish($ts) end diff --git a/test/test.jl b/test/test.jl index bfd2aba90dcfc..8fa9dee8cbfb2 100644 --- a/test/test.jl +++ b/test/test.jl @@ -58,6 +58,9 @@ try @test "not bool" @test error() end + + error("exceptions in testsets should be caught") + @test 1 == 1 # this test will not be run end @testset "loop with desc" begin @@ -89,7 +92,7 @@ catch ex @test isa(ex, Test.TestSetException) @test ex.pass == 21 @test ex.fail == 5 - @test ex.error == 2 + @test ex.error == 3 end # Test @test_approx_eq @@ -101,4 +104,4 @@ end @test_throws ErrorException Test.test_approx_eq(ones(10),zeros(10),1e-8,"a","b") # Test @test_approx_eq_eps -# TODO \ No newline at end of file +# TODO From 6b749dd561998bb4153ecd4423cb5dfd1b8e4a88 Mon Sep 17 00:00:00 2001 From: Michael Hatherly Date: Wed, 23 Sep 2015 08:17:21 +0200 Subject: [PATCH 0222/1938] Move docsystem to end of sysimg.jl Fixes #13274. Having the docsystem defined last should avoid the inconsistencies in available doc syntax in different parts of `Base`. --- base/docs/bootstrap.jl | 2 +- base/sysimg.jl | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/base/docs/bootstrap.jl b/base/docs/bootstrap.jl index 7084219bc9596..293e417c5e8a8 100644 --- a/base/docs/bootstrap.jl +++ b/base/docs/bootstrap.jl @@ -19,7 +19,7 @@ setexpand!(f) = global _expand_ = f setexpand!() do str, obj global docs = List((ccall(:jl_get_current_module, Any, ()), str, obj), docs) - return esc(Expr(:toplevel, obj)) + (isa(obj, Expr) && obj.head == :call) ? nothing : esc(Expr(:toplevel, obj)) end """ diff --git a/base/sysimg.jl b/base/sysimg.jl index f352779f71bbb..90e6e066fc285 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -234,13 +234,6 @@ include("REPLCompletions.jl") include("REPL.jl") include("client.jl") -# Documentation - -include("markdown/Markdown.jl") -include("docs/Docs.jl") -using .Docs -using .Markdown - # misc useful functions & macros include("util.jl") @@ -292,6 +285,13 @@ importall .Profile include("Dates.jl") import .Dates: Date, DateTime, now +# Documentation + +include("markdown/Markdown.jl") +include("docs/Docs.jl") +using .Docs +using .Markdown + # deprecated functions include("deprecated.jl") From 62a00d2d5f1524f8b555f75d04a67b0a61c9e1ce Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Wed, 23 Sep 2015 12:23:02 +0530 Subject: [PATCH 0223/1938] edit sync write faq entry [ci skip] --- doc/manual/faq.rst | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/doc/manual/faq.rst b/doc/manual/faq.rst index 06023948b336d..3e9e8d49174bf 100644 --- a/doc/manual/faq.rst +++ b/doc/manual/faq.rst @@ -871,37 +871,35 @@ is fully asynchronous. The following:: @sync for i in 1:3 - @async print(i, " Foo ", " Bar ") + @async write(STDOUT, string(i), " Foo ", " Bar ") end results in:: 123 Foo Foo Foo Bar Bar Bar -This is happening because, while ``print(i, " Foo ", " Bar ")`` is synchronous, -internally, the writing of each argument yields to other tasks while waiting for -that part of the I/O to complete. +This is happening because, while the ``write`` call is synchronous, the writing of +each argument yields to other tasks while waiting for that part of the I/O to complete. -``println`` to asynchronous streams like STDOUT, TcpSockets, "locks" the stream -during a call. Consequently changing ``print`` to ``println`` in the above example -results in:: +``print`` and ``println`` "lock" the stream during a call. Consequently changing ``write`` to +``println`` in the above example results in:: 1 Foo Bar 2 Foo Bar 3 Foo Bar -For other functions and streams, etc, you could lock your writes with a ``ReentrantLock`` -like this:: +You can lock your writes with a ``ReentrantLock`` like this:: l = ReentrantLock() @sync for i in 1:3 @async begin lock(l) try - print(i, " Foo ", " Bar ") + write(STDOUT, string(i), " Foo ", " Bar ") finally unlock(l) end + end end From e5b3cc654b192e905bf5916a7cdcb5035f18ff6a Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Mon, 21 Sep 2015 09:08:06 -0400 Subject: [PATCH 0224/1938] Widen setindex_shape_check method signature This removes the restriction on setindex_shape_check that the index lengths must all be Int, allowing data structures to return non-Int lengths (e.g., `UInt(1):UInt(2)` or `big(1):big(2)`). Fixes #13250. --- base/operators.jl | 12 ++++++------ test/arrayops.jl | 7 +++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/base/operators.jl b/base/operators.jl index ad7e66213b9b3..10e5a5982abef 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -252,7 +252,7 @@ end # for permutations that leave array elements in the same linear order. # those are the permutations that preserve the order of the non-singleton # dimensions. -function setindex_shape_check(X::AbstractArray, I::Int...) +function setindex_shape_check(X::AbstractArray, I...) li = ndims(X) lj = length(I) i = j = 1 @@ -289,16 +289,16 @@ end setindex_shape_check(X::AbstractArray) = (length(X)==1 || throw_setindex_mismatch(X,())) -setindex_shape_check(X::AbstractArray, i::Int) = +setindex_shape_check(X::AbstractArray, i) = (length(X)==i || throw_setindex_mismatch(X, (i,))) -setindex_shape_check{T}(X::AbstractArray{T,1}, i::Int) = +setindex_shape_check{T}(X::AbstractArray{T,1}, i) = (length(X)==i || throw_setindex_mismatch(X, (i,))) -setindex_shape_check{T}(X::AbstractArray{T,1}, i::Int, j::Int) = +setindex_shape_check{T}(X::AbstractArray{T,1}, i, j) = (length(X)==i*j || throw_setindex_mismatch(X, (i,j))) -function setindex_shape_check{T}(X::AbstractArray{T,2}, i::Int, j::Int) +function setindex_shape_check{T}(X::AbstractArray{T,2}, i, j) if length(X) != i*j throw_setindex_mismatch(X, (i,j)) end @@ -307,7 +307,7 @@ function setindex_shape_check{T}(X::AbstractArray{T,2}, i::Int, j::Int) throw_setindex_mismatch(X, (i,j)) end end -setindex_shape_check(X, I::Int...) = nothing # Non-arrays broadcast to all idxs +setindex_shape_check(X, I...) = nothing # Non-arrays broadcast to all idxs # convert to a supported index type (Array, Colon, or Int) to_index(i::Int) = i diff --git a/test/arrayops.jl b/test/arrayops.jl index 20e239ab380bc..392938a19d6ed 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1340,3 +1340,10 @@ copy!(B, A) @test A + 1 == B + 1 @test 2*A == 2*B @test A/3 == B/3 + +# issue #13250 +x13250 = zeros(3) +x13250[UInt(1):UInt(2)] = 1.0 +@test x13250[1] == 1.0 +@test x13250[2] == 1.0 +@test x13250[3] == 0.0 From 4a55d382798038fb353b56c1cb847a1282ddbdd9 Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Wed, 23 Sep 2015 11:20:28 -0400 Subject: [PATCH 0225/1938] fix spacing issue in generated docs --- doc/stdlib/base.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 931f92c2d2b0d..89e9e7c615b86 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -630,8 +630,8 @@ Generic Functions .. doctest:: - julia> method_exists(length, Tuple{Array}) - true + julia> method_exists(length, Tuple{Array}) + true .. function:: applicable(f, args...) -> Bool From bc1d9a5561e1991587172f2a1d3d7a0284883f31 Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Wed, 23 Sep 2015 12:45:52 -0400 Subject: [PATCH 0226/1938] now using separate error type for floating errors --- base/test.jl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/base/test.jl b/base/test.jl index 77b1927eb39f6..d77048c562982 100644 --- a/base/test.jl +++ b/base/test.jl @@ -110,11 +110,14 @@ function Base.show(io::IO, t::Error) print( io, " Value: ", t.value) elseif t.test_type == :test_error println(io, " Test threw an exception of type ", typeof(t.value)) - # don't print an empty expression. Probably from an error thrown in - # a @testset and we don't have the expression for it - if t.orig_expr != :() - println(io, " Expression: ", t.orig_expr) - end + println(io, " Expression: ", t.orig_expr) + # Capture error message and indent to match + errmsg = sprint(showerror, t.value, t.backtrace) + print(io, join(map(line->string(" ",line), + split(errmsg, "\n")), "\n")) + elseif t.test_type == :nontest_error + # we had an error outside of a @test + println(io, " Got an exception of type $(typeof(t.value)) outside of a @test") # Capture error message and indent to match errmsg = sprint(showerror, t.value, t.backtrace) print(io, join(map(line->string(" ",line), @@ -496,7 +499,7 @@ macro testset(args...) catch err # something in the test block threw an error. Count that as an # error in this test set - record($ts, Error(:test_error, :(), err, catch_backtrace())) + record($ts, Error(:nontest_error, :(), err, catch_backtrace())) end pop_testset() finish($ts) From 0903653415304d5c975e5c11baac1d82b39ac0ca Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Wed, 23 Sep 2015 13:36:33 -0400 Subject: [PATCH 0227/1938] Use tbaa_immut for getfield of immutable types --- src/cgutils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index ee71e8bda810d..3ddcf7a8d6653 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1361,7 +1361,7 @@ static jl_cgval_t emit_getfield_knownidx(const jl_cgval_t &strct, unsigned idx, LLVM37_param(cast(strct.V->getType()->getScalarType())->getElementType()) strct.V, 0, idx); assert(!jt->mutabl); - return typed_load(addr, NULL, jfty, ctx, NULL); + return typed_load(addr, NULL, jfty, ctx, tbaa_immut); } else { assert(strct.V->getType()->isVectorTy()); From 21c6dd73dd4c0f52bd5e953b213c5084afccc634 Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Wed, 23 Sep 2015 13:37:55 -0400 Subject: [PATCH 0228/1938] adds try...catch block to testloop as well --- base/test.jl | 8 +++++++- test/test.jl | 9 +++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/base/test.jl b/base/test.jl index d77048c562982..22fb41d99da9f 100644 --- a/base/test.jl +++ b/base/test.jl @@ -555,7 +555,13 @@ macro testloop(args...) blk = quote $ts = DefaultTestSet($(esc(desc))) add_testset($ts) - $(esc(tests)) + try + $(esc(tests)) + catch err + # something in the test block threw an error. Count that as an + # error in this test set + record($ts, Error(:nontest_error, :(), err, catch_backtrace())) + end pop_testset() finish($ts) end diff --git a/test/test.jl b/test/test.jl index 8fa9dee8cbfb2..67822f048ef7f 100644 --- a/test/test.jl +++ b/test/test.jl @@ -81,6 +81,11 @@ try @testloop for i in 1:5 @test i <= rand(1:10) end + # should add 3 errors and 3 passing tests + @testloop for i in 1:6 + i % 2 == 0 || error("error outside of test") + @test true # only gets run if the above passed + end end end # These lines shouldn't be called @@ -90,9 +95,9 @@ catch ex redirect_stdout(OLD_STDOUT) @test isa(ex, Test.TestSetException) - @test ex.pass == 21 + @test ex.pass == 24 @test ex.fail == 5 - @test ex.error == 3 + @test ex.error == 6 end # Test @test_approx_eq From 52e242e697432d4176006866c46f2475234856f0 Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Wed, 23 Sep 2015 16:07:52 -0400 Subject: [PATCH 0229/1938] Fix eigs documentation and move it to the source code --- base/docs/helpdb.jl | 47 ---------------------- base/linalg/arnoldi.jl | 88 ++++++++++++++++++++++++++++++++++++++++++ doc/stdlib/linalg.rst | 48 +++++++++++++++++++++-- 3 files changed, 133 insertions(+), 50 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index e562757f4a91f..16a5e0f8edc22 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -3021,53 +3021,6 @@ Create a PipeBuffer to operate on a data vector, optionally specifying a size be """ PipeBuffer(data) -doc""" -```rst -.. eigs(A, [B,]; nev=6, which="LM", tol=0.0, maxiter=300, sigma=nothing, ritzvec=true, v0=zeros((0,))) -> (d,[v,],nconv,niter,nmult,resid) - -Computes eigenvalues ``d`` of ``A`` using Lanczos or Arnoldi iterations for -real symmetric or general nonsymmetric matrices respectively. If ``B`` is -provided, the generalized eigenproblem is solved. - -The following keyword arguments are supported: - * ``nev``: Number of eigenvalues - * ``ncv``: Number of Krylov vectors used in the computation; should satisfy ``nev+1 <= ncv <= n`` for real symmetric problems and ``nev+2 <= ncv <= n`` for other problems, where ``n`` is the size of the input matrix ``A``. The default is ``ncv = max(20,2*nev+1)``. - - Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. - * ``which``: type of eigenvalues to compute. See the note below. - - ========= ====================================================================================================================== - ``which`` type of eigenvalues - --------- ---------------------------------------------------------------------------------------------------------------------- - ``:LM`` eigenvalues of largest magnitude (default) - ``:SM`` eigenvalues of smallest magnitude - ``:LR`` eigenvalues of largest real part - ``:SR`` eigenvalues of smallest real part - ``:LI`` eigenvalues of largest imaginary part (nonsymmetric or complex ``A`` only) - ``:SI`` eigenvalues of smallest imaginary part (nonsymmetric or complex ``A`` only) - ``:BE`` compute half of the eigenvalues from each end of the spectrum, biased in favor of the high end. (real symmetric ``A`` only) - ========= ====================================================================================================================== - - * ``tol``: tolerance (:math:`tol \le 0.0` defaults to ``DLAMCH('EPS')``) - * ``maxiter``: Maximum number of iterations (default = 300) - * ``sigma``: Specifies the level shift used in inverse iteration. If ``nothing`` (default), defaults to ordinary (forward) iterations. Otherwise, find eigenvalues close to ``sigma`` using shift and invert iterations. - * ``ritzvec``: Returns the Ritz vectors ``v`` (eigenvectors) if ``true`` - * ``v0``: starting vector from which to start the iterations - -``eigs`` returns the ``nev`` requested eigenvalues in ``d``, the corresponding Ritz vectors ``v`` (only if ``ritzvec=true``), the number of converged eigenvalues ``nconv``, the number of iterations ``niter`` and the number of matrix vector multiplications ``nmult``, as well as the final residual vector ``resid``. - -.. note:: The ``sigma`` and ``which`` keywords interact: the description of eigenvalues searched for by ``which`` do _not_ necessarily refer to the eigenvalues of ``A``, but rather the linear operator constructed by the specification of the iteration mode implied by ``sigma``. - - =============== ================================== ================================== - ``sigma`` iteration mode ``which`` refers to eigenvalues of - --------------- ---------------------------------- ---------------------------------- - ``nothing`` ordinary (forward) :math:`A` - real or complex inverse with level shift ``sigma`` :math:`(A - \sigma I )^{-1}` - =============== ================================== ================================== -``` -""" -eigs - doc""" ```rst .. sortperm(v, [alg=,] [by=,] [lt=,] [rev=false]) diff --git a/base/linalg/arnoldi.jl b/base/linalg/arnoldi.jl index aab2d2d01c0d2..5cc69c167672c 100644 --- a/base/linalg/arnoldi.jl +++ b/base/linalg/arnoldi.jl @@ -3,9 +3,97 @@ using .ARPACK ## eigs +doc""" +```rst +.. eigs(A; nev=6, ncv=max(20,2*nev+1), which="LM", tol=0.0, maxiter=300, sigma=nothing, ritzvec=true, v0=zeros((0,))) -> (d,[v,],nconv,niter,nmult,resid) +Computes eigenvalues ``d`` of ``A`` using Lanczos or Arnoldi iterations for +real symmetric or general nonsymmetric matrices respectively. + +The following keyword arguments are supported: + * ``nev``: Number of eigenvalues + * ``ncv``: Number of Krylov vectors used in the computation; should satisfy ``nev+1 <= ncv <= n`` for real symmetric problems and ``nev+2 <= ncv <= n`` for other problems, where ``n`` is the size of the input matrix ``A``. The default is ``ncv = max(20,2*nev+1)``. + + Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. + * ``which``: type of eigenvalues to compute. See the note below. + + ========= ====================================================================================================================== + ``which`` type of eigenvalues + --------- ---------------------------------------------------------------------------------------------------------------------- + ``:LM`` eigenvalues of largest magnitude (default) + ``:SM`` eigenvalues of smallest magnitude + ``:LR`` eigenvalues of largest real part + ``:SR`` eigenvalues of smallest real part + ``:LI`` eigenvalues of largest imaginary part (nonsymmetric or complex ``A`` only) + ``:SI`` eigenvalues of smallest imaginary part (nonsymmetric or complex ``A`` only) + ``:BE`` compute half of the eigenvalues from each end of the spectrum, biased in favor of the high end. (real symmetric ``A`` only) + ========= ====================================================================================================================== + + * ``tol``: tolerance (:math:`tol \le 0.0` defaults to ``DLAMCH('EPS')``) + * ``maxiter``: Maximum number of iterations (default = 300) + * ``sigma``: Specifies the level shift used in inverse iteration. If ``nothing`` (default), defaults to ordinary (forward) iterations. Otherwise, find eigenvalues close to ``sigma`` using shift and invert iterations. + * ``ritzvec``: Returns the Ritz vectors ``v`` (eigenvectors) if ``true`` + * ``v0``: starting vector from which to start the iterations + +``eigs`` returns the ``nev`` requested eigenvalues in ``d``, the corresponding Ritz vectors ``v`` (only if ``ritzvec=true``), the number of converged eigenvalues ``nconv``, the number of iterations ``niter`` and the number of matrix vector multiplications ``nmult``, as well as the final residual vector ``resid``. + +.. note:: The ``sigma`` and ``which`` keywords interact: the description of eigenvalues searched for by ``which`` do _not_ necessarily refer to the eigenvalues of ``A``, but rather the linear operator constructed by the specification of the iteration mode implied by ``sigma``. + + =============== ================================== ================================== + ``sigma`` iteration mode ``which`` refers to eigenvalues of + --------------- ---------------------------------- ---------------------------------- + ``nothing`` ordinary (forward) :math:`A` + real or complex inverse with level shift ``sigma`` :math:`(A - \sigma I )^{-1}` + =============== ================================== ================================== +``` +""" eigs(A; args...) = eigs(A, I; args...) + +doc""" +```rst +.. eigs(A, B; nev=6, ncv=max(20,2*nev+1), which="LM", tol=0.0, maxiter=300, sigma=nothing, ritzvec=true, v0=zeros((0,))) -> (d,[v,],nconv,niter,nmult,resid) + +Computes generalized eigenvalues ``d`` of ``A`` and ``B`` using Lanczos or Arnoldi iterations for +real symmetric or general nonsymmetric matrices respectively. + +The following keyword arguments are supported: + * ``nev``: Number of eigenvalues + * ``ncv``: Number of Krylov vectors used in the computation; should satisfy ``nev+1 <= ncv <= n`` for real symmetric problems and ``nev+2 <= ncv <= n`` for other problems, where ``n`` is the size of the input matrices ``A`` and ``B``. The default is ``ncv = max(20,2*nev+1)``. + + Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. + * ``which``: type of eigenvalues to compute. See the note below. + + ========= ====================================================================================================================== + ``which`` type of eigenvalues + --------- ---------------------------------------------------------------------------------------------------------------------- + ``:LM`` eigenvalues of largest magnitude (default) + ``:SM`` eigenvalues of smallest magnitude + ``:LR`` eigenvalues of largest real part + ``:SR`` eigenvalues of smallest real part + ``:LI`` eigenvalues of largest imaginary part (nonsymmetric or complex ``A`` only) + ``:SI`` eigenvalues of smallest imaginary part (nonsymmetric or complex ``A`` only) + ``:BE`` compute half of the eigenvalues from each end of the spectrum, biased in favor of the high end. (real symmetric ``A`` only) + ========= ====================================================================================================================== + + * ``tol``: tolerance (:math:`tol \le 0.0` defaults to ``DLAMCH('EPS')``) + * ``maxiter``: Maximum number of iterations (default = 300) + * ``sigma``: Specifies the level shift used in inverse iteration. If ``nothing`` (default), defaults to ordinary (forward) iterations. Otherwise, find eigenvalues close to ``sigma`` using shift and invert iterations. + * ``ritzvec``: Returns the Ritz vectors ``v`` (eigenvectors) if ``true`` + * ``v0``: starting vector from which to start the iterations + +``eigs`` returns the ``nev`` requested eigenvalues in ``d``, the corresponding Ritz vectors ``v`` (only if ``ritzvec=true``), the number of converged eigenvalues ``nconv``, the number of iterations ``niter`` and the number of matrix vector multiplications ``nmult``, as well as the final residual vector ``resid``. + +.. note:: The ``sigma`` and ``which`` keywords interact: the description of eigenvalues searched for by ``which`` do _not_ necessarily refer to the eigenvalue problem :math:`Av = Bv\lambda`, but rather the linear operator constructed by the specification of the iteration mode implied by ``sigma``. + + =============== ================================== ================================== + ``sigma`` iteration mode ``which`` refers to the problem + --------------- ---------------------------------- ---------------------------------- + ``nothing`` ordinary (forward) :math:`Av = Bv\lambda` + real or complex inverse with level shift ``sigma`` :math:`(A - \sigma B )^{-1}B = v\nu` + =============== ================================== ================================== +``` +""" function eigs(A, B; nev::Integer=6, ncv::Integer=max(20,2*nev+1), which=:LM, tol=0.0, maxiter::Integer=300, sigma=nothing, v0::Vector=zeros(eltype(A),(0,)), diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 126e13dd99cae..1b803d235f46e 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -917,13 +917,12 @@ Linear algebra functions in Julia are largely implemented by calling functions f Conjugate transpose array ``src`` and store the result in the preallocated array ``dest``\ , which should have a size corresponding to ``(size(src,2),size(src,1))``\ . No in-place transposition is supported and unexpected results will happen if ``src`` and ``dest`` have overlapping memory regions. -.. function:: eigs(A, [B,]; nev=6, ncv=max(20,2*nev+1), which="LM", tol=0.0, maxiter=300, sigma=nothing, ritzvec=true, v0=zeros((0,))) -> (d,[v,],nconv,niter,nmult,resid) +.. function:: eigs(A; nev=6, ncv=max(20,2*nev+1), which="LM", tol=0.0, maxiter=300, sigma=nothing, ritzvec=true, v0=zeros((0,))) -> (d,[v,],nconv,niter,nmult,resid) .. Docstring generated from Julia source Computes eigenvalues ``d`` of ``A`` using Lanczos or Arnoldi iterations for - real symmetric or general nonsymmetric matrices respectively. If ``B`` is - provided, the generalized eigenproblem is solved. + real symmetric or general nonsymmetric matrices respectively. The following keyword arguments are supported: * ``nev``: Number of eigenvalues @@ -961,6 +960,49 @@ Linear algebra functions in Julia are largely implemented by calling functions f real or complex inverse with level shift ``sigma`` :math:`(A - \sigma I )^{-1}` =============== ================================== ================================== +.. function:: eigs(A, B; nev=6, ncv=max(20,2*nev+1), which="LM", tol=0.0, maxiter=300, sigma=nothing, ritzvec=true, v0=zeros((0,))) -> (d,[v,],nconv,niter,nmult,resid) + + .. Docstring generated from Julia source + + Computes generalized eigenvalues ``d`` of ``A`` and ``B`` using Lanczos or Arnoldi iterations for + real symmetric or general nonsymmetric matrices respectively. + + The following keyword arguments are supported: + * ``nev``: Number of eigenvalues + * ``ncv``: Number of Krylov vectors used in the computation; should satisfy ``nev+1 <= ncv <= n`` for real symmetric problems and ``nev+2 <= ncv <= n`` for other problems, where ``n`` is the size of the input matrices ``A`` and ``B``. The default is ``ncv = max(20,2*nev+1)``. + + Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. + * ``which``: type of eigenvalues to compute. See the note below. + + ========= ====================================================================================================================== + ``which`` type of eigenvalues + --------- ---------------------------------------------------------------------------------------------------------------------- + ``:LM`` eigenvalues of largest magnitude (default) + ``:SM`` eigenvalues of smallest magnitude + ``:LR`` eigenvalues of largest real part + ``:SR`` eigenvalues of smallest real part + ``:LI`` eigenvalues of largest imaginary part (nonsymmetric or complex ``A`` only) + ``:SI`` eigenvalues of smallest imaginary part (nonsymmetric or complex ``A`` only) + ``:BE`` compute half of the eigenvalues from each end of the spectrum, biased in favor of the high end. (real symmetric ``A`` only) + ========= ====================================================================================================================== + + * ``tol``: tolerance (:math:`tol \le 0.0` defaults to ``DLAMCH('EPS')``) + * ``maxiter``: Maximum number of iterations (default = 300) + * ``sigma``: Specifies the level shift used in inverse iteration. If ``nothing`` (default), defaults to ordinary (forward) iterations. Otherwise, find eigenvalues close to ``sigma`` using shift and invert iterations. + * ``ritzvec``: Returns the Ritz vectors ``v`` (eigenvectors) if ``true`` + * ``v0``: starting vector from which to start the iterations + + ``eigs`` returns the ``nev`` requested eigenvalues in ``d``, the corresponding Ritz vectors ``v`` (only if ``ritzvec=true``), the number of converged eigenvalues ``nconv``, the number of iterations ``niter`` and the number of matrix vector multiplications ``nmult``, as well as the final residual vector ``resid``. + + .. note:: The ``sigma`` and ``which`` keywords interact: the description of eigenvalues searched for by ``which`` do _not_ necessarily refer to the eigenvalue problem :math:`Av = Bv\lambda`, but rather the linear operator constructed by the specification of the iteration mode implied by ``sigma``. + + =============== ================================== ================================== + ``sigma`` iteration mode ``which`` refers to the problem + --------------- ---------------------------------- ---------------------------------- + ``nothing`` ordinary (forward) :math:`Av = Bv\lambda` + real or complex inverse with level shift ``sigma`` :math:`(A - \sigma B )^{-1}B = v\nu` + =============== ================================== ================================== + .. function:: svds(A; nsv=6, ritzvec=true, tol=0.0, maxiter=1000) -> (left_sv, s, right_sv, nconv, niter, nmult, resid) .. Docstring generated from Julia source From 670854353f1df77815f68d50814fd89e00eb914a Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Tue, 22 Sep 2015 19:17:43 -0400 Subject: [PATCH 0230/1938] Fix reflection macros when using splatting by fixing the splatting branch of gen_call_with_extracted_types --- base/interactiveutil.jl | 18 +++++++----------- test/reflection.jl | 3 +++ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 74dbed0b5de15..8a9ec7f13e0be 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -244,19 +244,15 @@ function gen_call_with_extracted_types(fcn, ex0) Expr(:call, typesof, map(esc, args[2:end])...)) end ex = expand(ex0) - if isa(ex, Expr) && ex.head == :call - return Expr(:call, fcn, esc(ex.args[1]), - Expr(:call, typesof, map(esc, ex.args[2:end])...)) - end - exret = Expr(:call, :error, "expression is not a function call or symbol") if !isa(ex, Expr) - # do nothing -> error + exret = Expr(:call, :error, "expression is not a function call or symbol") elseif ex.head == :call if any(e->(isa(e, Expr) && e.head==:(...)), ex0.args) && - isa(ex.args[1], TopNode) && ex.args[1].name == :apply - exret = Expr(:call, ex.args[1], fcn, - Expr(:tuple, esc(ex.args[2])), - Expr(:call, typesof, map(esc, ex.args[3:end])...)) + isa(ex.args[1], TopNode) && ex.args[1].name == :_apply + # check for splatting + exret = Expr(:call, ex.args[1], ex.args[2], fcn, + Expr(:tuple, esc(ex.args[3]), + Expr(:call, typesof, map(esc, ex.args[4:end])...))) else exret = Expr(:call, fcn, esc(ex.args[1]), Expr(:call, typesof, map(esc, ex.args[2:end])...)) @@ -272,7 +268,7 @@ function gen_call_with_extracted_types(fcn, ex0) end elseif ex.head == :thunk exret = Expr(:call, :error, "expression is not a function call, " - * "or is too complex for @which to analyze; " + * "or is too complex for @$fcn to analyze; " * "break it down to simpler parts if possible") end exret diff --git a/test/reflection.jl b/test/reflection.jl index 71772e0922199..2e2070e813aee 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -190,3 +190,6 @@ let end @test_throws ArgumentError which(is, Tuple{Int, Int}) + +# issue #13264 +@test isa((@which vcat(1...)), Method) From aca6c3d9f8f1bb7fc0de215ad9078917913cbe8c Mon Sep 17 00:00:00 2001 From: Andy Hayden Date: Wed, 23 Sep 2015 13:03:44 -0700 Subject: [PATCH 0231/1938] Document that write returns the number of bytes written --- base/docs/helpdb.jl | 7 +++++++ doc/manual/networking-and-streams.rst | 22 ++++++++++++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index e562757f4a91f..25ed8c4e75906 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -4927,6 +4927,13 @@ doc""" write(stream, x) Write the canonical binary representation of a value to the given stream. +Returns the number of bytes written into the stream. + +You can write multiple values with the same :func:`write` call. +i.e. the following are equivalent: + + write(stream, x, y...) + write(stream, x) + write(stream, y...) """ write diff --git a/doc/manual/networking-and-streams.rst b/doc/manual/networking-and-streams.rst index 82e534428304a..ed9cab6d7b7f3 100644 --- a/doc/manual/networking-and-streams.rst +++ b/doc/manual/networking-and-streams.rst @@ -16,18 +16,24 @@ functionality. Basic Stream I/O ---------------- -All Julia streams expose at least a :func:`read` and a :func:`write` method, taking the stream as their first argument, e.g.:: +All Julia streams expose at least a :func:`read` and a :func:`write` method, +taking the stream as their first argument, e.g.:: - julia> write(STDOUT,"Hello World") + julia> write(STDOUT,"Hello World"); # suppress return value 11 with ; Hello World julia> read(STDIN,Char) '\n' -Note that I pressed enter again so that Julia would read the newline. Now, as you can see from this example, +Note that :func:`write` returns 11, the number of bytes (in ``"Hello World"``) written to :const:`STDOUT`, +but this return value is suppressed with the ``;``. + +Here Enter was pressed again so that Julia would read the newline. Now, as you can see from this example, :func:`write` takes the data to write as its second argument, while :func:`read` takes the type of the -data to be read as the second argument. For example, to read a simple byte array, we could do:: +data to be read as the second argument. + +For example, to read a simple byte array, we could do:: julia> x = zeros(UInt8,4) 4-element Array{UInt8,1}: @@ -80,12 +86,16 @@ or :func:`read` if you wanted to read by character instead:: Text I/O -------- -Note that the write method mentioned above operates on binary streams. In particular, values do not get converted to any canonical text +Note that the :func:`write` method mentioned above operates on binary streams. +In particular, values do not get converted to any canonical text representation but are written out as is:: - julia> write(STDOUT,0x61) + julia> write(STDOUT,0x61); # suppress return value 1 with ; a +Note that ``a`` is written to :const:`STDOUT` by the :func:`write` function and +that the returned value is ``1`` (since ``0x61`` is one byte). + For text I/O, use the :func:`print` or :func:`show` methods, depending on your needs (see the standard library reference for a detailed discussion of the difference between the two):: From 31a0c5543e05095af26b0d4190d53a09c8efb7ed Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Tue, 22 Sep 2015 14:08:24 -0400 Subject: [PATCH 0232/1938] Make `@deprecate_binding` work with `--depwarn` cmdline flags --- src/module.c | 12 ++++++++++-- test/cmdlineargs.jl | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index fb5817f0f66e4..d535ba13a3f3d 100644 --- a/src/module.c +++ b/src/module.c @@ -482,10 +482,12 @@ DLLEXPORT int jl_is_binding_deprecated(jl_module_t *m, jl_sym_t *var) void jl_binding_deprecation_warning(jl_binding_t *b) { if (b->deprecated && jl_options.depwarn) { + if (jl_options.depwarn != JL_OPTIONS_DEPWARN_ERROR) + jl_printf(JL_STDERR, "WARNING: "); if (b->owner) - jl_printf(JL_STDERR, "WARNING: %s.%s is deprecated", b->owner->name->name, b->name->name); + jl_printf(JL_STDERR, "%s.%s is deprecated", b->owner->name->name, b->name->name); else - jl_printf(JL_STDERR, "WARNING: %s is deprecated", b->name->name); + jl_printf(JL_STDERR, "%s is deprecated", b->name->name); jl_value_t *v = b->value; if (v && (jl_is_type(v) || (jl_is_function(v) && jl_is_gf(v)))) { jl_printf(JL_STDERR, ", use "); @@ -493,6 +495,12 @@ void jl_binding_deprecation_warning(jl_binding_t *b) jl_printf(JL_STDERR, " instead"); } jl_printf(JL_STDERR, ".\n"); + if (jl_options.depwarn == JL_OPTIONS_DEPWARN_ERROR) { + if (b->owner) + jl_errorf("deprecated binding: %s.%s", b->owner->name->name, b->name->name); + else + jl_errorf("deprecated binding: %s", b->name->name); + } } } diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 664c25c4a0b88..46512b85c0def 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -152,6 +152,39 @@ let exename = `$(joinpath(JULIA_HOME, Base.julia_exename())) --precompiled=yes` foo() " --depwarn=error`) + # test deprecated bindings + let code = """ + module Foo + import Base: @deprecate_binding + + const NotDeprecated = true + @deprecate_binding Deprecated NotDeprecated + end + + Foo.Deprecated + """ + + @test !success(`$exename -E "$code" --depwarn=error`) + + @unix_only let out = Pipe(), + proc = spawn(pipeline(`$exename -E "$code" --depwarn=yes`, stderr=out)) + + wait(proc) + close(out.in) + @test success(proc) + @test readchomp(out) == "WARNING: Foo.Deprecated is deprecated." + end + + @unix_only let out = Pipe(), + proc = spawn(pipeline(`$exename -E "$code" --depwarn=no`, stderr=out)) + + wait(proc) + close(out.in) + @test success(proc) + @test isempty(readall(out)) + end + end + # --inline @test readchomp(`$exename -E "Bool(Base.JLOptions().can_inline)"`) == "true" @test readchomp(`$exename --inline=yes -E "Bool(Base.JLOptions().can_inline)"`) == "true" From 699dd2dabde3f50fc91fc750bc445fcb8411701c Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 23 Sep 2015 23:30:20 -0400 Subject: [PATCH 0233/1938] Cache Emitted Symbol Table in Function Mover Turns out MCJIT does this in O(M) in the number of modules. However, since we do this for EVERY SINGLE call instruction we emit during bootstrap, that ends up quite hot. This commit gives about a 15% improvement in sysimg build time. --- src/cgutils.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index ee71e8bda810d..0d9996ed4d826 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -156,6 +156,10 @@ typedef struct {Value* gv; int32_t index;} jl_value_llvm; // uses 1-based indexi static std::map jl_value_to_llvm; DLLEXPORT std::map jl_llvm_to_jl_value; +// In imaging mode, cache a fast mapping of Function * to code address +// because this is queried in the hot path +static std::map emitted_function_symtab; + #ifdef USE_MCJIT class FunctionMover : public ValueMaterializer { @@ -233,16 +237,18 @@ class FunctionMover : public ValueMaterializer if (shadow != NULL && !shadow->isDeclaration()) { // Not truly external // Check whether we already emitted it once + if (emitted_function_symtab.find(shadow) != emitted_function_symtab.end()) + return destModule->getOrInsertFunction(F->getName(),F->getFunctionType()); uint64_t addr = jl_mcjmm->getSymbolAddress(F->getName()); - if (addr == 0) { - Function *oldF = destModule->getFunction(F->getName()); - if (oldF) - return oldF; - return CloneFunctionProto(F); - } - else { + if (addr) { + emitted_function_symtab[shadow] = addr; return destModule->getOrInsertFunction(F->getName(),F->getFunctionType()); } + + Function *oldF = destModule->getFunction(F->getName()); + if (oldF) + return oldF; + return CloneFunctionProto(F); } else if (!F->isDeclaration()) { return CloneFunctionProto(F); From 890c006dda4d5ab6b62b5bb8720d79b20e04be9a Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Wed, 23 Sep 2015 22:54:11 -0700 Subject: [PATCH 0234/1938] Add FIXME comment --- test/cmdlineargs.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 46512b85c0def..2ed49d782c316 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -152,7 +152,7 @@ let exename = `$(joinpath(JULIA_HOME, Base.julia_exename())) --precompiled=yes` foo() " --depwarn=error`) - # test deprecated bindings + # test deprecated bindings, #13269 let code = """ module Foo import Base: @deprecate_binding @@ -166,6 +166,7 @@ let exename = `$(joinpath(JULIA_HOME, Base.julia_exename())) --precompiled=yes` @test !success(`$exename -E "$code" --depwarn=error`) + # FIXME these should also be run on windows once the bug causing them to hang gets fixed @unix_only let out = Pipe(), proc = spawn(pipeline(`$exename -E "$code" --depwarn=yes`, stderr=out)) From 37f8943ced95ca16638784ae17ab8d099616b5eb Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sat, 25 Apr 2015 18:57:30 -0400 Subject: [PATCH 0235/1938] fixed missing IPv6 binding added `getsockname` for determining a socket name (address & port) of the running server fixed corresponding docimentation added `uv_tcp_getpeername` to `getsockname` for getting the address and the port of the peer connected to the TCP server socket fixed generalizing `bind`, Union() => Union{} fixed AF_INET6 for OSX added docs --- base/docs/helpdb.jl | 15 +++++-- base/exports.jl | 1 + base/managers.jl | 5 +-- base/socket.jl | 88 ++++++++++++++++++++++++--------------- doc/stdlib/base.rst | 5 ++- doc/stdlib/io-network.rst | 13 ++++-- src/jl_uv.c | 57 +++++++++++++++++++------ test/socket.jl | 53 +++++++++++++++++++++++ 8 files changed, 178 insertions(+), 59 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index e562757f4a91f..5cbea53b7f91e 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -7463,9 +7463,10 @@ true all(p, itr) doc""" - bind(socket::Union{UDPSocket, TCPSocket}, host::IPv4, port::Integer) + bind(socket::Union{UDPSocket, TCPSocket}, host::IPAddr, port::Integer; ipv6only=false) Bind `socket` to the given `host:port`. Note that `0.0.0.0` will listen on all devices. +`ipv6only` parameter disables dual stack mode. If it's `true`, only IPv6 stack is created. """ bind @@ -8474,9 +8475,9 @@ Get the current working directory. pwd doc""" - getipaddr() -> AbstractString + getipaddr() -> IPAddr -Get the IP address of the local machine, as a string of the form "x.x.x.x". +Get the IP address of the local machine. """ getipaddr @@ -12031,3 +12032,11 @@ Bitwise exclusive or ``` """ Base.(:$)(x, y) + +doc""" + getsockname(sock::Union{TCPServer, TCPSocket}) -> (IPAddr,UInt16) + +Get the IP address and the port of the TCP server socket, to which it is bound, +or the peer connected to the TCP server socket. +""" +getsockname \ No newline at end of file diff --git a/base/exports.jl b/base/exports.jl index cfc1574f57046..6c572b1c04865 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1152,6 +1152,7 @@ export getaddrinfo, gethostname, getipaddr, + getsockname, htol, hton, ismarked, diff --git a/base/managers.jl b/base/managers.jl index 5102eecb70097..b02aac2e59770 100644 --- a/base/managers.jl +++ b/base/managers.jl @@ -313,10 +313,7 @@ function socket_reuse_port() elseif rc < 0 throw(SystemError("setsockopt() SO_REUSEPORT : ")) end - - ccall(:jl_tcp_getsockname_v4, Int32, - (Ptr{Void}, Ref{Cuint}, Ref{Cushort}), - s.handle, client_host, client_port) < 0 && throw(SystemError("getsockname() : ")) + getsockname(s) catch e # This is an issue only on systems with lots of client connections, hence delay the warning.... nworkers() > 128 && warn_once("Error trying to reuse client port number, falling back to plain socket : ", e) diff --git a/base/socket.jl b/base/socket.jl index 8c5491fe86a66..8a38c586bb958 100644 --- a/base/socket.jl +++ b/base/socket.jl @@ -343,16 +343,6 @@ accept(server::TCPServer) = accept(server, TCPSocket()) accept(server::PipeServer) = accept(server, init_pipe!(PipeEndpoint(); readable=false, writable=false, julia_only=true)) -## - -bind(sock::TCPServer, addr::InetAddr) = bind(sock,addr.host,addr.port) - -_bind(sock::TCPServer, host::IPv4, port::UInt16) = ccall(:jl_tcp_bind, Int32, (Ptr{Void}, UInt16, UInt32, Cuint), - sock.handle, hton(port), hton(host.host), 0) - -_bind(sock::TCPServer, host::IPv6, port::UInt16) = ccall(:jl_tcp_bind6, Int32, (Ptr{Void}, UInt16, Ptr{UInt128}, Cuint), - sock.handle, hton(port), &hton(host.host), 0) - # UDP type UDPSocket <: LibuvStream @@ -402,6 +392,9 @@ function _uv_hook_close(sock::UDPSocket) notify_error(sock.recvnotify,EOFError()) end +# Disables dual stack mode. +const UV_TCP_IPV6ONLY = 1 + # Disables dual stack mode. Only available when using ipv6 binf const UV_UDP_IPV6ONLY = 1 @@ -409,36 +402,32 @@ const UV_UDP_IPV6ONLY = 1 # remainder was discarded by the OS. const UV_UDP_PARTIAL = 2 -function bind(sock::Union{TCPServer,UDPSocket}, host::IPv4, port::Integer) - if sock.status != StatusInit - error("$(typeof(sock)) is not initialized") - end - err = _bind(sock,host,UInt16(port)) - if err < 0 - if err != UV_EADDRINUSE && err != UV_EACCES - #TODO: this codepath is not currently tested - throw(UVError("bind",err)) - else - return false - end - end - sock.status = StatusOpen - true -end +## + +_bind(sock::TCPServer, host::IPv4, port::UInt16, flags::UInt32 = UInt32(0)) = ccall(:jl_tcp_bind, Int32, (Ptr{Void}, UInt16, UInt32, Cuint), + sock.handle, hton(port), hton(host.host), flags) -_bind(sock::UDPSocket, host::IPv4, port::UInt16) = ccall(:jl_udp_bind, Int32, (Ptr{Void}, UInt16, UInt32, UInt32), - sock.handle, hton(port), hton(host.host), 0) +_bind(sock::TCPServer, host::IPv6, port::UInt16, flags::UInt32 = UInt32(0)) = ccall(:jl_tcp_bind6, Int32, (Ptr{Void}, UInt16, Ptr{UInt128}, Cuint), + sock.handle, hton(port), Ref(hton(host.host)), flags) + +_bind(sock::UDPSocket, host::IPv4, port::UInt16, flags::UInt32 = UInt32(0)) = ccall(:jl_udp_bind, Int32, (Ptr{Void}, UInt16, UInt32, UInt32), + sock.handle, hton(port), hton(host.host), flags) _bind(sock::UDPSocket, host::IPv6, port::UInt16, flags::UInt32 = UInt32(0)) = ccall(:jl_udp_bind6, Int32, (Ptr{Void}, UInt16, Ptr{UInt128}, UInt32), - sock.handle, hton(port), &hton(host.host), flags) + sock.handle, hton(port), Ref(hton(host.host)), flags) -function bind(sock::UDPSocket, host::IPv6, port::UInt16; ipv6only = false) +function bind(sock::Union{TCPServer, UDPSocket}, host::IPAddr, port::Integer; ipv6only = false) if sock.status != StatusInit - error("UDPSocket is not initialized") + error("$(typeof(sock)) is not initialized") + end + flags = if isa(host,IPv6) && ipv6only + isa(sock, UDPSocket) ? UV_UDP_IPV6ONLY : UV_TCP_IPV6ONLY + else + 0 end - err = _bind(sock,host,port, UInt32(ipv6only ? UV_UDP_IPV6ONLY : 0)) + err = _bind(sock,host,UInt16(port),UInt32(flags)) if err < 0 - if err != UV_EADDRINUSE && err != UV_EACCES + if err != UV_EADDRINUSE && err != UV_EACCES && err != UV_EADDRNOTAVAIL #TODO: this codepath is not currently tested throw(UVError("bind",err)) else @@ -449,6 +438,7 @@ function bind(sock::UDPSocket, host::IPv6, port::UInt16; ipv6only = false) true end +bind(sock::TCPServer, addr::InetAddr) = bind(sock,addr.host,addr.port) function setopt(sock::UDPSocket; multicast_loop = nothing, multicast_ttl=nothing, enable_broadcast=nothing, ttl=nothing) if sock.status == StatusUninit @@ -732,3 +722,35 @@ function listenany(default_port) end end end + +function getsockname(sock::Union{TCPServer,TCPSocket}) + rport = Ref{Cushort}(0) + raddress = zeros(UInt8, 16) + rfamily = Ref{Cuint}(0) + r = if isa(sock, TCPServer) + ccall(:jl_tcp_getsockname, Int32, + (Ptr{Void}, Ref{Cushort}, Ptr{Void}, Ref{Cuint}), + sock.handle, rport, raddress, rfamily) + else + ccall(:jl_tcp_getpeername, Int32, + (Ptr{Void}, Ref{Cushort}, Ptr{Void}, Ref{Cuint}), + sock.handle, rport, raddress, rfamily) + end + uv_error("cannot obtain socket name", r); + if r == 0 + port = ntoh(rport[]) + if rfamily[] == 2 # AF_INET + addrv4 = raddress[1:4] + naddr = ntoh(unsafe_load(Ptr{Cuint}(pointer(addrv4)), 1)) + addr = IPv4(naddr) + elseif rfamily[] == @windows? 23 : (@osx? 30 : 10) # AF_INET6 + naddr = ntoh(unsafe_load(Ptr{UInt128}(pointer(raddress)), 1)) + addr = IPv6(naddr) + else + error("unsupported address family: $(getindex(rfamily))") + end + else + error("cannot obtain socket name") + end + return addr, port +end diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 89e9e7c615b86..804f73a785ca9 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -893,11 +893,12 @@ System Get the local machine's host name. -.. function:: getipaddr() -> AbstractString +.. function:: getipaddr() .. Docstring generated from Julia source - Get the IP address of the local machine, as a string of the form "x.x.x.x". + Get the IP address of the local machine. + .. function:: getpid() -> Int32 diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index f273319517e74..a098371589575 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -766,7 +766,7 @@ Memory-mapped I/O Network I/O ----------- -.. function:: connect([host],port) -> TcpSocket +.. function:: connect([host],port) -> TCPSocket .. Docstring generated from Julia source @@ -778,7 +778,7 @@ Network I/O Connect to the Named Pipe / Domain Socket at ``path`` -.. function:: listen([addr,]port) -> TcpServer +.. function:: listen([addr,]port) -> TCPServer .. Docstring generated from Julia source @@ -796,6 +796,11 @@ Network I/O Gets the IP address of the ``host`` (may have to do a DNS lookup) +.. function:: getsockname(sock::Union{TCPServer, TCPSocket}) -> (IPAddr,UInt16) + + Get the IP address and the port of the TCP server socket, to which it is bound, + or the peer connected to the TCP server socket. + .. function:: parseip(addr) .. Docstring generated from Julia source @@ -826,11 +831,11 @@ Network I/O Accepts a connection on the given server and returns a connection to the client. An uninitialized client stream may be provided, in which case it will be used instead of creating a new stream. -.. function:: listenany(port_hint) -> (UInt16,TcpServer) +.. function:: listenany(port_hint) -> (UInt16,TCPServer) .. Docstring generated from Julia source - Create a TcpServer on any port, using hint as a starting point. Returns a tuple of the actual port that the server was created on and the server itself. + Create a TCPServer on any port, using hint as a starting point. Returns a tuple of the actual port that the server was created on and the server itself. .. function:: poll_fd(fd, timeout_s::Real; readable=false, writable=false) diff --git a/src/jl_uv.c b/src/jl_uv.c index 2cb47ac708b9a..e4dd947f6e4a7 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -507,6 +507,50 @@ DLLEXPORT int jl_tcp_bind6(uv_tcp_t *handle, uint16_t port, void *host, unsigned return uv_tcp_bind(handle, (struct sockaddr*)&addr, flags); } +DLLEXPORT int jl_tcp_getsockname(uv_tcp_t *handle, uint16_t* port, void* host, uint32_t* family) +{ + int namelen; + struct sockaddr_storage addr; + memset(&addr, 0, sizeof(struct sockaddr_storage)); + namelen = sizeof addr; + int res = uv_tcp_getsockname(handle, (struct sockaddr*)&addr, &namelen); + *family = addr.ss_family; + if (addr.ss_family == AF_INET) { + struct sockaddr_in* addr4 = (struct sockaddr_in*)&addr; + *port = addr4->sin_port; + memcpy(host, &(addr4->sin_addr), 4); + } else if (addr.ss_family == AF_INET6) { + struct sockaddr_in6* addr6 = (struct sockaddr_in6*)&addr; + *port = addr6->sin6_port; + memcpy(host, &(addr6->sin6_addr), 16); + } else { + return -1; + } + return res; +} + +DLLEXPORT int jl_tcp_getpeername(uv_tcp_t *handle, uint16_t* port, void* host, uint32_t* family) +{ + int namelen; + struct sockaddr_storage addr; + memset(&addr, 0, sizeof(struct sockaddr_storage)); + namelen = sizeof addr; + int res = uv_tcp_getpeername(handle, (struct sockaddr*)&addr, &namelen); + *family = addr.ss_family; + if (addr.ss_family == AF_INET) { + struct sockaddr_in* addr4 = (struct sockaddr_in*)&addr; + *port = addr4->sin_port; + memcpy(host, &(addr4->sin_addr), 4); + } else if (addr.ss_family == AF_INET6) { + struct sockaddr_in6* addr6 = (struct sockaddr_in6*)&addr; + *port = addr6->sin6_port; + memcpy(host, &(addr6->sin6_addr), 16); + } else { + return -1; + } + return res; +} + DLLEXPORT int jl_udp_bind(uv_udp_t *handle, uint16_t port, uint32_t host, uint32_t flags) { struct sockaddr_in addr; @@ -699,19 +743,6 @@ DLLEXPORT int jl_tcp_reuseport(uv_tcp_t *handle) #endif } -DLLEXPORT int jl_tcp_getsockname_v4(uv_tcp_t *handle, uint32_t * ip, uint16_t * port) -{ - struct sockaddr_in name; - int len = sizeof(name); - if (uv_tcp_getsockname(handle, (struct sockaddr *)&name, &len)) { - return -1; - } - - *ip = ntohl(name.sin_addr.s_addr); - *port = ntohs(name.sin_port); - return 0; -} - #ifndef _OS_WINDOWS_ DLLEXPORT int jl_uv_unix_fd_is_watched(int fd, uv_poll_t *handle, uv_loop_t *loop) diff --git a/test/socket.jl b/test/socket.jl index 1810dfac1f1d7..f31e3b5f54424 100644 --- a/test/socket.jl +++ b/test/socket.jl @@ -185,3 +185,56 @@ if @unix? true : (Base.windows_version() >= Base.WINDOWS_VISTA_VER) send(b, ip"::1", port, "Hello World") wait(tsk) end + +begin + default_port = UInt16(11011) + default_addr = IPv4("127.0.0.1") + + sock = Base.TCPServer() + bind(sock,Base.InetAddr(default_addr,default_port)) + listen(sock) + + new_addr, new_port = getsockname(sock) + + @test default_addr == new_addr + @test default_port == new_port + close(sock) +end + +begin + default_port = UInt16(21011) + default_addr = IPv6("::1") + + sock = Base.TCPServer() + addr = Base.InetAddr(default_addr,default_port) + bind(sock,addr) + listen(sock) + + new_addr, new_port = getsockname(sock) + + @test default_addr == new_addr + @test default_port == new_port + close(sock) +end + +begin + default_port = UInt16(11011) + default_addr = getipaddr() + + sock = Base.TCPServer() + bind(sock,Base.InetAddr(default_addr,default_port)) + listen(sock) + + @async begin + sleep(1) + ssock = connect(default_addr, default_port) + end + + csock = accept(sock) + new_addr, new_port = getsockname(csock) + + @test default_addr == new_addr + @test new_port > 0 + close(csock) + close(sock) +end From 5c0eb6d4e5426fa8a1b0226fab5afc7b0d08d109 Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Tue, 22 Sep 2015 16:06:00 -0400 Subject: [PATCH 0236/1938] Fix #13261, isvatuple MethodError in inference --- base/inference.jl | 4 +++- test/core.jl | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/base/inference.jl b/base/inference.jl index 2611f17c8fa5d..00acb4392723c 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -783,6 +783,8 @@ function precise_container_types(args, types, vtypes, sv) if any(isvarargtype, result[i]) return nothing end + elseif ti === Union{} + return nothing elseif ti<:Tuple && (i==n || !isvatuple(ti)) result[i] = ti.parameters elseif ti<:AbstractArray && i==n @@ -2813,7 +2815,7 @@ function inlining_pass(e::Expr, sv, ast) newargs[i-3] = aarg.args[2:end] elseif isa(aarg, Tuple) newargs[i-3] = Any[ QuoteNode(x) for x in aarg ] - elseif (t<:Tuple) && !isvatuple(t) && effect_free(aarg,sv,true) + elseif (t<:Tuple) && t !== Union{} && !isvatuple(t) && effect_free(aarg,sv,true) # apply(f,t::(x,y)) => f(t[1],t[2]) tp = t.parameters newargs[i-3] = Any[ mk_getfield(aarg,j,tp[j]) for j=1:length(tp) ] diff --git a/test/core.jl b/test/core.jl index c25fd8b99c77b..0f898a9504273 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3390,3 +3390,8 @@ f8932(a::Vec3_8932, b::Vec3_8932) = Vec3_8932(a.x % b.x, a.y % b.y, a.z % b.z) a8932 = Vec3_8932(1,1,1) b8932 = Vec3_8932(2,2,2) @test f8932(a8932, b8932) == Vec3_8932(1.0, 1.0, 1.0) + +# issue #13261 +f13261() = (x = (error("oops"),); +(x...)) +g13261() = f13261() +@test_throws ErrorException g13261() From 926d990c4354d0eef0ce3610e393a548dba4d822 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 24 Sep 2015 15:29:15 -0400 Subject: [PATCH 0237/1938] fix #13302, parsing destructuring assignment in catch block --- src/julia-parser.scm | 2 +- test/parse.jl | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/julia-parser.scm b/src/julia-parser.scm index 9e86e6e7fbdc3..0c43298b0da8d 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -1223,7 +1223,7 @@ '(block) #f finalb) - (let* ((var (parse-eq* s)) + (let* ((var (if nl (parse-eq s) (parse-eq* s))) (var? (and (not nl) (or (symbol? var) (and (length= var 2) (eq? (car var) '$))))) (catch-block (if (eq? (require-token s) 'finally) '(block) diff --git a/test/parse.jl b/test/parse.jl index e463d8734f586..333bdc4dc2e15 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -293,3 +293,14 @@ parse(""" # issue #12771 @test -(3)^2 == -9 + +# issue #13302 +let p = parse("try + a + catch + b, c = t + end") + @test isa(p,Expr) && p.head === :try + @test p.args[2] === false + @test p.args[3].args[end] == parse("b,c = t") +end From 56048be83c1f0458308ad01e8a6e11c8a975409c Mon Sep 17 00:00:00 2001 From: wildart Date: Thu, 24 Sep 2015 16:27:31 -0400 Subject: [PATCH 0238/1938] doc fix --- doc/stdlib/base.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 804f73a785ca9..0c9e04a6be5af 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -893,7 +893,7 @@ System Get the local machine's host name. -.. function:: getipaddr() +.. function:: getipaddr() -> IPAddr .. Docstring generated from Julia source From c961b01b70635c07d2dd5d608f639f2fe592b912 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Tue, 15 Sep 2015 11:15:04 -0700 Subject: [PATCH 0239/1938] Start of the doc quest --- base/linalg/lapack.jl | 1106 +++++++++++++++++++++++++++++++++++++---- doc/stdlib/linalg.rst | 559 +++++++++++++++++++++ 2 files changed, 1563 insertions(+), 102 deletions(-) diff --git a/base/linalg/lapack.jl b/base/linalg/lapack.jl index 10724e6c70005..5a57bfebb7202 100644 --- a/base/linalg/lapack.jl +++ b/base/linalg/lapack.jl @@ -59,6 +59,7 @@ for (gbtrf, gbtrs, elty) in (:zgbtrf_,:zgbtrs_,:Complex128), (:cgbtrf_,:cgbtrs_,:Complex64)) @eval begin + # SUBROUTINE DGBTRF( M, N, KL, KU, AB, LDAB, IPIV, INFO ) # * .. Scalar Arguments .. # INTEGER INFO, KL, KU, LDAB, M, N @@ -78,6 +79,7 @@ for (gbtrf, gbtrs, elty) in @lapackerror AB, ipiv end + # SUBROUTINE DGBTRS( TRANS, N, KL, KU, NRHS, AB, LDAB, IPIV, B, LDB, INFO) # * .. Scalar Arguments .. # CHARACTER TRANS @@ -107,6 +109,27 @@ for (gbtrf, gbtrs, elty) in end end +""" + gbtrf!(kl, ku, m, AB) -> (AB, ipiv) + +Compute the LU factorization of a banded matrix `AB`. `kl` is the first +subdiagonal containing a nonzero band, `ku` is the last superdiagonal +containing one, and `m` is the first dimension of the matrix `AB`. Returns +the LU factorization in-place and `ipiv`, the vector of pivots used. +""" +gbtrf!(kl::Integer, ku::Integer, m::Integer, AB::StridedMatrix) + +""" + gbtrs!(trans, kl, ku, m, AB, ipiv, B) + +Solve the equation `AB * X = B`. `trans` determines the orientation of `AB`. It may +be `N` (no transpose), `T` (transpose), or `C` (conjugate transpose). `kl` is the +first subdiagonal containing a nonzero band, `ku` is the last superdiagonal +containing one, and `m` is the first dimension of the matrix `AB`. `ipiv` is the vector +of pivots returned from `gbtrf!`. Returns the vector or matrix `X`, overwriting `B` in-place. +""" +gbtrs!(trans::Char, kl::Integer, ku::Integer, m::Integer, AB::StridedMatrix, ipiv::Vector{BlasInt}, B::StridedVecOrMat) + ## (GE) general matrices: balancing and back-transforming for (gebal, gebak, elty, relty) in ((:dgebal_, :dgebak_, :Float64, :Float64), @@ -114,6 +137,7 @@ for (gebal, gebak, elty, relty) in (:zgebal_, :zgebak_, :Complex128, :Float64), (:cgebal_, :cgebak_, :Complex64, :Float32)) @eval begin + # SUBROUTINE DGEBAL( JOB, N, A, LDA, ILO, IHI, SCALE, INFO ) #* .. Scalar Arguments .. # CHARACTER JOB @@ -134,6 +158,7 @@ for (gebal, gebak, elty, relty) in @lapackerror ilo[1], ihi[1], scale end + # SUBROUTINE DGEBAK( JOB, SIDE, N, ILO, IHI, SCALE, M, V, LDV, INFO ) #* .. Scalar Arguments .. # CHARACTER JOB, SIDE @@ -157,16 +182,29 @@ for (gebal, gebak, elty, relty) in end end +""" + gebal!(job, A) -> (ilo, ihi, scale) + +Balance the matrix `A` before computing its eigensystem or Schur factorization. +`job` can be one of `N` (`A` will not be permuted or scaled), `P` (`A` will only +be permuted), `S` (`A` will only be scaled), or `B` (`A` will be both permuted +and scaled). Modifies `A` in-place and returns `ilo`, `ihi`, and `scale`. If +permuting was turned on, `A[i,j] = 0` if `j > i` and `1 < j < ilo` or `j > ihi`. +`scale` contains information about the scaling/permutations performed. +""" +gebal!(job::Char, A::StridedMatrix) + +""" + gebak!(job, side, ilo, ihi, scale, V) + +Transform the eigenvectors `V` of a matrix balanced using `gebal!` to +the unscaled/unpermuted eigenvectors of the original matrix. Modifies `V` +in-place. `side` can be `L` (left eigenvectors are transformed) or `R` +(right eigenvectors are transformed). +""" +gebak!(job::Char, side::Char, ilo::BlasInt, ihi::BlasInt, scale::Vector, V::StridedMatrix) + # (GE) general matrices, direct decompositions -# gebrd - reduction to bidiagonal form Q'*A*P=B where Q and P are orthogonal -# gelqf - unpivoted LQ decomposition -# geqlf - unpivoted QL decomposition -# geqrf - unpivoted QR decomposition -# gegp3 - pivoted QR decomposition -# geqrt - unpivoted QR by WY representation -# geqrt3! - recursive algorithm producing compact WY representation of Q -# gerqf - unpivoted RQ decomposition -# getrf - LU decomposition # # These mutating functions take as arguments all the values they # return, even if the value of the function does not depend on them @@ -179,6 +217,7 @@ for (gebrd, gelqf, geqlf, geqrf, geqp3, geqrt, geqrt3, gerqf, getrf, elty, relty (:zgebrd_,:zgelqf_,:zgeqlf_,:zgeqrf_,:zgeqp3_,:zgeqrt_,:zgeqrt3_,:zgerqf_,:zgetrf_,:Complex128,:Float64), (:cgebrd_,:cgelqf_,:cgeqlf_,:cgeqrf_,:cgeqp3_,:cgeqrt_,:cgeqrt3_,:cgerqf_,:cgetrf_,:Complex64,:Float32)) @eval begin + # SUBROUTINE DGEBRD( M, N, A, LDA, D, E, TAUQ, TAUP, WORK, LWORK, # INFO ) # .. Scalar Arguments .. @@ -213,6 +252,7 @@ for (gebrd, gelqf, geqlf, geqrf, geqp3, geqrt, geqrt3, gerqf, getrf, elty, relty end A, d, e, tauq, taup end + # SUBROUTINE DGELQF( M, N, A, LDA, TAU, WORK, LWORK, INFO ) # * .. Scalar Arguments .. # INTEGER INFO, LDA, LWORK, M, N @@ -242,6 +282,7 @@ for (gebrd, gelqf, geqlf, geqrf, geqp3, geqrt, geqrt3, gerqf, getrf, elty, relty end A, tau end + # SUBROUTINE DGEQLF( M, N, A, LDA, TAU, WORK, LWORK, INFO ) # * .. Scalar Arguments .. # INTEGER INFO, LDA, LWORK, M, N @@ -271,6 +312,7 @@ for (gebrd, gelqf, geqlf, geqrf, geqp3, geqrt, geqrt3, gerqf, getrf, elty, relty end A, tau end + # SUBROUTINE DGEQP3( M, N, A, LDA, JPVT, TAU, WORK, LWORK, INFO ) # * .. Scalar Arguments .. # INTEGER INFO, LDA, LWORK, M, N @@ -321,6 +363,7 @@ for (gebrd, gelqf, geqlf, geqrf, geqp3, geqrt, geqrt3, gerqf, getrf, elty, relty end return A, tau, jpvt end + function geqrt!(A::StridedMatrix{$elty}, T::Matrix{$elty}) chkstride1(A) m, n = size(A) @@ -344,6 +387,7 @@ for (gebrd, gelqf, geqlf, geqrf, geqp3, geqrt, geqrt3, gerqf, getrf, elty, relty end A, T end + function geqrt3!(A::StridedMatrix{$elty}, T::Matrix{$elty}) chkstride1(A); chkstride1(T) m, n = size(A); p, q = size(T) @@ -364,6 +408,7 @@ for (gebrd, gelqf, geqlf, geqrf, geqp3, geqrt, geqrt3, gerqf, getrf, elty, relty end A, T end + ## geqrfp! - positive elements on diagonal of R - not defined yet # SUBROUTINE DGEQRFP( M, N, A, LDA, TAU, WORK, LWORK, INFO ) # * .. Scalar Arguments .. @@ -392,6 +437,7 @@ for (gebrd, gelqf, geqlf, geqrf, geqp3, geqrt, geqrt3, gerqf, getrf, elty, relty end A, tau end + # SUBROUTINE DGERQF( M, N, A, LDA, TAU, WORK, LWORK, INFO ) # * .. Scalar Arguments .. # INTEGER INFO, LDA, LWORK, M, N @@ -419,6 +465,7 @@ for (gebrd, gelqf, geqlf, geqrf, geqp3, geqrt, geqrt3, gerqf, getrf, elty, relty end A, tau end + # SUBROUTINE DGETRF( M, N, A, LDA, IPIV, INFO ) # * .. Scalar Arguments .. # INTEGER INFO, LDA, M, N @@ -441,6 +488,112 @@ for (gebrd, gelqf, geqlf, geqrf, geqp3, geqrt, geqrt3, gerqf, getrf, elty, relty end end +""" + gebrd!(A) -> (A, d, e, tauq, taup) + +Reduce `A` in-place to bidiagonal form `A = QBP'`. Returns `A`, containing the +bidiagonal matrix `B`; `d`, containing the diagonal elements of `B`; `e`, +containing the off-diagonal elements of `B`; `tauq`, containing the +elementary reflectors representing `Q`; and `taup`, containing the +elementary reflectors representing `P`. +""" +gebrd!(A::StridedMatrix) + +""" + gelqf!(A, tau) + +Compute the `LQ` factorization of `A`, `A = LQ`. `tau` contains scalars +which parameterize the elementary reflectors of the factorization. `tau` +must have length greater than or equal to the smallest dimension of `A`. + +Returns +`A` and `tau` modified in-place. +""" +gelqf!(A::StridedMatrix, tau::Vector) + +""" + geqlf!(A, tau) + +Compute the `QL` factorization of `A`, `A = QL`. `tau` contains scalars +which parameterize the elementary reflectors of the factorization. `tau` +must have length greater than or equal to the smallest dimension of `A`. + +Returns `A` and `tau` modified in-place. +""" +geqlf!(A::StridedMatrix, tau::Vector) + +""" + geqp3!(A, jpvt, tau) + +Compute the pivoted `QR` factorization of `A`, `AP = QR` using BLAS level 3. +`P` is a pivoting matrix, represented by `jpvt`. `tau` stores the elementary +reflectors. `jpvt` must have length length greater than or equal to `n` if `A` +is an `(m x n)` matrix. `tau` must have length greater than or equal to the +smallest dimension of `A`. + +`A`, `jpvt`, and `tau` are modified in-place. +""" +geqp3!(A::StridedMatrix, jpvt::Vector{BlasInt}, tau::Vector) + +""" + geqrt!(A, T) + +Compute the blocked `QR` factorization of `A`, `A = QR`. `T` contains upper +triangular block reflectors which parameterize the elementary reflectors of +the factorization. The first dimension of `T` sets the block size and it must +be between 1 and `n`. The second dimension of `T` must equal the smallest +dimension of `A`. + +Returns `A` and `T` modified in-place. +""" +geqrt!(A::StridedMatrix, T::Matrix) + +""" + geqrt3!(A, T) + +Recursively computes the blocked `QR` factorization of `A`, `A = QR`. `T` +contains upper triangular block reflectors which parameterize the +elementary reflectors of the factorization. The first dimension of `T` sets the +block size and it must be between 1 and `n`. The second dimension of `T` must +equal the smallest dimension of `A`. + +Returns `A` and `T` modified in-place. +""" +geqrt3!(A::StridedMatrix, T::Matrix) + +""" + geqrf!(A, tau) + +Compute the `QR` factorization of `A`, `A = QR`. `tau` contains scalars +which parameterize the elementary reflectors of the factorization. `tau` +must have length greater than or equal to the smallest dimension of `A`. + +Returns `A` and `tau` modified in-place. +""" +geqrf!(A::StridedMatrix, tau::Vector) + +""" + gerqf!(A, tau) + +Compute the `RQ` factorization of `A`, `A = RQ`. `tau` contains scalars +which parameterize the elementary reflectors of the factorization. `tau` +must have length greater than or equal to the smallest dimension of `A`. + +Returns `A` and `tau` modified in-place. +""" +gerqf!(A::StridedMatrix, tau::Vector) + +""" + getrf!(A) -> (A, ipiv, info) + +Compute the pivoted `LU` factorization of `A`, `A = LU`. + +Returns `A`, modified in-place, `ipiv`, the pivoting information, and an `info` +code which indicates success (`info = 0`), a singular value in `U` +(`info = i`, in which case `U[i,i]` is singular), or an error code (`info < 0`). +""" +getrf!(A::StridedMatrix, tau::Vector) + gelqf!{T<:BlasFloat}(A::StridedMatrix{T}) = ((m,n)=size(A); gelqf!(A,similar(A,T,min(m,n)))) geqlf!{T<:BlasFloat}(A::StridedMatrix{T}) = ((m,n)=size(A); geqlf!(A,similar(A,T,min(m,n)))) geqrt!{T<:BlasFloat}(A::StridedMatrix{T}, nb::Integer) = geqrt!(A,similar(A,T,nb,minimum(size(A)))) @@ -464,6 +617,7 @@ for (tzrzf, ormrz, elty) in (:ztzrzf_,:zunmrz_,:Complex128), (:ctzrzf_,:cunmrz_,:Complex64)) @eval begin + # * SUBROUTINE ZTZRZF( M, N, A, LDA, TAU, WORK, LWORK, INFO ) # 22 * # 23 * .. Scalar Arguments .. @@ -504,6 +658,7 @@ for (tzrzf, ormrz, elty) in # 27 * .. # 28 * .. Array Arguments .. # 29 * COMPLEX*16 A( LDA, * ), C( LDC, * ), TAU( * ), WORK( * ) + function ormrz!(side::Char, trans::Char, A::StridedMatrix{$elty}, tau::StridedVector{$elty}, C::StridedMatrix{$elty}) chktrans(trans) chkside(side) @@ -536,6 +691,26 @@ for (tzrzf, ormrz, elty) in end end +""" + ormrz!(side, trans, A, tau, C) + +Multiplies the matrix `C` by `Q` from the transformation supplied by +`tzrzf!`. Depending on `side` or `trans` the multiplication can be +left-sided (`side = L, Q*C`) or right-sided (`side = R, C*Q`) and `Q` +can be unmodified (`trans = N`), transposed (`trans = T`), or conjugate +transposed (`trans = C`). Returns matrix `C` which is modified in-place +with the result of the multiplication. +""" +ormrz!(side::Char, trans::Char, A::StridedMatrix, tau::StridedVector, C::StridedMatrix) + +""" + tzrzf!(A) -> (A, tau) + +Transforms the upper trapezoidal matrix `A` to upper triangular form in-place. +Returns `A` and `tau`, the scalar parameters for the elementary reflectors +of the transformation. +""" +tzrzf!(A::StridedMatrix) ## (GE) general matrices, solvers with factorization, solver and inverse for (gels, gesv, getrs, getri, elty) in @@ -659,6 +834,50 @@ for (gels, gesv, getrs, getri, elty) in end end end + +""" + gels!(trans, A, B) -> (F, B, ssr) + +Solves the linear equation `A * X = B`, `A.' * X =B`, or `A' * X = B` using +a QR or LQ factorization. Modifies the matrix/vector `B` in place with the +solution. `A` is overwritten with its `QR` or `LQ` factorization. `trans` +may be one of `N` (no modification), `T` (transpose), or `C` (conjugate +transpose). `gels!` searches for the minimum norm/least squares solution. +`A` may be under or over determined. The solution is returned in `B`. +""" +gels!(trans::Char, A::StridedMatrix, B::StridedVecOrMat) + +""" + gesv!(A, B) -> (B, A, ipiv) + +Solves the linear equation `A * X = B` where `A` is a square matrix using +the `LU` factorization of `A`. `A` is overwritten with its `LU` +factorization and `B` is overwritten with the solution `X`. `ipiv` contains the +pivoting information for the `LU` factorization of `A`. +""" +gesv!(A::StridedMatrix, B::StridedVecOrMat) + +""" + getrs!(trans, A, ipiv, B) + +Solves the linear equation `A * X = B`, `A.' * X =B`, or `A' * X = B` for +square `A`. Modifies the matrix/vector `B` in place with the solution. `A` +is the `LU` factorization from `getrf!`, with `ipiv` the pivoting +information. `trans` may be one of `N` (no modification), `T` (transpose), +or `C` (conjugate transpose). +""" +getrs!(trans::Char, A::StridedMatrix, ipiv::Vector{BlasInt}, B::StridedVecOrMat) + +""" + getri!(A, ipiv) + +Computes the inverse of `A`, using its `LU` factorization found by +`getrf!`. `ipiv` is the pivot information output and `A` +contains the `LU` factorization of `getrf!`. `A` is overwritten with +its inverse. +""" +getri!(A::StridedMatrix, ipiv::Vector{BlasInt}) + for (gesvx, elty) in ((:dgesvx_,:Float64), (:sgesvx_,:Float32)) @@ -710,7 +929,6 @@ for (gesvx, elty) in #WORK(1) contains the reciprocal pivot growth factor norm(A)/norm(U) X, equed, R, C, B, rcond[1], ferr, berr, work[1] end - #Wrapper for the no-equilibration, no-transpose calculation function gesvx!(A::StridedMatrix{$elty}, B::StridedVecOrMat{$elty}) n=size(A,1) X, equed, R, C, B, rcond, ferr, berr, rpgf = gesvx!('N', 'N', A, similar(A, $elty, n, n), similar(A, BlasInt, n), 'N', similar(A, $elty, n), similar(A, $elty, n), B) @@ -775,7 +993,41 @@ for (gesvx, elty, relty) in X, rcond, ferr, berr, rpgf end end - end +end + +""" + gesvx!(fact, trans, A, AF, ipiv, equed, R, C, B) -> (X, equed, R, C, B, rcond, ferr, berr, work) + +Solves the linear equation `A * X = B` (`trans = N`), `A.' * X =B` +(`trans = T`), or `A' * X = B` (`trans = C`) using the `LU` factorization +of `A`. `fact` may be `E`, in which case `A` will be equilibrated and copied +to `AF`; `F`, in which case `AF` and `ipiv` from a previous `LU` factorization +are inputs; or `N`, in which case `A` will be copied to `AF` and then +factored. If `fact = F`, `equed` may be `N`, meaning `A` has not been +equilibrated; `R`, meaning `A` was multiplied by `diagm(R)` from the left; +`C`, meaning `A` was multiplied by `diagm(C)` from the right; or `B`, meaning +`A` was multiplied by `diagm(R)` from the left and `diagm(C)` from the right. +If `fact = F` and `equed = R` or `B` the elements of `R` must all be positive. +If `fact = F` and `equed = C` or `B` the elements of `C` must all be positive. + +Returns the solution `X`; `equed`, which is an output if `fact` is not `N`, +and describes the equilibration that was performed; `R`, the row equilibration +diagonal; `C`, the column equilibration diagonal; `B`, which may be overwritten +with its equilibrated form `diagm(R)*B` (if `trans = N` and `equed = R,B`) or +`diagm(C)*B` (if `trans = T,C` and `equed = C,B`); `rcond`, the reciprocal +condition number of `A` after equilbrating; `ferr`, the forward error bound for +each solution vector in `X`; `berr`, the forward error bound for each solution +vector in `X`; and `work`, the reciprocal pivot growth factor. +""" +gesvx!(fact::Char, trans::Char, A::StridedMatrix, AF::StridedMatrix, + ipiv::Vector{BlasInt}, equed::Char, R::Vector, C::Vector, B::StridedVecOrMat) + +""" + gesvx!(A, B) + +The no-equilibration, no-transpose simplification of `gesvx!`. +""" +gesvx!(A::StridedMatrix, B::StridedVecOrMat) for (gelsd, gelsy, elty) in ((:dgelsd_,:dgelsy_,:Float64), @@ -821,7 +1073,6 @@ for (gelsd, gelsy, elty) in end subsetrows(B, newB, n), rnk[1] end - # SUBROUTINE DGELSY( M, N, NRHS, A, LDA, B, LDB, JPVT, RCOND, RANK, # $ WORK, LWORK, INFO ) # * .. Scalar Arguments .. @@ -964,6 +1215,28 @@ for (gelsd, gelsy, elty, relty) in end end +""" + gelsd!(A, B, rcond) -> (B, rnk) + +Computes the least norm solution of `A * X = B` by finding the `SVD` +factorization of `A`, then dividing-and-conquering the problem. `B` +is overwritten with the solution `X`. Singular values below `rcond` +will be treated as zero. Returns the solution in `B` and the effective rank +of `A` in `rnk`. +""" +gelsd!(A::StridedMatrix, B::StridedVecOrMat, rcond::Real) + +""" + gelsy!(A, B, rcond) -> (B, rnk) + +Computes the least norm solution of `A * X = B` by finding the full `QR` +factorization of `A`, then dividing-and-conquering the problem. `B` +is overwritten with the solution `X`. Singular values below `rcond` +will be treated as zero. Returns the solution in `B` and the effective rank +of `A` in `rnk`. +""" +gelsy!(A::StridedMatrix, B::StridedVecOrMat, rcond::Real) + for (gglse, elty) in ((:dgglse_, :Float64), (:sgglse_, :Float32), (:zgglse_, :Complex128), @@ -1014,6 +1287,15 @@ for (gglse, elty) in ((:dgglse_, :Float64), end end +""" + gglse!(A, c, B, d) -> (X,res) + +Solves the equation `A * x = c` where `x` is subject to the equality +constraint `B * x = d`. Uses the formula `||c - A*x||^2 = 0` to solve. +Returns `X` and the residual sum-of-squares. +""" +gglse!(A::StridedMatrix, c::StridedVector, B::StridedMatrix, d::StridedVector) + # (GE) general matrices eigenvalue-eigenvector and singular value decompositions for (geev, gesvd, gesdd, ggsvd, elty, relty) in ((:dgeev_,:dgesvd_,:dgesdd_,:dggsvd_,:Float64,:Float64), @@ -1256,6 +1538,58 @@ for (geev, gesvd, gesdd, ggsvd, elty, relty) in end end end + +""" + geev!(jobvl, jobvr, A) -> (W, VL, VR) + +Finds the eigensystem of `A`. If `jobvl = N`, the left eigenvectors of +`A` aren't computed. If `jobvr = N`, the right eigenvectors of `A` +aren't computed. If `jobvl = V` or `jobvr = V`, the corresponding +eigenvectors are computed. Returns the eigenvalues in `W`, the right +eigenvectors in `VR`, and the left eigenvectors in `VL`. +""" +geev!(jobvl::Char, jobvr::Char, A::StridedMatrix) + +""" + gesdd!(job, A) -> (U, S, VT) + +Finds the singular value decomposition of `A`, `A = U * S * V'`, +using a divide and conquer approach. If `job = A`, all the columns of `U` and +the rows of `V'` are computed. If `job = N`, no columns of `U` or rows of `V'` +are computed. If `job = O`, `A` is overwritten with the columns of (thin) `U` +and the rows of (thin) `V'`. If `job = S`, the columns of (thin) `U` and the +rows of (thin) `V'` are computed and returned separately. +""" +gesdd!(job::Char, A::StridedMatrix) + +""" + gesvd!(jobu, jobvt, A) -> (U, S, VT) + +Finds the singular value decomposition of `A`, `A = U * S * V'`. +If `jobu = A`, all the columns of `U` are computed. If `jobvt = A` all the rows +of `V'` are computed. If `jobu = N`, no columns of `U` are computed. If +`jobvt = N` no rows of `V'` are computed. If `jobu = O`, `A` is overwritten with +the columns of (thin) `U`. If `jobvt = O`, `A` is overwritten with the rows +of (thin) `V'`. If `jobu = S`, the columns of (thin) `U` are computed +and returned separately. If `jobvt = S` the rows of (thin) `V'` are +computed and returned separately. `jobu` and `jobvt` can't both be `O`. + +Returns `U`, `S`, and `Vt`, where `S` are the singular values of `A`. +""" +gesvd!(jobu::Char, jobvt::Char, A::StridedMatrix) + +""" + ggsvd!(jobu, jobv, jobq, A, B) -> (U, V, Q, alpha, beta, k, l, R) + +Finds the generalized singular value decomposition of `A` and `B`, `U'*A*Q = D1*R` +and `V'*B*Q = D2*R`. `D1` has `alpha` on its diagonal and `D2` has `beta` on its +diagonal. If `jobu = U`, the orthogonal/unitary matrix `U` is computed. If +`jobv = V` the orthogonal/unitary matrix `V` is computed. If `jobq = Q`, +the orthogonal/unitary matrix `Q` is computed. If `job{u,v,q} = N`, that +matrix is not computed. +""" +ggsvd!(jobu::Char, jobv::Char, jobq::Char, A::Matrix, B::Matrix) + ## Expert driver and generalized eigenvalue problem for (geevx, ggev, elty) in ((:dgeevx_,:dggev_,:Float64), @@ -1275,71 +1609,70 @@ for (geevx, ggev, elty) in # DOUBLE PRECISION A( LDA, * ), RCONDE( * ), RCONDV( * ), # $ SCALE( * ), VL( LDVL, * ), VR( LDVR, * ), # $ WI( * ), WORK( * ), WR( * ) - function geevx!(balanc::Char, jobvl::Char, jobvr::Char, sense::Char, A::StridedMatrix{$elty}) - n = chksquare(A) - lda = max(1,stride(A,2)) - wr = similar(A, $elty, n) - wi = similar(A, $elty, n) - if balanc ∉ ['N', 'P', 'S', 'B'] - throw(ArgumentError("balanc must be 'N', 'P', 'S', or 'B', but $balanc was passed")) - end - ldvl = 0 - if jobvl == 'V' - ldvl = n - elseif jobvl == 'N' + function geevx!(balanc::Char, jobvl::Char, jobvr::Char, sense::Char, A::StridedMatrix{$elty}) + n = chksquare(A) + lda = max(1,stride(A,2)) + wr = similar(A, $elty, n) + wi = similar(A, $elty, n) + if balanc ∉ ['N', 'P', 'S', 'B'] + throw(ArgumentError("balanc must be 'N', 'P', 'S', or 'B', but $balanc was passed")) + end ldvl = 0 - else - throw(ArgumentError("jobvl must be 'V' or 'N', but $jobvl was passed")) - end - VL = similar(A, $elty, ldvl, n) - ldvr = 0 - if jobvr == 'V' - ldvr = n - elseif jobvr == 'N' + if jobvl == 'V' + ldvl = n + elseif jobvl == 'N' + ldvl = 0 + else + throw(ArgumentError("jobvl must be 'V' or 'N', but $jobvl was passed")) + end + VL = similar(A, $elty, ldvl, n) ldvr = 0 - else - throw(ArgumentError("jobvr must be 'V' or 'N', but $jobvr was passed")) - end - VR = similar(A, $elty, ldvr, n) - ilo = Array(BlasInt, 1) - ihi = Array(BlasInt, 1) - scale = similar(A, $elty, n) - abnrm = Array($elty, 1) - rconde = similar(A, $elty, n) - rcondv = similar(A, $elty, n) - work = Array($elty, 1) - lwork::BlasInt = -1 - iworksize = 0 - if sense == 'N' || sense == 'E' + if jobvr == 'V' + ldvr = n + elseif jobvr == 'N' + ldvr = 0 + else + throw(ArgumentError("jobvr must be 'V' or 'N', but $jobvr was passed")) + end + VR = similar(A, $elty, ldvr, n) + ilo = Array(BlasInt, 1) + ihi = Array(BlasInt, 1) + scale = similar(A, $elty, n) + abnrm = Array($elty, 1) + rconde = similar(A, $elty, n) + rcondv = similar(A, $elty, n) + work = Array($elty, 1) + lwork::BlasInt = -1 iworksize = 0 - elseif sense == 'V' || sense == 'B' - iworksize = 2*n - 2 - else - throw(ArgumentError("sense must be 'N', 'E', 'V' or 'B', but $sense was passed")) - end - iwork = Array(BlasInt, iworksize) - info = Array(BlasInt, 1) - for i = 1:2 - ccall(($(blasfunc(geevx)), liblapack), Void, - (Ptr{UInt8}, Ptr{UInt8}, Ptr{UInt8}, Ptr{UInt8}, - Ptr{BlasInt}, Ptr{$elty}, Ptr{BlasInt}, Ptr{$elty}, - Ptr{$elty}, Ptr{$elty}, Ptr{BlasInt}, Ptr{$elty}, - Ptr{BlasInt}, Ptr{BlasInt}, Ptr{BlasInt}, Ptr{$elty}, - Ptr{$elty}, Ptr{$elty}, Ptr{$elty}, Ptr{$elty}, - Ptr{BlasInt}, Ptr{BlasInt}, Ptr{BlasInt}), - &balanc, &jobvl, &jobvr, &sense, - &n, A, &lda, wr, - wi, VL, &max(1,ldvl), VR, - &max(1,ldvr), ilo, ihi, scale, - abnrm, rconde, rcondv, work, - &lwork, iwork, info) - lwork = convert(BlasInt, work[1]) - work = Array($elty, lwork) + if sense == 'N' || sense == 'E' + iworksize = 0 + elseif sense == 'V' || sense == 'B' + iworksize = 2*n - 2 + else + throw(ArgumentError("sense must be 'N', 'E', 'V' or 'B', but $sense was passed")) + end + iwork = Array(BlasInt, iworksize) + info = Array(BlasInt, 1) + for i = 1:2 + ccall(($(blasfunc(geevx)), liblapack), Void, + (Ptr{UInt8}, Ptr{UInt8}, Ptr{UInt8}, Ptr{UInt8}, + Ptr{BlasInt}, Ptr{$elty}, Ptr{BlasInt}, Ptr{$elty}, + Ptr{$elty}, Ptr{$elty}, Ptr{BlasInt}, Ptr{$elty}, + Ptr{BlasInt}, Ptr{BlasInt}, Ptr{BlasInt}, Ptr{$elty}, + Ptr{$elty}, Ptr{$elty}, Ptr{$elty}, Ptr{$elty}, + Ptr{BlasInt}, Ptr{BlasInt}, Ptr{BlasInt}), + &balanc, &jobvl, &jobvr, &sense, + &n, A, &lda, wr, + wi, VL, &max(1,ldvl), VR, + &max(1,ldvr), ilo, ihi, scale, + abnrm, rconde, rcondv, work, + &lwork, iwork, info) + lwork = convert(BlasInt, work[1]) + work = Array($elty, lwork) + end + @lapackerror + A, wr, wi, VL, VR, ilo[1], ihi[1], scale, abnrm[1], rconde, rcondv end - @lapackerror - A, wr, wi, VL, VR, ilo[1], ihi[1], scale, abnrm[1], rconde, rcondv - end - # SUBROUTINE DGGEV( JOBVL, JOBVR, N, A, LDA, B, LDB, ALPHAR, ALPHAI, # $ BETA, VL, LDVL, VR, LDVR, WORK, LWORK, INFO ) # * .. Scalar Arguments .. @@ -1547,6 +1880,35 @@ for (geevx, ggev, elty, relty) in end end +""" + geevx!(balanc, jobvl, jobvr, sense, A) -> (A, w, VL, VR, ilo, ihi, scale, abnrm, rconde, rcondv) + +Finds the eigensystem of `A` with matrix balancing. If `jobvl = N`, the +left eigenvectors of `A` aren't computed. If `jobvr = N`, the right +eigenvectors of `A` aren't computed. If `jobvl = V` or `jobvr = V`, the +corresponding eigenvectors are computed. If `balanc = N`, no balancing is +performed. If `balanc = P`, `A` is permuted but not scaled. If +`balanc = S`, `A` is scaled but not permuted. If `balanc = B`, `A` is +permuted and scaled. If `sense = N`, no reciprocal condition numbers are +computed. If `sense = E`, reciprocal condition numbers are computed for +the eigenvalues only. If `sense = V`, reciprocal condition numbers are +computed for the right eigenvectors only. If `sense = B`, reciprocal +condition numbers are computed for the right eigenvectors and the +eigenvectors. If `sense = E,B`, the right and left eigenvectors must be +computed. +""" +geevx!(balanc::Char, jobvl::Char, jobvr::Char, sense::Char, A::StridedMatrix) + +""" + ggev!(jobvl, jobvr, A, B) -> (alpha, beta, vl, vr) + +Finds the generalized eigendecomposition of `A` and `B`. If `jobvl = N`, +the left eigenvectors aren't computed. If `jobvr = N`, the right +eigenvectors aren't computed. If `jobvl = V` or `jobvr = V`, the +corresponding eigenvectors are computed. +""" +ggev!(jobvl::Char, jobvr::Char, A::StridedMatrix, B::StridedMatrix) + # One step incremental condition estimation of max/min singular values for (laic1, elty) in ((:dlaic1_,:Float64), @@ -1612,7 +1974,6 @@ for (laic1, elty, relty) in end end - # (GT) General tridiagonal, decomposition, solver and direct solver for (gtsv, gttrf, gttrs, elty) in ((:dgtsv_,:dgttrf_,:dgttrs_,:Float64), @@ -1707,6 +2068,38 @@ for (gtsv, gttrf, gttrs, elty) in end end +""" + gtsv!(dl, d, du, B) + +Solves the equation `A * X = B` where `A` is a tridiagonal matrix with +`dl` on the subdiagonal, `d` on the diagonal, and `du` on the +superdiagonal. + +Overwrites `B` with the solution `X` and returns it. +""" +gtsv!(dl::Vector, d::Vector, du::Vector, B::StridedVecOrMat) + +""" + gttrf!(dl, d, du) -> (dl, d, du, du2, ipiv) + +Finds the `LU` factorization of a tridiagonal matrix with `dl` on the +subdiagonal, `d` on the diagonal, and `du` on the superdiagonal. + +Modifies `dl`, `d`, and `du` in-place and returns them and the second +superdiagonal `du2` and the pivoting vector `ipiv`. +""" +gttrf!(dl::Vector, d::Vector, du::Vector) + +""" + gttrs!(trans, dl, d, du, du2, ipiv, B) + +Solves the equation `A * X = B` (`trans = N`), `A.' * X = B` (`trans = T`), +or `A' * X = B` (`trans = C`) using the `LU` factorization computed by +`gttrf!`. `B` is overwritten with the solution `X`. +""" +gttrs!(trans::Char, dl::Vector, d::Vector, du::Vector, du2::Vector, + ipiv::Vector{BlasInt}, B::StridedVecOrMat) + ## (OR) orthogonal (or UN, unitary) matrices, extractors and multiplication for (orglq, orgqr, orgql, orgrq, ormlq, ormqr, ormql, ormrq, gemqrt, elty) in ((:dorglq_,:dorgqr_,:dorgql_,:dorgrq_,:dormlq_,:dormqr_,:dormql_,:dormrq_,:dgemqrt_,:Float64), @@ -1714,6 +2107,7 @@ for (orglq, orgqr, orgql, orgrq, ormlq, ormqr, ormql, ormrq, gemqrt, elty) in (:zunglq_,:zungqr_,:zungql_,:zungrq_,:zunmlq_,:zunmqr_,:zunmql_,:zunmrq_,:zgemqrt_,:Complex128), (:cunglq_,:cungqr_,:cungql_,:cungrq_,:cunmlq_,:cunmqr_,:cunmql_,:cunmrq_,:cgemqrt_,:Complex64)) @eval begin + # SUBROUTINE DORGLQ( M, N, K, A, LDA, TAU, WORK, LWORK, INFO ) # * .. Scalar Arguments .. # INTEGER INFO, K, LDA, LWORK, M, N @@ -2087,8 +2481,89 @@ for (orglq, orgqr, orgql, orgrq, ormlq, ormqr, ormql, ormrq, gemqrt, elty) in end end +""" + orglq!(A, tau, k = length(tau)) + +Explicitly finds the matrix `Q` of a `LQ` factorization after calling +`gelqf!` on `A`. Uses the output of `gelqf!`. `A` is overwritten by `Q`. +""" +orglq!(A::StridedMatrix, tau::Vector, k::Integer = length(tau)) + +""" + orgqr!(A, tau, k = length(tau)) + +Explicitly finds the matrix `Q` of a `QR` factorization after calling +`geqrf!` on `A`. Uses the output of `geqrf!`. `A` is overwritten by `Q`. +""" +orgqr!(A::StridedMatrix, tau::Vector, k::Integer = length(tau)) + +""" + orgql!(A, tau, k = length(tau)) + +Explicitly finds the matrix `Q` of a `QL` factorization after calling +`geqlf!` on `A`. Uses the output of `geqlf!`. `A` is overwritten by `Q`. +""" +orgql!(A::StridedMatrix, tau::Vector, k::Integer = length(tau)) + +""" + orgrq!(A, tau, k = length(tau)) + +Explicitly finds the matrix `Q` of a `RQ` factorization after calling +`gerqf!` on `A`. Uses the output of `gerqf!`. `A` is overwritten by `Q`. +""" +orgrq!(A::StridedMatrix, tau::Vector, k::Integer = length(tau)) + +""" + ormlq!(side, trans, A, tau, C) + +Computes `Q * C` (`trans = N`), `Q.' * C` (`trans = T`), `Q' * C` +(`trans = C`) for `side = L` or the equivalent right-sided multiplication +for `side = R` using `Q` from a `LQ` factorization of `A` computed using +`gelqf!`. `C` is overwritten. +""" +ormlq!(side::Char, trans::Char, A::StridedMatrix, tau::Vector, C::StridedVecOrMat) + +""" + ormqr!(side, trans, A, tau, C) + +Computes `Q * C` (`trans = N`), `Q.' * C` (`trans = T`), `Q' * C` +(`trans = C`) for `side = L` or the equivalent right-sided multiplication +for `side = R` using `Q` from a `QR` factorization of `A` computed using +`geqrf!`. `C` is overwritten. +""" +ormqr!(side::Char, trans::Char, A::StridedMatrix, tau::Vector, C::StridedVecOrMat) + +""" + ormql!(side, trans, A, tau, C) + +Computes `Q * C` (`trans = N`), `Q.' * C` (`trans = T`), `Q' * C` +(`trans = C`) for `side = L` or the equivalent right-sided multiplication +for `side = R` using `Q` from a `QL` factorization of `A` computed using +`geqlf!`. `C` is overwritten. +""" +ormql!(side::Char, trans::Char, A::StridedMatrix, tau::Vector, C::StridedVecOrMat) + +""" + ormrq!(side, trans, A, tau, C) + +Computes `Q * C` (`trans = N`), `Q.' * C` (`trans = T`), `Q' * C` +(`trans = C`) for `side = L` or the equivalent right-sided multiplication +for `side = R` using `Q` from a `RQ` factorization of `A` computed using +`gerqf!`. `C` is overwritten. +""" +ormrq!(side::Char, trans::Char, A::StridedMatrix, tau::Vector, C::StridedVecOrMat) + +""" + gemqrt!(side, trans, V, T, C) + +Computes `Q * C` (`trans = N`), `Q.' * C` (`trans = T`), `Q' * C` +(`trans = C`) for `side = L` or the equivalent right-sided multiplication +for `side = R` using `Q` from a `QR` factorization of `A` computed using +`geqrt!`. `C` is overwritten. +""" +gemqrt!(side::Char, trans::Char, V::Matrix, T::Matrix, C::StridedVecOrMat) + # (PO) positive-definite symmetric matrices, -# Cholesky decomposition, solvers (direct and factored) and inverse. for (posv, potrf, potri, potrs, pstrf, elty, rtyp) in ((:dposv_,:dpotrf_,:dpotri_,:dpotrs_,:dpstrf_,:Float64,:Float64), (:sposv_,:spotrf_,:spotri_,:spotrs_,:spstrf_,:Float32,:Float32), @@ -2212,6 +2687,61 @@ for (posv, potrf, potri, potrs, pstrf, elty, rtyp) in end end +""" + posv!(uplo, A, B) -> (A, B) + +Finds the solution to `A * X = B` where `A` is a symmetric or Hermitian +positive definite matrix. If `uplo = U` the upper Cholesky decomposition +of `A` is computed. If `uplo = L` the lower Cholesky decomposition of `A` +is computed. `A` is overwritten by its Cholesky decomposition. `B` is +overwritten with the solution `X`. +""" +posv!(uplo::Char, A::StridedMatrix, B::StridedVecOrMat) + +""" + potrf!(uplo, A) + +Computes the Cholesky (upper if `uplo = U`, lower if `uplo = L`) +decomposition of positive-definite matrix `A`. `A` is overwritten and +returned with an info code. +""" +potrf!(uplo::Char, A::StridedMatrix) + +""" + potri!(uplo, A) + +Computes the inverse of positive-definite matrix `A` after calling +`potrf!` to find its (upper if `uplo = U`, lower if `uplo = L`) Cholesky +decomposition. + +`A` is overwritten by its inverse and returned. +""" +potri!(uplo::Char, A::StridedMatrix) + +""" + potrs!(uplo, A, B) + +Finds the solution to `A * X = B` where `A` is a symmetric or Hermitian +positive definite matrix whose Cholesky decomposition was computed by +`potrf!`. If `uplo = U` the upper Cholesky decomposition of `A` was +computed. If `uplo = L` the lower Cholesky decomposition of `A` was +computed. `B` is overwritten with the solution `X`. +""" +potrs!(uplo::Char, A::StridedMatrix, B::StridedVecOrMat) + +""" + pstrf!(uplo, A, tol) -> (A, piv, rank, info) + +Computes the (upper if `uplo = U`, lower if `uplo = L`) pivoted Cholesky +decomposition of positive-definite matrix `A` with a user-set tolerance +`tol`. `A` is overwritten by its Cholesky decomposition. + +Returns `A`, the pivots `piv`, the rank of `A`, and an `info` code. If `info = 0`, +the factorization succeeded. If `info = i > 0 `, then `A` is indefinite or +rank-deficient. +""" +pstrf!(uplo::Char, A::StridedMatrix, tol::Real) + ## (PT) positive-definite, symmetric, tri-diagonal matrices ## Direct solvers for general tridiagonal and symmetric positive-definite tridiagonal for (ptsv, pttrf, elty, relty) in @@ -2261,6 +2791,25 @@ for (ptsv, pttrf, elty, relty) in end end end + +""" + ptsv!(D, E, B) + +Solves `A * X = B` for positive-definite tridiagonal `A`. `D` is the +diagonal of `A` and `E` is the off-diagonal. `B` is overwritten with the +solution `X` and returned. +""" +ptsv!(D::Vector, E::Vector, B::StridedVecOrMat) + +""" + pttrf!(D, E) + +Computes the LDLt factorization of a positive-definite tridiagonal matrix +with `D` as diagonal and `E` as off-diagonal. `D` and `E` are overwritten +and returned. +""" +pttrf!(D::Vector, E::Vector) + for (pttrs, elty, relty) in ((:dpttrs_,:Float64,:Float64), (:spttrs_,:Float32,:Float32)) @@ -2322,6 +2871,15 @@ for (pttrs, elty, relty) in end end +""" + pttrs!(D, E, B) + +Solves `A * X = B` for positive-definite tridiagonal `A` with diagonal +`D` and off-diagonal `E` after computing `A`'s LDLt factorization using +`pttrf!`. `B` is overwritten with the solution `X`. +""" +pttrs!(D::Vector, E::Vector, B::StridedVecOrMat) + ## (TR) triangular matrices: solver and inverse for (trtri, trtrs, elty) in ((:dtrtri_,:dtrtrs_,:Float64), @@ -2377,6 +2935,27 @@ for (trtri, trtrs, elty) in end end +""" + trtri!(uplo, diag, A) + +Finds the inverse of (upper if `uplo = U`, lower if `uplo = L`) +triangular matrix `A`. If `diag = N`, `A` has non-unit diagonal elements. +If `diag = U`, all diagonal elements of `A` are one. `A` is overwritten +with its inverse. +""" +trtri!(uplo::Char, diag::Char, A::StridedMatrix) + +""" + trtrs!(uplo, trans, diag, A, B) + +Solves `A * X = B` (`trans = N`), `A.' * X = B` (`trans = T`), or +`A' * X = B` (`trans = C`) for (upper if `uplo = U`, lower if `uplo = L`) +triangular matrix `A`. If `diag = N`, `A` has non-unit diagonal elements. +If `diag = U`, all diagonal elements of `A` are one. `B` is overwritten +with the solution `X`. +""" +trtrs!(uplo::Char, trans::Char, diag::Char, A::StridedMatrix, B::StridedVecOrMat) + #Eigenvector computation and condition number estimation for (trcon, trevc, trrfs, elty) in ((:dtrcon_,:dtrevc_,:dtrrfs_,:Float64), @@ -2630,6 +3209,46 @@ for (trcon, trevc, trrfs, elty, relty) in end end +""" + trcon!(norm, uplo, diag, A) + +Finds the reciprocal condition number of (upper if `uplo = U`, lower if +`uplo = L`) triangular matrix `A`. If `diag = N`, `A` has non-unit +diagonal elements. If `diag = U`, all diagonal elements of `A` are one. +If `norm = I`, the condition number is found in the infinity norm. If +`norm = O` or `1`, the condition number is found in the one norm. +""" +trcon!(norm::Char, uplo::Char, diag::Char, A::StridedMatrix) + +""" + trevc!(side, howmny, select, T, VL = similar(T), VR = similar(T)) + +Finds the eigensystem of an upper triangular matrix `T`. If `side = R`, +the right eigenvectors are computed. If `side = L`, the left +eigenvectors are computed. If `side = B`, both sets are computed. If +`howmny = A`, all eigenvectors are found. If `howmny = B`, all +eigenvectors are found and backtransformed using `VL` and `VR`. If +`howmny = S`, only the eigenvectors corresponding to the values in +`select` are computed. +""" +trevc!(side::Char, howmny::Char, select::Vector{BlasInt}, T::StridedMatrix, + VL::StridedMatrix = similar(T), VR::StridedMatrix = similar(T)) + +""" + trrfs!(uplo, trans, diag, A, B, X, Ferr, Berr) -> (Ferr, Berr) + +Estimates the error in the solution to `A * X = B` (`trans = N`), +`A.' * X = B` (`trans = T`), `A' * X = B` (`trans = C`) for `side = L`, +or the equivalent equations a right-handed `side = R` `X * A` after +computing `X` using `trtrs!`. If `uplo = U`, `A` is upper triangular. +If `uplo = L`, `A` is lower triangular. If `diag = N`, `A` has non-unit +diagonal elements. If `diag = U`, all diagonal elements of `A` are one. +`Ferr` and `Berr` are optional inputs. `Ferr` is the forward error and +`Berr` is the backward error, each component-wise. +""" +trrfs!(uplo::Char, trans::Char, diag::Char, A::StridedMatrix, B::StridedVecOrMat, + X::StridedVecOrMat, Ferr::StridedVector, Berr::StridedVector) + ## (ST) Symmetric tridiagonal - eigendecomposition for (stev, stebz, stegr, stein, elty) in ((:dstev_,:dstebz_,:dstegr_,:dstein_,:Float64), @@ -2638,8 +3257,6 @@ for (stev, stebz, stegr, stein, elty) in # , (:cstev_,:Complex64) ) @eval begin - #* DSTEV computes all eigenvalues and, optionally, eigenvectors of a - #* real symmetric tridiagonal matrix A. function stev!(job::Char, dv::Vector{$elty}, ev::Vector{$elty}) n = length(dv) if length(ev) != n - 1 @@ -2687,14 +3304,6 @@ for (stev, stebz, stegr, stein, elty) in @lapackerror w[1:m[1]], iblock[1:m[1]], isplit[1:nsplit[1]] end - #* DSTEGR computes selected eigenvalues and, optionally, eigenvectors - #* of a real symmetric tridiagonal matrix T. Any such unreduced matrix has - #* a well defined set of pairwise different real eigenvalues, the corresponding - #* real eigenvectors are pairwise orthogonal. - #* - #* The spectrum may be computed either completely or partially by specifying - #* either an interval (VL,VU] or a range of indices IL:IU for the desired - #* eigenvalues. function stegr!(jobz::Char, range::Char, dv::Vector{$elty}, ev::Vector{$elty}, vl::Real, vu::Real, il::Integer, iu::Integer) n = length(dv) if length(ev) != n - 1 @@ -2734,13 +3343,7 @@ for (stev, stebz, stegr, stein, elty) in @lapackerror w[1:m[1]], Z[:,1:m[1]] end - #* DSTEIN computes the eigenvectors of a real symmetric tridiagonal - #* matrix T corresponding to specified eigenvalues, using inverse - #* iteration. - # SUBROUTINE DSTEIN( N, D, E, M, W, IBLOCK, ISPLIT, Z, LDZ, WORK, - # $ IWORK, IFAIL, INFO ) - # We allow the user to specify exactly which eigenvectors to get by - # specifying the eigenvalues (which may be approximate) via w_in + function stein!(dv::Vector{$elty}, ev_in::Vector{$elty}, w_in::Vector{$elty}, iblock_in::Vector{BlasInt}, isplit_in::Vector{BlasInt}) n = length(dv) if length(ev_in) != n - 1 @@ -2797,12 +3400,60 @@ stein!(dv::Vector, ev::Vector, w_in::Vector)=stein!(dv, ev, w_in, zeros(BlasInt, # Allow user to specify just one eigenvector to get in stein! stein!(dv::Vector, ev::Vector, eval::Real)=stein!(dv, ev, [eval], zeros(BlasInt,0), zeros(BlasInt,0)) +""" + stev!(job, dv, ev) -> (dv, Zmat) + +Computes the eigensystem for a symmetric tridiagonal matrix with `dv` as +diagonal and `ev` as off-diagonal. If `job = N` only the eigenvalues are +found and returned in `dv`. If `job = V` then the eigenvectors are also found +and returned in `Zmat`. +""" +stev!(job::Char, dv::Vector, ev::Vector) + +""" + stebz!(range, order, vl, vu, il, iu, abstol, dv, ev) -> (dv, iblock, isplit) + +Computes the eigenvalues for a symmetric tridiagonal matrix with `dv` as +diagonal and `ev` as off-diagonal. If `range = A`, all the eigenvalues +are found. If `range = V`, the eigenvalues in the half-open interval +`(vl, vu]` are found. If `range = I`, the eigenvalues with indices between +`il` and `iu` are found. If `order = B`, eigvalues are ordered within a +block. If `order = E`, they are ordered across all the blocks. +`abstol` can be set as a tolerance for convergence. +""" +stebz!(range::Char, order::Char, vl, vu, il::Integer, iu::Integer, abstol::Real, dv::Vector, ev::Vector) + +""" + stegr!(jobz, range, dv, ev, vl, vu, il, iu) -> (w, Z) + +Computes the eigenvalues (`jobz = N`) or eigenvalues and eigenvectors +(`jobz = V`) for a symmetric tridiagonal matrix with `dv` as diagonal +and `ev` as off-diagonal. If `range = A`, all the eigenvalues +are found. If `range = V`, the eigenvalues in the half-open interval +`(vl, vu]` are found. If `range = I`, the eigenvalues with indices between +`il` and `iu` are found. The eigenvalues are returned in `w` and the eigenvectors +in `Z`. +""" +stegr!(jobz::Char, range::Char, dv::Vector, ev::Vector, vl::Real, vu::Real, il::Integer, iu::Integer) + +""" + stein!(dv, ev_in, w_in, iblock_in, isplit_in) + +Computes the eigenvectors for a symmetric tridiagonal matrix with `dv` +as diagonal and `ev_in` as off-diagonal. `w_in` specifies the input +eigenvalues for which to find corresponding eigenvectors. `iblock_in` +specifies the submatrices corresponding to the eigenvalues in `w_in`. +`isplit_in` specifies the splitting points between the submatrix blocks. +""" +stein!(dv::Vector, ev_in::Vector, w_in::Vector, iblock_in::Vector{BlasInt}, isplit_in::Vector{BlasInt}) + ## (SY) symmetric real matrices - Bunch-Kaufman decomposition, ## solvers (direct and factored) and inverse. for (syconv, sysv, sytrf, sytri, sytrs, elty) in ((:dsyconv_,:dsysv_,:dsytrf_,:dsytri_,:dsytrs_,:Float64), (:ssyconv_,:ssysv_,:ssytrf_,:ssytri_,:ssytrs_,:Float32)) @eval begin + # SUBROUTINE DSYCONV( UPLO, WAY, N, A, LDA, IPIV, WORK, INFO ) # * .. Scalar Arguments .. # CHARACTER UPLO, WAY @@ -2964,6 +3615,7 @@ for (syconv, sysv, sytrf, sytri, sytrs, elty) in end end end + ## (SY) hermitian matrices - eigendecomposition, Bunch-Kaufman decomposition, ## solvers (direct and factored) and inverse. for (syconv, hesv, hetrf, hetri, hetrs, elty, relty) in @@ -3129,6 +3781,7 @@ for (syconv, hesv, hetrf, hetri, hetrs, elty, relty) in end end end + for (sysv, sytrf, sytri, sytrs, elty, relty) in ((:zsysv_,:zsytrf_,:zsytri_,:zsytrs_,:Complex128, :Float64), (:csysv_,:csytrf_,:csytri_,:csytrs_,:Complex64, :Float32)) @@ -3277,6 +3930,100 @@ for (sysv, sytrf, sytri, sytrs, elty, relty) in end end +""" + syconv!(uplo, A, ipiv) -> (A, work) + +Converts a symmetric matrix `A` (which has been factorized into a +triangular matrix) into two matrices `L` and `D`. If `uplo = U`, `A` +is upper triangular. If `uplo = L`, it is lower triangular. `ipiv` is +the pivot vector from the triangular factorization. `A` is overwritten +by `L` and `D`. +""" +syconv!(uplo::Char, A::StridedMatrix, ipiv::Vector{BlasInt}) + +""" + sysv!(uplo, A, B) -> (B, A, ipiv) + +Finds the solution to `A * X = B` for symmetric matrix `A`. If `uplo = U`, +the upper half of `A` is stored. If `uplo = L`, the lower half is stored. +`B` is overwritten by the solution `X`. `A` is overwritten by its +Bunch-Kaufman factorization. `ipiv` contains pivoting information about the +factorization. +""" +sysv!(uplo::Char, A::StridedMatrix, B::StridedVecOrMat) + +""" + sytrf!(uplo, A) -> (A, ipiv) + +Computes the Bunch-Kaufman factorization of a symmetric matrix `A`. If +`uplo = U`, the upper half of `A` is stored. If `uplo = L`, the lower +half is stored. + +Returns `A`, overwritten by the factorization, and a pivot vector `ipiv`. +""" +sytrf!(uplo::Char, A::StridedMatrix) + +""" + sytri!(uplo, A, ipiv) + +Computes the inverse of a symmetric matrix `A` using the results of +`sytrf!`. If `uplo = U`, the upper half of `A` is stored. If `uplo = L`, +the lower half is stored. `A` is overwritten by its inverse. +""" +sytri!(uplo::Char, A::StridedMatrix, ipiv::Vector{BlasInt}) + +""" + sytrs!(uplo, A, ipiv, B) + +Solves the equation `A * X = B` for a symmetric matrix `A` using the +results of `sytrf!`. If `uplo = U`, the upper half of `A` is stored. +If `uplo = L`, the lower half is stored. `B` is overwritten by the +solution `X`. +""" +sytrs!(uplo::Char, A::StridedMatrix, ipiv::Vector{BlasInt}, B::StridedVecOrMat) + + +""" + hesv!(uplo, A, B) -> (B, A, ipiv) + +Finds the solution to `A * X = B` for Hermitian matrix `A`. If `uplo = U`, +the upper half of `A` is stored. If `uplo = L`, the lower half is stored. +`B` is overwritten by the solution `X`. `A` is overwritten by its +Bunch-Kaufman factorization. `ipiv` contains pivoting information about the +factorization. +""" +hesv!(uplo::Char, A::StridedMatrix, B::StridedVecOrMat) + +""" + hetrf!(uplo, A) -> (A, ipiv) + +Computes the Bunch-Kaufman factorization of a Hermitian matrix `A`. If +`uplo = U`, the upper half of `A` is stored. If `uplo = L`, the lower +half is stored. + +Returns `A`, overwritten by the factorization, and a pivot vector. +""" +hetrf!(uplo::Char, A::StridedMatrix) + +""" + hetri!(uplo, A, ipiv) + +Computes the inverse of a Hermitian matrix `A` using the results of +`sytrf!`. If `uplo = U`, the upper half of `A` is stored. If `uplo = L`, +the lower half is stored. `A` is overwritten by its inverse. +""" +hetri!(uplo::Char, A::StridedMatrix, ipiv::Vector{BlasInt}) + +""" + hetrs!(uplo, A, ipiv, B) + +Solves the equation `A * X = B` for a Hermitian matrix `A` using the +results of `sytrf!`. If `uplo = U`, the upper half of `A` is stored. +If `uplo = L`, the lower half is stored. `B` is overwritten by the +solution `X`. +""" +hetrs!(uplo::Char, A::StridedMatrix, ipiv::Vector{BlasInt}, B::StridedVecOrMat) + # Symmetric (real) eigensolvers for (syev, syevr, sygvd, elty) in ((:dsyev_,:dsyevr_,:dsygvd_,:Float64), @@ -3574,6 +4321,44 @@ for (syev, syevr, sygvd, elty, relty) in end end +""" + syev!(jobz, uplo, A) + +Finds the eigenvalues (`jobz = N`) or eigenvalues and eigenvectors +(`jobz = V`) of a symmetric matrix `A`. If `uplo = U`, the upper triangle +of `A` is used. If `uplo = L`, the lower triangle of `A` is used. +""" +syev!(jobz::Char, uplo::Char, A::StridedMatrix) + +""" + syevr!(jobz, range, uplo, A, vl, vu, il, iu, abstol) -> (W, Z) + +Finds the eigenvalues (`jobz = N`) or eigenvalues and eigenvectors +(`jobz = V`) of a symmetric matrix `A`. If `uplo = U`, the upper triangle +of `A` is used. If `uplo = L`, the lower triangle of `A` is used. If +`range = A`, all the eigenvalues are found. If `range = V`, the +eigenvalues in the half-open interval `(vl, vu]` are found. +If `range = I`, the eigenvalues with indices between `il` and `iu` are +found. `abstol` can be set as a tolerance for convergence. + +The eigenvalues are returned in `W` and the eigenvectors in `Z`. +""" +syevr!(jobz::Char, range::Char, uplo::Char, A::StridedMatrix, vl::AbstractFloat, vu::AbstractFloat, il::Integer, iu::Integer, abstol::AbstractFloat) + +""" + sygvd!(jobz, range, uplo, A, vl, vu, il, iu, abstol) -> (w, A, B) + +Finds the generalized eigenvalues (`jobz = N`) or eigenvalues and +eigenvectors (`jobz = V`) of a symmetric matrix `A` and symmetric +positive-definite matrix `B`. If `uplo = U`, the upper triangles +of `A` and `B` are used. If `uplo = L`, the lower triangles of `A` and +`B` are used. If `itype = 1`, the problem to solve is +`A * x = lambda * B * x`. If `itype = 2`, the problem to solve is +`A * B * x = lambda * x`. If `itype = 3`, the problem to solve is +`B * A * x = lambda * x`. +""" +sygvd!(itype::Integer, jobz::Char, uplo::Char, A::StridedMatrix, B::StridedMatrix) + ## (BD) Bidiagonal matrices - singular value decomposition for (bdsqr, relty, elty) in ((:dbdsqr_,:Float64,:Float64), @@ -3581,10 +4366,6 @@ for (bdsqr, relty, elty) in (:zbdsqr_,:Float64,:Complex128), (:cbdsqr_,:Float32,:Complex64)) @eval begin - #*> DBDSQR computes the singular values and, optionally, the right and/or - #*> left singular vectors from the singular value decomposition (SVD) of - #*> a real N-by-N (upper or lower) bidiagonal matrix B using the implicit - #*> zero-shift QR algorithm. function bdsqr!(uplo::Char, d::Vector{$relty}, e_::Vector{$relty}, Vt::StridedMatrix{$elty}, U::StridedMatrix{$elty}, C::StridedMatrix{$elty}) @@ -3631,6 +4412,18 @@ for (bdsqr, relty, elty) in end end +""" + bdsqr!(uplo, d, e_, Vt, U, C) -> (d, Vt, U, C) + +Computes the singular value decomposition of a bidiagonal matrix with +`d` on the diagonal and `e_` on the off-diagonal. If `uplo = U`, `e_` is +the superdiagonal. If `uplo = L`, `e_` is the subdiagonal. Can optionally also +compute the product `Q' * C`. + +Returns the singular values in `d`, and the matrix `C` overwritten with `Q' * C`. +""" +bdsqr!(uplo::Char, d::Vector, e_::Vector, Vt::StridedMatrix, U::StridedMatrix, C::StridedMatrix) + #Defined only for real types for (bdsdc, elty) in ((:dbdsdc_,:Float64), @@ -3687,7 +4480,21 @@ for (bdsdc, elty) in end end -# Estimate condition number +""" + bdsdc!(uplo, compq, d, e_) -> (d, e, u, vt, q, iq) + +Computes the singular value decomposition of a bidiagonal matrix with `d` on the +diagonal and `e_` on the off-diagonal using a divide and conqueq method. +If `uplo = U`, `e_` is the superdiagonal. If `uplo = L`, `e_` is the subdiagonal. +If `compq = N`, only the singular values are found. If `compq = I`, the singular +values and vectors are found. If `compq = P`, the singular values +and vectors are found in compact form. Only works for real types. + +Returns the singular values in `d`, and if `compq = P`, the compact singular +vectors in `iq`. +""" +bdsdc!(uplo::Char, compq::Char, d::Vector, e_::Vector) + for (gecon, elty) in ((:dgecon_,:Float64), (:sgecon_,:Float32)) @@ -3755,7 +4562,16 @@ for (gecon, elty, relty) in end end -# Hessenberg form +""" + gecon!(normtype, A, anorm) + +Finds the reciprocal condition number of matrix `A`. If `normtype = I`, +the condition number is found in the infinity norm. If `normtype = O` or +`1`, the condition number is found in the one norm. `A` must be the +result of `getrf!` and `anorm` is the norm of `A` in the relevant norm. +""" +gecon!(normtype::Char, A::StridedMatrix, anorm) + for (gehrd, elty) in ((:dgehrd_,:Float64), (:sgehrd_,:Float32), @@ -3794,7 +4610,16 @@ for (gehrd, elty) in end gehrd!(A::StridedMatrix) = gehrd!(1, size(A, 1), A) -# construct Q from Hessenberg +""" + gehrd!(ilo, ihi, A) -> (A, tau) + +Converts a matrix `A` to Hessenberg form. If `A` is balanced with `gebal!` +then `ilo` and `ihi` are the outputs of `gebal!`. Otherwise they should be +`ilo = 1` and `ihi = size(A,2)`. `tau` contains the elementary reflectors of +the factorization. +""" +gehrd!(ilo::Integer, ihi::Integer, A::StridedMatrix) + for (orghr, elty) in ((:dorghr_,:Float64), (:sorghr_,:Float32), @@ -3833,7 +4658,15 @@ for (orghr, elty) in end end end -# Schur forms + +""" + orghr!(ilo, ihi, A, tau) + +Explicitly finds `Q`, the orthogonal/unitary matrix from `gehrd!`. `ilo`, +`ihi`, `A`, and `tau` must correspond to the input/output to `gehrd!`. +""" +orghr!(ilo::Integer, ihi::Integer, A::StridedMatrix, tau::StridedVector) + for (gees, gges, elty) in ((:dgees_,:dgges_,:Float64), (:sgees_,:sgges_,:Float32)) @@ -4018,7 +4851,31 @@ for (gees, gges, elty, relty) in end end end -# Reorder Schur forms + +""" + gees!(jobvs, A) -> (A, vs, w) + +Computes the eigenvalues (`jobvs = N`) or the eigenvalues and Schur +vectors (`jobvs = V`) of matrix `A`. `A` is overwritten by its Schur form. + +Returns `A`, `vs` containing the Schur vectors, and `w`, containing the +eigenvalues. +""" +gees!(jobvs::Char, A::StridedMatrix) + + +""" + gges!(jobvsl, jobvsr, A, B) -> (A, B, alpha, beta, vsl, vsr) + +Computes the generalized eigenvalues, generalized Schur form, left Schur +vectors (`jobsvl = V`), or right Schur vectors (`jobvsr = V`) of `A` and +`B`. + +The generalized eigenvalues are returned in `alpha` and `beta`. The left Schur +vectors are returned in `vsl` and the right Schur vectors are returned in `vsr`. +""" +gges!(jobvsl::Char, jobvsr::Char, A::StridedMatrix, B::StridedMatrix) + for (trexc, trsen, tgsen, elty) in ((:dtrexc_, :dtrsen_, :dtgsen_, :Float64), (:strexc_, :strsen_, :stgsen_, :Float32)) @@ -4309,7 +5166,39 @@ for (trexc, trsen, tgsen, elty) in end end -# Solves the real Sylvester matrix equation: op(A)*X +- X*op(B) = scale*C and A and B are both upper quasi triangular. +""" + trexc!(compq, ifst, ilst, T, Q) -> (T, Q) + +Reorder the Schur factorization of a matrix. If `compq = V`, the Schur +vectors `Q` are reordered. If `compq = N` they are not modified. `ifst` +and `ilst` specify the reordering of the vectors. +""" +trexc!(compq::Char, ifst::BlasInt, ilst::BlasInt, T::StridedMatrix, Q::StridedMatrix) + +""" + trsen!(compq, job, select, T, Q) -> (T, Q, w) + +Reorder the Schur factorization of a matrix and optionally finds reciprocal +condition numbers. If `job = N`, no condition numbers are found. If `job = E`, +only the condition number for this cluster of eigenvalues is found. If +`job = V`, only the condition number for the invariant subspace is found. +If `job = B` then the condition numbers for the cluster and subspace are +found. If `compq = V` the Schur vectors `Q` are updated. If `compq = N` +the Schur vectors are not modified. `select` determines which +eigenvalues are in the cluster. + +Returns `T`, `Q`, and reordered eigenvalues in `w`. +""" +trsen!(compq::Char, job::Char, select::StridedVector{BlasInt}, T::StridedMatrix, Q::StridedMatrix) + +""" + tgsen!(select, S, T, Q, Z) -> (S, T, alpha, beta, Q, Z) + +Reorders the vectors of a generalized Schur decomposition. `select` specifices +the eigenvalues in each cluster. +""" +tgsen!(select::StridedVector{BlasInt}, S::StridedMatrix, T::StridedMatrix, Q::StridedMatrix, Z::StridedMatrix) + for (fn, elty, relty) in ((:dtrsyl_, :Float64, :Float64), (:strsyl_, :Float32, :Float32), (:ztrsyl_, :Complex128, :Float64), @@ -4342,5 +5231,18 @@ for (fn, elty, relty) in ((:dtrsyl_, :Float64, :Float64), end end +""" + trsyl!(transa, transb, A, B, C, isgn=1) -> (C, scale) + +Solves the Sylvester matrix equation `A * X +/- X * B = scale*C` where `A` and +`B` are both quasi-upper triangular. If `transa = N`, `A` is not modified. +If `transa = T`, `A` is transposed. If `transa = C`, `A` is conjugate +transposed. Similarly for `transb` and `B`. If `isgn = 1`, the equation +`A * X + X * B = scale * C` is solved. If `isgn = -1`, the equation +`A * X - X * B = scale * C` is solved. + +Returns `X` (overwriting `C`) and `scale`. +""" +trsyl!(transa::Char, transb::Char, A::StridedMatrix, B::StridedMatrix, C::StridedMatrix, isgn::Int=1) end # module diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 126e13dd99cae..57103fadc3aaf 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -1265,3 +1265,562 @@ Usually a function has 4 methods defined, one each for ``Float64``, .. data:: I An object of type ``UniformScaling``, representing an identity matrix of any size. + +LAPACK Functions +-------------- + +.. module:: Base.LinAlg.LAPACK + +:mod:`Base.LinAlg.LAPACK` provides wrappers for some of the LAPACK functions for +linear algebra. Those functions that overwrite one of the input +arrays have names ending in ``'!'``. + +Usually a function has 4 methods defined, one each for ``Float64``, +``Float32``, ``Complex128`` and ``Complex64`` arrays. + +.. currentmodule:: Base.LinAlg.LAPACK + +.. function:: gbtrf!(kl, ku, m, AB) -> (AB, ipiv) + + .. Docstring generated from Julia source + + Compute the LU factorization of a banded matrix ``AB``\ . ``kl`` is the first subdiagonal containing a nonzero band, ``ku`` is the last superdiagonal containing one, and ``m`` is the first dimension of the matrix ``AB``\ . Returns the LU factorization in-place and ``ipiv``\ , the vector of pivots used. + +.. function:: gbtrs!(trans, kl, ku, m, AB, ipiv, B) + + .. Docstring generated from Julia source + + Solve the equation ``AB * X = B``\ . ``trans`` determines the orientation of ``AB``\ . It may be ``N`` (no transpose), ``T`` (transpose), or ``C`` (conjugate transpose). ``kl`` is the first subdiagonal containing a nonzero band, ``ku`` is the last superdiagonal containing one, and ``m`` is the first dimension of the matrix ``AB``\ . ``ipiv`` is the vector of pivots returned from ``gbtrf!``\ . Returns the vector or matrix ``X``\ , overwriting ``B`` in-place. + +.. function:: gebal!(job, A) -> (ilo, ihi, scale) + + .. Docstring generated from Julia source + + Balance the matrix ``A`` before computing its eigensystem or Schur factorization. ``job`` can be one of ``N`` (``A`` will not be permuted or scaled), ``P`` (``A`` will only be permuted), ``S`` (``A`` will only be scaled), or ``B`` (``A`` will be both permuted and scaled). Modifies ``A`` in-place and returns ``ilo``\ , ``ihi``\ , and ``scale``\ . If permuting was turned on, ``A[i,j] = 0`` if ``j > i`` and ``1 < j < ilo`` or ``j > ihi``\ . ``scale`` contains information about the scaling/permutations performed. + +.. function:: gebak!(job, side, ilo, ihi, scale, V) + + .. Docstring generated from Julia source + + Transform the eigenvectors ``V`` of a matrix balanced using ``gebal!`` to the unscaled/unpermuted eigenvectors of the original matrix. Modifies ``V`` in-place. ``side`` can be ``L`` (left eigenvectors are transformed) or ``R`` (right eigenvectors are transformed). + +.. function:: gebrd!(A) -> (A, d, e, tauq, taup) + + .. Docstring generated from Julia source + + Reduce ``A`` in-place to bidiagonal form ``A = QBP'``\ . Returns ``A``\ , containing the bidiagonal matrix ``B``\ ; ``d``\ , containing the diagonal elements of ``B``\ ; ``e``\ , containing the off-diagonal elements of ``B``\ ; ``tauq``\ , containing the elementary reflectors representing ``Q``\ ; and ``taup``\ , containing the elementary reflectors representing ``P``\ . + +.. function:: gelqf!(A, tau) + + .. Docstring generated from Julia source + + Compute the ``LQ`` factorization of ``A``\ , ``A = LQ``\ . ``tau`` contains scalars which parameterize the elementary reflectors of the factorization. ``tau`` must have length greater than or equal to the smallest dimension of ``A``\ . + + Returns ``A`` and ``tau`` modified in-place. + +.. function:: geqlf!(A, tau) + + .. Docstring generated from Julia source + + Compute the ``QL`` factorization of ``A``\ , ``A = QL``\ . ``tau`` contains scalars which parameterize the elementary reflectors of the factorization. ``tau`` must have length greater than or equal to the smallest dimension of ``A``\ . + + Returns ``A`` and ``tau`` modified in-place. + +.. function:: geqrf!(A, tau) + + .. Docstring generated from Julia source + + Compute the ``QR`` factorization of ``A``\ , ``A = QR``\ . ``tau`` contains scalars which parameterize the elementary reflectors of the factorization. ``tau`` must have length greater than or equal to the smallest dimension of ``A``\ . + + Returns ``A`` and ``tau`` modified in-place. + +.. function:: geqp3!(A, jpvt, tau) + + .. Docstring generated from Julia source + + Compute the pivoted ``QR`` factorization of ``A``\ , ``AP = QR`` using BLAS level 3. ``P`` is a pivoting matrix, represented by ``jpvt``\ . ``tau`` stores the elementary reflectors. ``jpvt`` must have length length greater than or equal to ``n`` if ``A`` is an ``(m x n)`` matrix. ``tau`` must have length greater than or equal to the smallest dimension of ``A``\ . + + ``A``\ , ``jpvt``\ , and ``tau`` are modified in-place. + +.. function:: gerqf!(A, tau) + + .. Docstring generated from Julia source + + Compute the ``RQ`` factorization of ``A``\ , ``A = RQ``\ . ``tau`` contains scalars which parameterize the elementary reflectors of the factorization. ``tau`` must have length greater than or equal to the smallest dimension of ``A``\ . + + Returns ``A`` and ``tau`` modified in-place. + +.. function:: geqrt!(A, T) + + .. Docstring generated from Julia source + + Compute the blocked ``QR`` factorization of ``A``\ , ``A = QR``\ . ``T`` contains upper triangular block reflectors which parameterize the elementary reflectors of the factorization. The first dimension of ``T`` sets the block size and it must be between 1 and ``n``\ . The second dimension of ``T`` must equal the smallest dimension of ``A``\ . + + Returns ``A`` and ``T`` modified in-place. + +.. function:: geqrt3!(A, T) + + .. Docstring generated from Julia source + + Recursively computes the blocked ``QR`` factorization of ``A``\ , ``A = QR``\ . ``T`` contains upper triangular block reflectors which parameterize the elementary reflectors of the factorization. The first dimension of ``T`` sets the block size and it must be between 1 and ``n``\ . The second dimension of ``T`` must equal the smallest dimension of ``A``\ . + + Returns ``A`` and ``T`` modified in-place. + +.. function:: getrf!(A) -> (A, ipiv, info) + + .. Docstring generated from Julia source + + Compute the pivoted ``LU`` factorization of ``A``\ , ``A = LU``\ . + + Returns ``A``\ , modified in-place, ``ipiv``\ , the pivoting information, and an ``info`` code which indicates success (``info = 0``\ ), a singular value in ``U`` (``info = i``\ , in which case ``U[i,i]`` is singular), or an error code (``info < 0``\ ). + +.. function:: tzrzf!(A) -> (A, tau) + + .. Docstring generated from Julia source + + Transforms the upper trapezoidal matrix ``A`` to upper triangular form in-place. Returns ``A`` and ``tau``\ , the scalar parameters for the elementary reflectors of the transformation. + +.. function:: ormrz!(side, trans, A, tau, C) + + .. Docstring generated from Julia source + + Multiplies the matrix ``C`` by ``Q`` from the transformation supplied by ``tzrzf!``\ . Depending on ``side`` or ``trans`` the multiplication can be left-sided (``side = L, Q*C``\ ) or right-sided (``side = R, C*Q``\ ) and ``Q`` can be unmodified (``trans = N``\ ), transposed (``trans = T``\ ), or conjugate transposed (``trans = C``\ ). Returns matrix ``C`` which is modified in-place with the result of the multiplication. + +.. function:: gels!(trans, A, B) -> (F, B, ssr) + + .. Docstring generated from Julia source + + Solves the linear equation ``A * X = B``\ , ``A.' * X =B``\ , or ``A' * X = B`` using a QR or LQ factorization. Modifies the matrix/vector ``B`` in place with the solution. ``A`` is overwritten with its ``QR`` or ``LQ`` factorization. ``trans`` may be one of ``N`` (no modification), ``T`` (transpose), or ``C`` (conjugate transpose). ``gels!`` searches for the minimum norm/least squares solution. ``A`` may be under or over determined. The solution is returned in ``B``\ . + +.. function:: gesv!(A, B) -> (B, A, ipiv) + + .. Docstring generated from Julia source + + Solves the linear equation ``A * X = B`` where ``A`` is a square matrix using the ``LU`` factorization of ``A``\ . ``A`` is overwritten with its ``LU`` factorization and ``B`` is overwritten with the solution ``X``\ . ``ipiv`` contains the pivoting information for the ``LU`` factorization of ``A``\ . + +.. function:: getrs!(trans, A, ipiv, B) + + .. Docstring generated from Julia source + + Solves the linear equation ``A * X = B``\ , ``A.' * X =B``\ , or ``A' * X = B`` for square ``A``\ . Modifies the matrix/vector ``B`` in place with the solution. ``A`` is the ``LU`` factorization from ``getrf!``\ , with ``ipiv`` the pivoting information. ``trans`` may be one of ``N`` (no modification), ``T`` (transpose), or ``C`` (conjugate transpose). + +.. function:: getri!(A, ipiv) + + .. Docstring generated from Julia source + + Computes the inverse of ``A``\ , using its ``LU`` factorization found by ``getrf!``\ . ``ipiv`` is the pivot information output and ``A`` contains the ``LU`` factorization of ``getrf!``\ . ``A`` is overwritten with its inverse. + +.. function:: gesvx!(fact, trans, A, AF, ipiv, equed, R, C, B) -> (X, equed, R, C, B, rcond, ferr, berr, work) + + .. Docstring generated from Julia source + + Solves the linear equation ``A * X = B`` (``trans = N``\ ), ``A.' * X =B`` (``trans = T``\ ), or ``A' * X = B`` (``trans = C``\ ) using the ``LU`` factorization of ``A``\ . ``fact`` may be ``E``\ , in which case ``A`` will be equilibrated and copied to ``AF``\ ; ``F``\ , in which case ``AF`` and ``ipiv`` from a previous ``LU`` factorization are inputs; or ``N``\ , in which case ``A`` will be copied to ``AF`` and then factored. If ``fact = F``\ , ``equed`` may be ``N``\ , meaning ``A`` has not been equilibrated; ``R``\ , meaning ``A`` was multiplied by ``diagm(R)`` from the left; ``C``\ , meaning ``A`` was multiplied by ``diagm(C)`` from the right; or ``B``\ , meaning ``A`` was multiplied by ``diagm(R)`` from the left and ``diagm(C)`` from the right. If ``fact = F`` and ``equed = R`` or ``B`` the elements of ``R`` must all be positive. If ``fact = F`` and ``equed = C`` or ``B`` the elements of ``C`` must all be positive. + + Returns the solution ``X``\ ; ``equed``\ , which is an output if ``fact`` is not ``N``\ , and describes the equilibration that was performed; ``R``\ , the row equilibration diagonal; ``C``\ , the column equilibration diagonal; ``B``\ , which may be overwritten with its equilibrated form ``diagm(R)*B`` (if ``trans = N`` and ``equed = R,B``\ ) or ``diagm(C)*B`` (if ``trans = T,C`` and ``equed = C,B``\ ); ``rcond``\ , the reciprocal condition number of ``A`` after equilbrating; ``ferr``\ , the forward error bound for each solution vector in ``X``\ ; ``berr``\ , the forward error bound for each solution vector in ``X``\ ; and ``work``\ , the reciprocal pivot growth factor. + +.. function:: gesvx!(A, B) + + .. Docstring generated from Julia source + + The no-equilibration, no-transpose simplification of ``gesvx!``\ . + +.. function:: gelsd!(A, B, rcond) -> (B, rnk) + + .. Docstring generated from Julia source + + Computes the least norm solution of ``A * X = B`` by finding the ``SVD`` factorization of ``A``\ , then dividing-and-conquering the problem. ``B`` is overwritten with the solution ``X``\ . Singular values below ``rcond`` will be treated as zero. Returns the solution in ``B`` and the effective rank of ``A`` in ``rnk``\ . + +.. function:: gelsy!(A, B, rcond) -> (B, rnk) + + .. Docstring generated from Julia source + + Computes the least norm solution of ``A * X = B`` by finding the full ``QR`` factorization of ``A``\ , then dividing-and-conquering the problem. ``B`` is overwritten with the solution ``X``\ . Singular values below ``rcond`` will be treated as zero. Returns the solution in ``B`` and the effective rank of ``A`` in ``rnk``\ . + +.. function:: gelsy!(A, B, rcond) -> (B, rnk) + + .. Docstring generated from Julia source + + Computes the least norm solution of ``A * X = B`` by finding the full ``QR`` factorization of ``A``\ , then dividing-and-conquering the problem. ``B`` is overwritten with the solution ``X``\ . Singular values below ``rcond`` will be treated as zero. Returns the solution in ``B`` and the effective rank of ``A`` in ``rnk``\ . + +.. function:: gglse!(A, c, B, d) -> (X,res) + + .. Docstring generated from Julia source + + Solves the equation ``A * x = c`` where ``x`` is subject to the equality constraint ``B * x = d``\ . Uses the formula ``||c - A*x||^2 = 0`` to solve. Returns ``X`` and the residual sum-of-squares. + +.. function:: geev!(jobvl, jobvr, A) -> (W, VL, VR) + + .. Docstring generated from Julia source + + Finds the eigensystem of ``A``\ . If ``jobvl = N``\ , the left eigenvectors of ``A`` aren't computed. If ``jobvr = N``\ , the right eigenvectors of ``A`` aren't computed. If ``jobvl = V`` or ``jobvr = V``\ , the corresponding eigenvectors are computed. Returns the eigenvalues in ``W``\ , the right eigenvectors in ``VR``\ , and the left eigenvectors in ``VL``\ . + +.. function:: gesdd!(job, A) -> (U, S, VT) + + .. Docstring generated from Julia source + + Finds the singular value decomposition of ``A``\ , ``A = U * S * V'``\ , using a divide and conquer approach. If ``job = A``\ , all the columns of ``U`` and the rows of ``V'`` are computed. If ``job = N``\ , no columns of ``U`` or rows of ``V'`` are computed. If ``job = O``\ , ``A`` is overwritten with the columns of (thin) ``U`` and the rows of (thin) ``V'``\ . If ``job = S``\ , the columns of (thin) ``U`` and the rows of (thin) ``V'`` are computed and returned separately. + +.. function:: gesvd!(jobu, jobvt, A) -> (U, S, VT) + + .. Docstring generated from Julia source + + Finds the singular value decomposition of ``A``\ , ``A = U * S * V'``\ . If ``jobu = A``\ , all the columns of ``U`` are computed. If ``jobvt = A`` all the rows of ``V'`` are computed. If ``jobu = N``\ , no columns of ``U`` are computed. If ``jobvt = N`` no rows of ``V'`` are computed. If ``jobu = O``\ , ``A`` is overwritten with the columns of (thin) ``U``\ . If ``jobvt = O``\ , ``A`` is overwritten with the rows of (thin) ``V'``\ . If ``jobu = S``\ , the columns of (thin) ``U`` are computed and returned separately. If ``jobvt = S`` the rows of (thin) ``V'`` are computed and returned separately. ``jobu`` and ``jobvt`` can't both be ``O``\ . + + Returns ``U``\ , ``S``\ , and ``Vt``\ , where ``S`` are the singular values of ``A``\ . + +.. function:: ggsvd!(jobu, jobv, jobq, A, B) -> (U, V, Q, alpha, beta, k, l, R) + + .. Docstring generated from Julia source + + Finds the generalized singular value decomposition of ``A`` and ``B``\ , ``U'*A*Q = D1*R`` and ``V'*B*Q = D2*R``\ . ``D1`` has ``alpha`` on its diagonal and ``D2`` has ``beta`` on its diagonal. If ``jobu = U``\ , the orthogonal/unitary matrix ``U`` is computed. If ``jobv = V`` the orthogonal/unitary matrix ``V`` is computed. If ``jobq = Q``\ , the orthogonal/unitary matrix ``Q`` is computed. If ``job{u,v,q} = N``\ , that matrix is not computed. + +.. function:: geevx!(balanc, jobvl, jobvr, sense, A) -> (A, w, VL, VR, ilo, ihi, scale, abnrm, rconde, rcondv) + + .. Docstring generated from Julia source + + Finds the eigensystem of ``A`` with matrix balancing. If ``jobvl = N``\ , the left eigenvectors of ``A`` aren't computed. If ``jobvr = N``\ , the right eigenvectors of ``A`` aren't computed. If ``jobvl = V`` or ``jobvr = V``\ , the corresponding eigenvectors are computed. If ``balanc = N``\ , no balancing is performed. If ``balanc = P``\ , ``A`` is permuted but not scaled. If ``balanc = S``\ , ``A`` is scaled but not permuted. If ``balanc = B``\ , ``A`` is permuted and scaled. If ``sense = N``\ , no reciprocal condition numbers are computed. If ``sense = E``\ , reciprocal condition numbers are computed for the eigenvalues only. If ``sense = V``\ , reciprocal condition numbers are computed for the right eigenvectors only. If ``sense = B``\ , reciprocal condition numbers are computed for the right eigenvectors and the eigenvectors. If ``sense = E,B``\ , the right and left eigenvectors must be computed. + +.. function:: ggev!(jobvl, jobvr, A, B) -> (alpha, beta, vl, vr) + + .. Docstring generated from Julia source + + Finds the generalized eigendecomposition of ``A`` and ``B``\ . If ``jobvl = N``\ , the left eigenvectors aren't computed. If ``jobvr = N``\ , the right eigenvectors aren't computed. If ``jobvl = V`` or ``jobvr = V``\ , the corresponding eigenvectors are computed. + +.. function:: gtsv!(dl, d, du, B) + + .. Docstring generated from Julia source + + Solves the equation ``A * X = B`` where ``A`` is a tridiagonal matrix with ``dl`` on the subdiagonal, ``d`` on the diagonal, and ``du`` on the superdiagonal. + + Overwrites ``B`` with the solution ``X`` and returns it. + +.. function:: gttrf!(dl, d, du) -> (dl, d, du, du2, ipiv) + + .. Docstring generated from Julia source + + Finds the ``LU`` factorization of a tridiagonal matrix with ``dl`` on the subdiagonal, ``d`` on the diagonal, and ``du`` on the superdiagonal. + + Modifies ``dl``\ , ``d``\ , and ``du`` in-place and returns them and the second superdiagonal ``du2`` and the pivoting vector ``ipiv``\ . + +.. function:: gttrs!(trans, dl, d, du, du2, ipiv, B) + + .. Docstring generated from Julia source + + Solves the equation ``A * X = B`` (``trans = N``\ ), ``A.' * X = B`` (``trans = T``\ ), or ``A' * X = B`` (``trans = C``\ ) using the ``LU`` factorization computed by ``gttrf!``\ . ``B`` is overwritten with the solution ``X``\ . + +.. function:: orglq!(A, tau, k = length(tau)) + + .. Docstring generated from Julia source + + Explicitly finds the matrix ``Q`` of a ``LQ`` factorization after calling ``gelqf!`` on ``A``\ . Uses the output of ``gelqf!``\ . ``A`` is overwritten by ``Q``\ . + +.. function:: orgqr!(A, tau, k = length(tau)) + + .. Docstring generated from Julia source + + Explicitly finds the matrix ``Q`` of a ``QR`` factorization after calling ``geqrf!`` on ``A``\ . Uses the output of ``geqrf!``\ . ``A`` is overwritten by ``Q``\ . + +.. function:: orgql!(A, tau, k = length(tau)) + + .. Docstring generated from Julia source + + Explicitly finds the matrix ``Q`` of a ``QL`` factorization after calling ``geqlf!`` on ``A``\ . Uses the output of ``geqlf!``\ . ``A`` is overwritten by ``Q``\ . + +.. function:: orgrq!(A, tau, k = length(tau)) + + .. Docstring generated from Julia source + + Explicitly finds the matrix ``Q`` of a ``RQ`` factorization after calling ``gerqf!`` on ``A``\ . Uses the output of ``gerqf!``\ . ``A`` is overwritten by ``Q``\ . + +.. function:: ormlq!(side, trans, A, tau, C) + + .. Docstring generated from Julia source + + Computes ``Q * C`` (``trans = N``\ ), ``Q.' * C`` (``trans = T``\ ), ``Q' * C`` (``trans = C``\ ) for ``side = L`` or the equivalent right-sided multiplication for ``side = R`` using ``Q`` from a ``LQ`` factorization of ``A`` computed using ``gelqf!``\ . ``C`` is overwritten. + +.. function:: ormqr!(side, trans, A, tau, C) + + .. Docstring generated from Julia source + + Computes ``Q * C`` (``trans = N``\ ), ``Q.' * C`` (``trans = T``\ ), ``Q' * C`` (``trans = C``\ ) for ``side = L`` or the equivalent right-sided multiplication for ``side = R`` using ``Q`` from a ``QR`` factorization of ``A`` computed using ``geqrf!``\ . ``C`` is overwritten. + +.. function:: ormql!(side, trans, A, tau, C) + + .. Docstring generated from Julia source + + Computes ``Q * C`` (``trans = N``\ ), ``Q.' * C`` (``trans = T``\ ), ``Q' * C`` (``trans = C``\ ) for ``side = L`` or the equivalent right-sided multiplication for ``side = R`` using ``Q`` from a ``QL`` factorization of ``A`` computed using ``geqlf!``\ . ``C`` is overwritten. + +.. function:: ormrq!(side, trans, A, tau, C) + + .. Docstring generated from Julia source + + Computes ``Q * C`` (``trans = N``\ ), ``Q.' * C`` (``trans = T``\ ), ``Q' * C`` (``trans = C``\ ) for ``side = L`` or the equivalent right-sided multiplication for ``side = R`` using ``Q`` from a ``RQ`` factorization of ``A`` computed using ``gerqf!``\ . ``C`` is overwritten. + +.. function:: gemqrt!(side, trans, V, T, C) + + .. Docstring generated from Julia source + + Computes ``Q * C`` (``trans = N``\ ), ``Q.' * C`` (``trans = T``\ ), ``Q' * C`` (``trans = C``\ ) for ``side = L`` or the equivalent right-sided multiplication for ``side = R`` using ``Q`` from a ``QR`` factorization of ``A`` computed using ``geqrt!``\ . ``C`` is overwritten. + +.. function:: posv!(uplo, A, B) -> (A, B) + + .. Docstring generated from Julia source + + Finds the solution to ``A * X = B`` where ``A`` is a symmetric or Hermitian positive definite matrix. If ``uplo = U`` the upper Cholesky decomposition of ``A`` is computed. If ``uplo = L`` the lower Cholesky decomposition of ``A`` is computed. ``A`` is overwritten by its Cholesky decomposition. ``B`` is overwritten with the solution ``X``\ . + +.. function:: potrf!(uplo, A) + + .. Docstring generated from Julia source + + Computes the Cholesky (upper if ``uplo = U``\ , lower if ``uplo = L``\ ) decomposition of positive-definite matrix ``A``\ . ``A`` is overwritten and returned with an info code. + +.. function:: potri!(uplo, A) + + .. Docstring generated from Julia source + + Computes the inverse of positive-definite matrix ``A`` after calling ``potrf!`` to find its (upper if ``uplo = U``\ , lower if ``uplo = L``\ ) Cholesky decomposition. + + ``A`` is overwritten by its inverse and returned. + +.. function:: potrs!(uplo, A, B) + + .. Docstring generated from Julia source + + Finds the solution to ``A * X = B`` where ``A`` is a symmetric or Hermitian positive definite matrix whose Cholesky decomposition was computed by ``potrf!``\ . If ``uplo = U`` the upper Cholesky decomposition of ``A`` was computed. If ``uplo = L`` the lower Cholesky decomposition of ``A`` was computed. ``B`` is overwritten with the solution ``X``\ . + +.. function:: pstrf!(uplo, A, tol) -> (A, piv, rank, info) + + .. Docstring generated from Julia source + + Computes the (upper if ``uplo = U``\ , lower if ``uplo = L``\ ) pivoted Cholesky decomposition of positive-definite matrix ``A`` with a user-set tolerance ``tol``\ . ``A`` is overwritten by its Cholesky decomposition. + + Returns ``A``\ , the pivots ``piv``\ , the rank of ``A``\ , and an ``info`` code. If ``info = 0``\ , the factorization succeeded. If ``info = i > 0 `, then `A`` is indefinite or rank-deficient. + +.. function:: ptsv!(D, E, B) + + .. Docstring generated from Julia source + + Solves ``A * X = B`` for positive-definite tridiagonal ``A``\ . ``D`` is the diagonal of ``A`` and ``E`` is the off-diagonal. ``B`` is overwritten with the solution ``X`` and returned. + +.. function:: pttrf!(D, E) + + .. Docstring generated from Julia source + + Computes the LDLt factorization of a positive-definite tridiagonal matrix with ``D`` as diagonal and ``E`` as off-diagonal. ``D`` and ``E`` are overwritten and returned. + +.. function:: pttrs!(D, E, B) + + .. Docstring generated from Julia source + + Solves ``A * X = B`` for positive-definite tridiagonal ``A`` with diagonal ``D`` and off-diagonal ``E`` after computing ``A``\ 's LDLt factorization using ``pttrf!``\ . ``B`` is overwritten with the solution ``X``\ . + +.. function:: trtri!(uplo, diag, A) + + .. Docstring generated from Julia source + + Finds the inverse of (upper if ``uplo = U``\ , lower if ``uplo = L``\ ) triangular matrix ``A``\ . If ``diag = N``\ , ``A`` has non-unit diagonal elements. If ``diag = U``\ , all diagonal elements of ``A`` are one. ``A`` is overwritten with its inverse. + +.. function:: trtrs!(uplo, trans, diag, A, B) + + .. Docstring generated from Julia source + + Solves ``A * X = B`` (``trans = N``\ ), ``A.' * X = B`` (``trans = T``\ ), or ``A' * X = B`` (``trans = C``\ ) for (upper if ``uplo = U``\ , lower if ``uplo = L``\ ) triangular matrix ``A``\ . If ``diag = N``\ , ``A`` has non-unit diagonal elements. If ``diag = U``\ , all diagonal elements of ``A`` are one. ``B`` is overwritten with the solution ``X``\ . + +.. function:: trcon!(norm, uplo, diag, A) + + .. Docstring generated from Julia source + + Finds the reciprocal condition number of (upper if ``uplo = U``\ , lower if ``uplo = L``\ ) triangular matrix ``A``\ . If ``diag = N``\ , ``A`` has non-unit diagonal elements. If ``diag = U``\ , all diagonal elements of ``A`` are one. If ``norm = I``\ , the condition number is found in the infinity norm. If ``norm = O`` or ``1``\ , the condition number is found in the one norm. + +.. function:: trevc!(side, howmny, select, T, VL = similar(T), VR = similar(T)) + + .. Docstring generated from Julia source + + Finds the eigensystem of an upper triangular matrix ``T``\ . If ``side = R``\ , the right eigenvectors are computed. If ``side = L``\ , the left eigenvectors are computed. If ``side = B``\ , both sets are computed. If ``howmny = A``\ , all eigenvectors are found. If ``howmny = B``\ , all eigenvectors are found and backtransformed using ``VL`` and ``VR``\ . If ``howmny = S``\ , only the eigenvectors corresponding to the values in ``select`` are computed. + +.. function:: trrfs!(uplo, trans, diag, A, B, X, Ferr, Berr) -> (Ferr, Berr) + + .. Docstring generated from Julia source + + Estimates the error in the solution to ``A * X = B`` (``trans = N``\ ), ``A.' * X = B`` (``trans = T``\ ), ``A' * X = B`` (``trans = C``\ ) for ``side = L``\ , or the equivalent equations a right-handed ``side = R`` ``X * A`` after computing ``X`` using ``trtrs!``\ . If ``uplo = U``\ , ``A`` is upper triangular. If ``uplo = L``\ , ``A`` is lower triangular. If ``diag = N``\ , ``A`` has non-unit diagonal elements. If ``diag = U``\ , all diagonal elements of ``A`` are one. ``Ferr`` and ``Berr`` are optional inputs. ``Ferr`` is the forward error and ``Berr`` is the backward error, each component-wise. + +.. function:: stev!(job, dv, ev) -> (dv, Zmat) + + .. Docstring generated from Julia source + + Computes the eigensystem for a symmetric tridiagonal matrix with ``dv`` as diagonal and ``ev`` as off-diagonal. If ``job = N`` only the eigenvalues are found and returned in ``dv``\ . If ``job = V`` then the eigenvectors are also found and returned in ``Zmat``\ . + +.. function:: stebz!(range, order, vl, vu, il, iu, abstol, dv, ev) -> (dv, iblock, isplit) + + .. Docstring generated from Julia source + + Computes the eigenvalues for a symmetric tridiagonal matrix with ``dv`` as diagonal and ``ev`` as off-diagonal. If ``range = A``\ , all the eigenvalues are found. If ``range = V``\ , the eigenvalues in the half-open interval ``(vl, vu]`` are found. If ``range = I``\ , the eigenvalues with indices between ``il`` and ``iu`` are found. If ``order = B``\ , eigvalues are ordered within a block. If ``order = E``\ , they are ordered across all the blocks. ``abstol`` can be set as a tolerance for convergence. + +.. function:: stegr!(jobz, range, dv, ev, vl, vu, il, iu) -> (w, Z) + + .. Docstring generated from Julia source + + Computes the eigenvalues (``jobz = N``\ ) or eigenvalues and eigenvectors (``jobz = V``\ ) for a symmetric tridiagonal matrix with ``dv`` as diagonal and ``ev`` as off-diagonal. If ``range = A``\ , all the eigenvalues are found. If ``range = V``\ , the eigenvalues in the half-open interval ``(vl, vu]`` are found. If ``range = I``\ , the eigenvalues with indices between ``il`` and ``iu`` are found. The eigenvalues are returned in ``w`` and the eigenvectors in ``Z``\ . + +.. function:: stein!(dv, ev_in, w_in, iblock_in, isplit_in) + + .. Docstring generated from Julia source + + Computes the eigenvectors for a symmetric tridiagonal matrix with ``dv`` as diagonal and ``ev_in`` as off-diagonal. ``w_in`` specifies the input eigenvalues for which to find corresponding eigenvectors. ``iblock_in`` specifies the submatrices corresponding to the eigenvalues in ``w_in``\ . ``isplit_in`` specifies the splitting points between the submatrix blocks. + +.. function:: syconv!(uplo, A, ipiv) -> (A, work) + + .. Docstring generated from Julia source + + Converts a symmetric matrix ``A`` (which has been factorized into a triangular matrix) into two matrices ``L`` and ``D``\ . If ``uplo = U``\ , ``A`` is upper triangular. If ``uplo = L``\ , it is lower triangular. ``ipiv`` is the pivot vector from the triangular factorization. ``A`` is overwritten by ``L`` and ``D``\ . + +.. function:: sysv!(uplo, A, B) -> (B, A, ipiv) + + .. Docstring generated from Julia source + + Finds the solution to ``A * X = B`` for symmetric matrix ``A``\ . If ``uplo = U``\ , the upper half of ``A`` is stored. If ``uplo = L``\ , the lower half is stored. ``B`` is overwritten by the solution ``X``\ . ``A`` is overwritten by its Bunch-Kaufman factorization. ``ipiv`` contains pivoting information about the factorization. + +.. function:: sytrf!(uplo, A) -> (A, ipiv) + + .. Docstring generated from Julia source + + Computes the Bunch-Kaufman factorization of a symmetric matrix ``A``\ . If ``uplo = U``\ , the upper half of ``A`` is stored. If ``uplo = L``\ , the lower half is stored. + + Returns ``A``\ , overwritten by the factorization, and a pivot vector ``ipiv``\ . + +.. function:: sytri!(uplo, A, ipiv) + + .. Docstring generated from Julia source + + Computes the inverse of a symmetric matrix ``A`` using the results of ``sytrf!``\ . If ``uplo = U``\ , the upper half of ``A`` is stored. If ``uplo = L``\ , the lower half is stored. ``A`` is overwritten by its inverse. + +.. function:: sytrs!(uplo, A, ipiv, B) + + .. Docstring generated from Julia source + + Solves the equation ``A * X = B`` for a symmetric matrix ``A`` using the results of ``sytrf!``\ . If ``uplo = U``\ , the upper half of ``A`` is stored. If ``uplo = L``\ , the lower half is stored. ``B`` is overwritten by the solution ``X``\ . + +.. function:: hesv!(uplo, A, B) -> (B, A, ipiv) + + .. Docstring generated from Julia source + + Finds the solution to ``A * X = B`` for Hermitian matrix ``A``\ . If ``uplo = U``\ , the upper half of ``A`` is stored. If ``uplo = L``\ , the lower half is stored. ``B`` is overwritten by the solution ``X``\ . ``A`` is overwritten by its Bunch-Kaufman factorization. ``ipiv`` contains pivoting information about the factorization. + +.. function:: hetrf!(uplo, A) -> (A, ipiv) + + .. Docstring generated from Julia source + + Computes the Bunch-Kaufman factorization of a Hermitian matrix ``A``\ . If ``uplo = U``\ , the upper half of ``A`` is stored. If ``uplo = L``\ , the lower half is stored. + + Returns ``A``\ , overwritten by the factorization, and a pivot vector. + +.. function:: hetri!(uplo, A, ipiv) + + .. Docstring generated from Julia source + + Computes the inverse of a Hermitian matrix ``A`` using the results of ``sytrf!``\ . If ``uplo = U``\ , the upper half of ``A`` is stored. If ``uplo = L``\ , the lower half is stored. ``A`` is overwritten by its inverse. + +.. function:: hetrs!(uplo, A, ipiv, B) + + .. Docstring generated from Julia source + + Solves the equation ``A * X = B`` for a Hermitian matrix ``A`` using the results of ``sytrf!``\ . If ``uplo = U``\ , the upper half of ``A`` is stored. If ``uplo = L``\ , the lower half is stored. ``B`` is overwritten by the solution ``X``\ . + +.. function:: syev!(jobz, uplo, A) + + .. Docstring generated from Julia source + + Finds the eigenvalues (``jobz = N``\ ) or eigenvalues and eigenvectors (``jobz = V``\ ) of a symmetric matrix ``A``\ . If ``uplo = U``\ , the upper triangle of ``A`` is used. If ``uplo = L``\ , the lower triangle of ``A`` is used. + +.. function:: syevr!(jobz, range, uplo, A, vl, vu, il, iu, abstol) -> (W, Z) + + .. Docstring generated from Julia source + + Finds the eigenvalues (``jobz = N``\ ) or eigenvalues and eigenvectors (``jobz = V``\ ) of a symmetric matrix ``A``\ . If ``uplo = U``\ , the upper triangle of ``A`` is used. If ``uplo = L``\ , the lower triangle of ``A`` is used. If ``range = A``\ , all the eigenvalues are found. If ``range = V``\ , the eigenvalues in the half-open interval ``(vl, vu]`` are found. If ``range = I``\ , the eigenvalues with indices between ``il`` and ``iu`` are found. ``abstol`` can be set as a tolerance for convergence. + + The eigenvalues are returned in ``W`` and the eigenvectors in ``Z``\ . + +.. function:: sygvd!(jobz, range, uplo, A, vl, vu, il, iu, abstol) -> (w, A, B) + + .. Docstring generated from Julia source + + Finds the generalized eigenvalues (``jobz = N``\ ) or eigenvalues and eigenvectors (``jobz = V``\ ) of a symmetric matrix ``A`` and symmetric positive-definite matrix ``B``\ . If ``uplo = U``\ , the upper triangles of ``A`` and ``B`` are used. If ``uplo = L``\ , the lower triangles of ``A`` and ``B`` are used. If ``itype = 1``\ , the problem to solve is ``A * x = lambda * B * x``\ . If ``itype = 2``\ , the problem to solve is ``A * B * x = lambda * x``\ . If ``itype = 3``\ , the problem to solve is ``B * A * x = lambda * x``\ . + +.. function:: bdsqr!(uplo, d, e_, Vt, U, C) -> (d, Vt, U, C) + + .. Docstring generated from Julia source + + Computes the singular value decomposition of a bidiagonal matrix with ``d`` on the diagonal and ``e_`` on the off-diagonal. If ``uplo = U``\ , ``e_`` is the superdiagonal. If ``uplo = L``\ , ``e_`` is the subdiagonal. Can optionally also compute the product ``Q' * C``\ . + + Returns the singular values in ``d``\ , and the matrix ``C`` overwritten with ``Q' * C``\ . + +.. function:: bdsdc!(uplo, compq, d, e_) -> (d, e, u, vt, q, iq) + + .. Docstring generated from Julia source + + Computes the singular value decomposition of a bidiagonal matrix with ``d`` on the diagonal and ``e_`` on the off-diagonal using a divide and conqueq method. If ``uplo = U``\ , ``e_`` is the superdiagonal. If ``uplo = L``\ , ``e_`` is the subdiagonal. If ``compq = N``\ , only the singular values are found. If ``compq = I``\ , the singular values and vectors are found. If ``compq = P``\ , the singular values and vectors are found in compact form. Only works for real types. + + Returns the singular values in ``d``\ , and if ``compq = P``\ , the compact singular vectors in ``iq``\ . + +.. function:: gecon!(normtype, A, anorm) + + .. Docstring generated from Julia source + + Finds the reciprocal condition number of matrix ``A``\ . If ``normtype = I``\ , the condition number is found in the infinity norm. If ``normtype = O`` or ``1``\ , the condition number is found in the one norm. ``A`` must be the result of ``getrf!`` and ``anorm`` is the norm of ``A`` in the relevant norm. + +.. function:: gehrd!(ilo, ihi, A) -> (A, tau) + + .. Docstring generated from Julia source + + Converts a matrix ``A`` to Hessenberg form. If ``A`` is balanced with ``gebal!`` then ``ilo`` and ``ihi`` are the outputs of ``gebal!``\ . Otherwise they should be ``ilo = 1`` and ``ihi = size(A,2)``\ . ``tau`` contains the elementary reflectors of the factorization. + +.. function:: orghr!(ilo, ihi, A, tau) + + .. Docstring generated from Julia source + + Explicitly finds ``Q``\ , the orthogonal/unitary matrix from ``gehrd!``\ . ``ilo``\ , ``ihi``\ , ``A``\ , and ``tau`` must correspond to the input/output to ``gehrd!``\ . + +.. function:: gees!(jobvs, A) -> (A, vs, w) + + .. Docstring generated from Julia source + + Computes the eigenvalues (``jobvs = N``\ ) or the eigenvalues and Schur vectors (``jobvs = V``\ ) of matrix ``A``\ . ``A`` is overwritten by its Schur form. + + Returns ``A``\ , ``vs`` containing the Schur vectors, and ``w``\ , containing the eigenvalues. + +.. function:: gges!(jobvsl, jobvsr, A, B) -> (A, B, alpha, beta, vsl, vsr) + + .. Docstring generated from Julia source + + Computes the generalized eigenvalues, generalized Schur form, left Schur vectors (``jobsvl = V``\ ), or right Schur vectors (``jobvsr = V``\ ) of ``A`` and ``B``\ . + + The generalized eigenvalues are returned in ``alpha`` and ``beta``\ . The left Schur vectors are returned in ``vsl`` and the right Schur vectors are returned in ``vsr``\ . + +.. function:: trexc!(compq, ifst, ilst, T, Q) -> (T, Q) + + .. Docstring generated from Julia source + + Reorder the Schur factorization of a matrix. If ``compq = V``\ , the Schur vectors ``Q`` are reordered. If ``compq = N`` they are not modified. ``ifst`` and ``ilst`` specify the reordering of the vectors. + +.. function:: trsen!(compq, job, select, T, Q) -> (T, Q, w) + + .. Docstring generated from Julia source + + Reorder the Schur factorization of a matrix and optionally finds reciprocal condition numbers. If ``job = N``\ , no condition numbers are found. If ``job = E``\ , only the condition number for this cluster of eigenvalues is found. If ``job = V``\ , only the condition number for the invariant subspace is found. If ``job = B`` then the condition numbers for the cluster and subspace are found. If ``compq = V`` the Schur vectors ``Q`` are updated. If ``compq = N`` the Schur vectors are not modified. ``select`` determines which eigenvalues are in the cluster. + + Returns ``T``\ , ``Q``\ , and reordered eigenvalues in ``w``\ . + +.. function:: tgsen!(select, S, T, Q, Z) -> (S, T, alpha, beta, Q, Z) + + .. Docstring generated from Julia source + + Reorders the vectors of a generalized Schur decomposition. ``select`` specifices the eigenvalues in each cluster. + +.. function:: trsyl!(transa, transb, A, B, C, isgn=1) -> (C, scale) + + .. Docstring generated from Julia source + + Solves the Sylvester matrix equation ``A * X +/- X * B = scale*C`` where ``A`` and ``B`` are both quasi-upper triangular. If ``transa = N``\ , ``A`` is not modified. If ``transa = T``\ , ``A`` is transposed. If ``transa = C``\ , ``A`` is conjugate transposed. Similarly for ``transb`` and ``B``\ . If ``isgn = 1``\ , the equation ``A * X + X * B = scale * C`` is solved. If ``isgn = -1``\ , the equation ``A * X - X * B = scale * C`` is solved. + + Returns ``X`` (overwriting ``C``\ ) and ``scale``\ . + From 35c5234ad0b970129bb8c2a10fd8d5080578c46d Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 24 Sep 2015 20:39:27 -0400 Subject: [PATCH 0240/1938] fix regression in error line numbers see e.g. #922 --- src/codegen.cpp | 1 + test/backtrace.jl | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/codegen.cpp b/src/codegen.cpp index 1db2fb5f47b4b..e178a4597c861 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4974,6 +4974,7 @@ static Function *emit_function(jl_lambda_info_t *lam) } if (do_coverage) coverageVisitLine(filename, lno); + ctx.lineno = lno; } if (jl_is_labelnode(stmt)) { if (prevlabel) continue; diff --git a/test/backtrace.jl b/test/backtrace.jl index 4b516dbc0c258..2e6231b1c0f06 100644 --- a/test/backtrace.jl +++ b/test/backtrace.jl @@ -71,3 +71,26 @@ linenum = @__LINE__; f12977(; args...) = () loc = functionloc(f12977) @test endswith(loc[1], "backtrace.jl") @test loc[2] == linenum + +code_loc(p, skipC=true) = ccall(:jl_lookup_code_address, Any, (Ptr{Void},Cint), p, skipC) + +@noinline function test_throw_commoning(x) + if x==1; throw(AssertionError()); end + if x==2; throw(AssertionError()); end +end + +let + local b1, b2 + try + test_throw_commoning(1) + catch + b1 = catch_backtrace() + end + try + test_throw_commoning(2) + catch + b2 = catch_backtrace() + end + @test code_loc(b1[4])[1] == code_loc(b2[4])[1] == :test_throw_commoning + @test code_loc(b1[4])[3] != code_loc(b2[4])[3] +end From 413471e54605ac14ea3ac023e21b67aeef4a61d6 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Thu, 24 Sep 2015 21:30:58 -0700 Subject: [PATCH 0241/1938] Fix testall rule when libdirs move --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 69770e3d5986d..be2d7157dac65 100644 --- a/Makefile +++ b/Makefile @@ -554,7 +554,7 @@ test: check-whitespace $(JULIA_BUILD_MODE) @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test default JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) testall: check-whitespace $(JULIA_BUILD_MODE) - cp $(build_prefix)/lib/julia/sys$(JULIA_LIBSUFFIX).$(SHLIB_EXT) $(BUILDROOT)/local.$(SHLIB_EXT) && $(JULIA_EXECUTABLE) -J $(BUILDROOT)/local.$(SHLIB_EXT) -e 'true' && rm $(BUILDROOT)/local.$(SHLIB_EXT) + cp $(build_private_libdir)/sys$(JULIA_LIBSUFFIX).$(SHLIB_EXT) $(BUILDROOT)/local.$(SHLIB_EXT) && $(JULIA_EXECUTABLE) -J $(BUILDROOT)/local.$(SHLIB_EXT) -e 'true' && rm $(BUILDROOT)/local.$(SHLIB_EXT) @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test all JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) testall1: check-whitespace $(JULIA_BUILD_MODE) From 51b6d24dfbc64d746cf592595b6532eaa5ca68aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Farneb=C3=83=C2=A4ck?= Date: Fri, 25 Sep 2015 13:57:51 +0200 Subject: [PATCH 0242/1938] Bugfix in vcat. --- base/abstractarray.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index d460af2ed1632..3fde6b72ed129 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -738,7 +738,7 @@ function hcat(X::Number...) end function vcat{T}(V::AbstractVector{T}...) - n = 0 + n::Int = 0 for Vk in V n += length(Vk) end From d5edd1148a9ef9637792bfbb61d6d92e3463d3f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Farneb=C3=83=C2=A4ck?= Date: Fri, 25 Sep 2015 14:39:08 +0200 Subject: [PATCH 0243/1938] Test case for issue 13315. --- test/abstractarray.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 58503c6cde14a..bd152586b9abd 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -452,6 +452,12 @@ function test_vcat_depwarn(::Type{TestAbstractArray}) end end +# Issue 13315 +function test_13315(::Type{TestAbstractArray}) + U = UInt(1):UInt(2) + @test [U;[U;]] == [UInt(1), UInt(2), UInt(1), UInt(2)] +end + #----- run tests -------------------------------------------------------------# for T in (T24Linear, TSlow), shape in ((24,), (2, 12), (2,3,4), (1,2,3,4), (4,3,2,1)) @@ -471,3 +477,4 @@ test_map(TestAbstractArray) test_map_promote(TestAbstractArray) test_UInt_indexing(TestAbstractArray) test_vcat_depwarn(TestAbstractArray) +test_13315(TestAbstractArray) From 14e8880ffdee9bed0b53b6b7ac217eb3138155d0 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Fri, 25 Sep 2015 17:20:01 -0400 Subject: [PATCH 0244/1938] docs: fix missing newline after code-block directive [skip ci] --- doc/stdlib/linalg.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index de07af4ee8a13..5807f72b04547 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -800,6 +800,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f Compute the matrix exponential of ``A``, defined by .. math:: + e^A = \sum_{n=0}^{\infty} \frac{A^n}{n!}. For symmetric or Hermitian ``A``, an eigendecomposition (:func:`eigfact`) is used, otherwise the scaling and squaring algorithm (see [H05]_) is chosen. From 262d6babf634181c456868182f972a90240a5863 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Fri, 25 Sep 2015 17:30:26 -0400 Subject: [PATCH 0245/1938] docs: fix missing newline at the source (cf 14e8880ffdee9bed0b53b6) --- base/docs/helpdb.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index fb9cfe134b367..23dbcac1673c6 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -7846,6 +7846,7 @@ doc""" Compute the matrix exponential of ``A``, defined by .. math:: + e^A = \sum_{n=0}^{\infty} \frac{A^n}{n!}. For symmetric or Hermitian ``A``, an eigendecomposition (:func:`eigfact`) is used, otherwise the scaling and squaring algorithm (see [H05]_) is chosen. From ce39312237e07c5cbf11b54fdfba3479e9f0de9b Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Fri, 25 Sep 2015 16:04:01 -0700 Subject: [PATCH 0246/1938] package docs: s/INSTALLED/REQUIRE/ this is the only mention of all-caps `INSTALLED` currently in the repo [av skip] --- doc/manual/packages.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/packages.rst b/doc/manual/packages.rst index 2202987b396d6..5432c1fe9bc60 100644 --- a/doc/manual/packages.rst +++ b/doc/manual/packages.rst @@ -143,7 +143,7 @@ same operating system and environment. :func:`Pkg.add` does the following within the package root directory: -1. Adds the name of the package to ``INSTALLED``. +1. Adds the name of the package to ``REQUIRE``. 2. Downloads the package to ``.cache``, then copies the package to the package root directory. 3. Recursively performs step 2 against all the packages listed in the package's ``REQUIRE`` file. 4. Runs :func:`Pkg.build` From 2c64d2c9dcd19a16903ec7e234b1bcf1e1cf1089 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Fri, 25 Sep 2015 18:10:26 -0700 Subject: [PATCH 0247/1938] attempt to fix osx backtrace test failure ref https://github.com/JuliaLang/julia/commit/35c5234ad0b970129bb8c2a10fd8d5080578c46d#commitcomment-13429310 --- test/backtrace.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/backtrace.jl b/test/backtrace.jl index 2e6231b1c0f06..c33bcd65ad8d3 100644 --- a/test/backtrace.jl +++ b/test/backtrace.jl @@ -91,6 +91,9 @@ let catch b2 = catch_backtrace() end - @test code_loc(b1[4])[1] == code_loc(b2[4])[1] == :test_throw_commoning - @test code_loc(b1[4])[3] != code_loc(b2[4])[3] + ind1 = find(:test_throw_commoning .== map(b->code_loc(b)[1], b1)) + ind2 = find(:test_throw_commoning .== map(b->code_loc(b)[1], b2)) + @test !isempty(ind1) + @test !isempty(ind2) + @test code_loc(b1[ind1])[3] != code_loc(b2[ind2])[3] end From 70e69635bde059cd35a224a003f114973a53cd5f Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Sat, 26 Sep 2015 22:51:46 -0400 Subject: [PATCH 0248/1938] Fix encoding of DFT docs --- base/dft.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/dft.jl b/base/dft.jl index f0eea127eaed5..95bdaf2c34059 100644 --- a/base/dft.jl +++ b/base/dft.jl @@ -91,12 +91,12 @@ To apply ``P`` to an array ``A``, use ``P * A``; in general, the syntax for applying plans is much like that of matrices. (A plan can only be applied to arrays of the same size as the ``A`` for which the plan was created.) You can also apply a plan with a -preallocated output array ``�`` by calling ``A_mul_B!(�, plan, +preallocated output array ``Â`` by calling ``A_mul_B!(Â, plan, A)``. You can compute the inverse-transform plan by ``inv(P)`` and -apply the inverse plan with ``P \ �`` (the inverse plan is cached +apply the inverse plan with ``P \ Â`` (the inverse plan is cached and reused for subsequent calls to ``inv`` or ``\``), and apply the inverse plan to a pre-allocated output array ``A`` with -``A_ldiv_B!(A, P, �)``. +``A_ldiv_B!(A, P, Â)``. The ``flags`` argument is a bitwise-or of FFTW planner flags, defaulting to ``FFTW.ESTIMATE``. e.g. passing ``FFTW.MEASURE`` or ``FFTW.PATIENT`` From 4e992510187b4b9e1031350843c5ec3f98a5c963 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Sat, 26 Sep 2015 22:54:57 -0400 Subject: [PATCH 0249/1938] Search unexported modules for docstrings Some docstrings are for things that end up getting exported from base, but live in unexported modules (like Base.DFT) --- doc/genstdlib.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/genstdlib.jl b/doc/genstdlib.jl index f0054b249ac1e..ceec399c10729 100644 --- a/doc/genstdlib.jl +++ b/doc/genstdlib.jl @@ -53,7 +53,7 @@ function add_all_docs_mod(m::Module) try add_all_docs_meta(m,Docs.meta(m)) end - for name in names(m) + for name in names(m, true) try sub_m = getfield(m,name) isa(sub_m, Module) || continue From 447ae0edecf451b32da76efcbf2c2db6e7376fcd Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Sat, 26 Sep 2015 22:55:56 -0400 Subject: [PATCH 0250/1938] Run genstdlib, remove duplicate gelsy! doc, fix ifft docstring to match doc [ci skip] --- base/dft.jl | 9 +-------- doc/stdlib/io-network.rst | 9 ++++++++- doc/stdlib/linalg.rst | 6 ------ 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/base/dft.jl b/base/dft.jl index 95bdaf2c34059..50151561e0579 100644 --- a/base/dft.jl +++ b/base/dft.jl @@ -395,7 +395,7 @@ fftshift(x,dim) ifftshift(x) = circshift(x, div([size(x)...],-2)) doc""" - ifftshift(x) + ifftshift(x, [dim]) Undoes the effect of `fftshift`. """ @@ -407,13 +407,6 @@ function ifftshift(x,dim) circshift(x, s) end -doc""" - ifftshift(x, [dim]) - -Undoes the effect of `fftshift`. -""" -ifftshift - ############################################################################## # FFTW module (may move to an external package at some point): diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index f273319517e74..30fcdedf32124 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -107,7 +107,14 @@ General I/O .. Docstring generated from Julia source - Write the canonical binary representation of a value to the given stream. + Write the canonical binary representation of a value to the given stream. Returns the number of bytes written into the stream. + + You can write multiple values with the same :func:``write`` call. i.e. the following are equivalent: + + .. code-block:: julia + + write(stream, x, y...) + write(stream, x) + write(stream, y...) .. function:: read(stream, type) diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 5807f72b04547..d909bab6b0a7a 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -1479,12 +1479,6 @@ Usually a function has 4 methods defined, one each for ``Float64``, Computes the least norm solution of ``A * X = B`` by finding the full ``QR`` factorization of ``A``\ , then dividing-and-conquering the problem. ``B`` is overwritten with the solution ``X``\ . Singular values below ``rcond`` will be treated as zero. Returns the solution in ``B`` and the effective rank of ``A`` in ``rnk``\ . -.. function:: gelsy!(A, B, rcond) -> (B, rnk) - - .. Docstring generated from Julia source - - Computes the least norm solution of ``A * X = B`` by finding the full ``QR`` factorization of ``A``\ , then dividing-and-conquering the problem. ``B`` is overwritten with the solution ``X``\ . Singular values below ``rcond`` will be treated as zero. Returns the solution in ``B`` and the effective rank of ``A`` in ``rnk``\ . - .. function:: gglse!(A, c, B, d) -> (X,res) .. Docstring generated from Julia source From 1b1e6c612efbf39949f63d4d56ae9b467ff8d703 Mon Sep 17 00:00:00 2001 From: Michael Hatherly Date: Sun, 27 Sep 2015 10:06:00 +0200 Subject: [PATCH 0251/1938] Minor docsystem refactoring Combines the `doc(::Function)` and `doc(::DataType)` methods since they were nearly identical. Enables searching for docstrings attached to specific constructors in the same manner as is currently allowed for normal methods. Prior to this it resulted in a no method error. Fixes #12701 where splatted method docs were shadowing others. Although not the original cause of the issue, with the change to storing signatures as `Tuple{...}` (in #12835) this meant that `f(x...)` was stored as `Tuple{Vararg{Any}}` which is equal to `Tuple`. Since the default value of `sig` was `Tuple` not all matching docs were returned in the example provided in #12701. --- base/docs/Docs.jl | 81 ++++++++++++++++++----------------------------- 1 file changed, 31 insertions(+), 50 deletions(-) diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index ce82dc069bc92..2067bf9574e4c 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -160,6 +160,11 @@ function doc(b::Binding) `$(b.mod === Main ? b.var : join((b.mod, b.var),'.'))` is $(isgeneric(v) ? "a generic" : "an anonymous") `Function`. """), functionsummary(v)) + elseif isa(v,DataType) + d = catdoc(Markdown.parse(""" + No documentation found. + + """), typesummary(v)) else T = typeof(v) d = catdoc(Markdown.parse(""" @@ -234,37 +239,6 @@ function doc!(f::Function, sig::ANY, data, source) fd.source[sig] = source end -doc(f::Function) = doc(f, Tuple) - -function doc(f::Function, sig::Type) - isgeneric(f) && isempty(methods(f,sig)) && return nothing - results, funcdocs = [], [] - for mod in modules - if (haskey(meta(mod), f) && isa(meta(mod)[f], FuncDoc)) - fd = meta(mod)[f] - push!(funcdocs, fd) - for msig in fd.order - # try to find specific matching method signatures - if sig <: msig - push!(results, (msig, fd.meta[msig])) - end - end - end - end - # if all method signatures are Union{} ( ⊥ ), concat all docstrings - if isempty(results) - for fd in funcdocs - append!(results, [fd.meta[msig] for msig in reverse(fd.order)]) - end - else - sort!(results, lt = (a, b) -> type_morespecific(first(a), first(b))) - results = [last(r) for r in results] - end - catdoc(results...) -end -doc(f::Function,args::Any...) = doc(f, Tuple{args...}) - - """ `catdoc(xs...)`: Combine the documentation metadata `xs` into a single meta object. """ @@ -317,32 +291,39 @@ function doc!(T::DataType, sig::ANY, data, source) td.meta[sig] = data end -function doc(T::DataType) - docs = [] - for mod in modules - if haskey(meta(mod), T) - Td = meta(mod)[T] - if isa(Td, TypeDoc) - if length(docs) == 0 && Td.main !== nothing - push!(docs, Td.main) +function doc(obj::Base.Callable, sig::Type = Union) + isgeneric(obj) && sig !== Union && isempty(methods(obj, sig)) && return nothing + results, groups = [], [] + for m in modules + if haskey(meta(m), obj) + docs = meta(m)[obj] + if isa(docs, FuncDoc) || isa(docs, TypeDoc) + push!(groups, docs) + for msig in docs.order + if sig <: msig + push!(results, (msig, docs.meta[msig])) + end end - for m in Td.order - push!(docs, Td.meta[m]) + if isempty(results) && docs.main !== nothing + push!(results, (Union{}, docs.main)) end - elseif length(docs) == 0 - return Td + else + push!(results, (Union{}, docs)) end end end - if isempty(docs) - catdoc(Markdown.parse(""" - No documentation found. - - """), typesummary(T)) - else - catdoc(docs...) + # If all method signatures are Union{} ( ⊥ ), concat all docstrings. + if isempty(results) + for group in groups + append!(results, [group.meta[s] for s in reverse(group.order)]) + end + else + sort!(results, lt = (a, b) -> type_morespecific(first(a), first(b))) + results = map(last, results) end + catdoc(results...) end +doc(f::Base.Callable, args::Any...) = doc(f, Tuple{args...}) function typesummary(T::DataType) parts = [ From 8c236e96d243b8396a5901ee22388dd75bf78c1d Mon Sep 17 00:00:00 2001 From: Michael Hatherly Date: Sun, 27 Sep 2015 10:15:53 +0200 Subject: [PATCH 0252/1938] Simplify `@repl` implementation. Enable `apropos` search via help mode by passing a string or regex: help?> "foo" help?> r"foo" This shouldn't interfere with any docstring lookup since we don't encourage documenting specific strings or regex objects. Docs for `@r_str` can still be found by searching for either an empty regex `r""` or `@r_str` directly. Enable specifying method signatures using help?> f(::Float64, ::Int) as well as the current behaviour help?> f(x, y) where `x` and `y` are already defined. This is quite useful when you don't have an instance of the required type. Prior to this an error was thrown in `gen_call_with_extracted_types` when using the `::` syntax. --- base/docs/utils.jl | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/base/docs/utils.jl b/base/docs/utils.jl index 23b4aac734b3b..407de84d0bd7c 100644 --- a/base/docs/utils.jl +++ b/base/docs/utils.jl @@ -110,21 +110,40 @@ end repl_corrections(s) = repl_corrections(STDOUT, s) -macro repl(ex) +macro repl(ex) repl(ex) end + +function repl(s::Symbol) quote - # Fuzzy Searching - $(isexpr(ex, Symbol)) && repl_search($(string(ex))) - if $(isa(ex, Symbol)) && - !(isdefined($(current_module()), $(Expr(:quote, ex))) || - haskey(keywords, $(Expr(:quote, ex)))) - repl_corrections($(string(ex))) - else - if $(isfield(ex) ? :(isa($(esc(ex.args[1])), DataType)) : false) - $(isfield(ex) ? :(fielddoc($(esc(ex.args[1])), $(ex.args[2]))) : nothing) + repl_search($(string(s))) + ($(isdefined(s) || haskey(keywords, s))) || repl_corrections($(string(s))) + $(_repl(s)) + end +end + +isregex(x) = isexpr(x, :macrocall, 2) && x.args[1] == symbol("@r_str") && !isempty(x.args[2]) + +repl(ex::Expr) = isregex(ex) ? :(apropos($ex)) : _repl(ex) + +repl(str::AbstractString) = :(apropos($str)) + +repl(other) = nothing + +function _repl(x) + docs = :(@doc $(esc(x))) + try + # Handles function call syntax where each argument is a symbol. + isexpr(x, :call) && (docs = Base.gen_call_with_extracted_types(doc, x)) + end + if isfield(x) + quote + if isa($(esc(x.args[1])), DataType) + fielddoc($(esc(x.args[1])), $(esc(x.args[2]))) else - $((isa(ex,Symbol) || isfield(ex) || isexpr(ex,:macrocall)) ? :(@doc ($(esc(ex)))) : Base.gen_call_with_extracted_types(doc, ex)) + $docs end end + else + docs end end From 6dd817f43564e49b4400caa3d977e27a09c8111b Mon Sep 17 00:00:00 2001 From: kshyatt Date: Sat, 26 Sep 2015 18:17:41 -0700 Subject: [PATCH 0253/1938] Added missing lapack docs --- base/linalg/lapack.jl | 69 ++++++++++++++++++++++++++++++++++++++++++ doc/stdlib/linalg.rst | 70 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 133 insertions(+), 6 deletions(-) diff --git a/base/linalg/lapack.jl b/base/linalg/lapack.jl index 5a57bfebb7202..fce204749e5b9 100644 --- a/base/linalg/lapack.jl +++ b/base/linalg/lapack.jl @@ -594,17 +594,86 @@ code which indicates success (`info = 0`), a singular value in `U` """ getrf!(A::StridedMatrix, tau::Vector) +""" + gelqf!(A) -> (A, tau) + +Compute the `LQ` factorization of `A`, `A = LQ`. + +Returns `A`, modified in-place, and `tau`, which contains scalars +which parameterize the elementary reflectors of the factorization. +""" gelqf!{T<:BlasFloat}(A::StridedMatrix{T}) = ((m,n)=size(A); gelqf!(A,similar(A,T,min(m,n)))) +""" + geqlf!(A) -> (A, tau) + +Compute the `QL` factorization of `A`, `A = QL`. + +Returns `A`, modified in-place, and `tau`, which contains scalars +which parameterize the elementary reflectors of the factorization. +""" geqlf!{T<:BlasFloat}(A::StridedMatrix{T}) = ((m,n)=size(A); geqlf!(A,similar(A,T,min(m,n)))) +""" + geqrt!(A, nb) -> (A, T) + +Compute the blocked `QR` factorization of `A`, `A = QR`. `nb` sets the block size +and it must be between 1 and `n`, the second dimension of `A`. + +Returns `A`, modified in-place, and `T`, which contains upper +triangular block reflectors which parameterize the elementary reflectors of +the factorization. +""" geqrt!{T<:BlasFloat}(A::StridedMatrix{T}, nb::Integer) = geqrt!(A,similar(A,T,nb,minimum(size(A)))) +""" + geqrt3!(A) -> (A, T) + +Recursively computes the blocked `QR` factorization of `A`, `A = QR`. + +Returns `A`, modified in-place, and `T`, which contains upper triangular block +reflectors which parameterize the elementary reflectors of the factorization. +""" geqrt3!{T<:BlasFloat}(A::StridedMatrix{T}) = (n=size(A,2); geqrt3!(A,similar(A,T,n,n))) +""" + geqrf!(A) -> (A, tau) + +Compute the `QR` factorization of `A`, `A = QR`. + +Returns `A`, modified in-place, and `tau`, which contains scalars +which parameterize the elementary reflectors of the factorization. +""" geqrf!{T<:BlasFloat}(A::StridedMatrix{T}) = ((m,n)=size(A); geqrf!(A,similar(A,T,min(m,n)))) +""" + gerqf!(A) -> (A, tau) + +Compute the `RQ` factorization of `A`, `A = RQ`. + +Returns `A`, modified in-place, and `tau`, which contains scalars +which parameterize the elementary reflectors of the factorization. +""" gerqf!{T<:BlasFloat}(A::StridedMatrix{T}) = ((m,n)=size(A); gerqf!(A,similar(A,T,min(m,n)))) +""" + geqp3!(A, jpvt) -> (A, jpvt, tau) + +Compute the pivoted `QR` factorization of `A`, `AP = QR` using BLAS level 3. +`P` is a pivoting matrix, represented by `jpvt`. `jpvt` must have length +greater than or equal to `n` if `A` is an `(m x n)` matrix. + +Returns `A` and `jpvt`, modified in-place, and `tau`, which stores the elementary +reflectors. +""" function geqp3!{T<:BlasFloat}(A::StridedMatrix{T},jpvt::Vector{BlasInt}) m,n = size(A) geqp3!(A,jpvt,similar(A,T,min(m,n))) end + +""" + geqp3!(A) -> (A, jpvt, tau) + +Compute the pivoted `QR` factorization of `A`, `AP = QR` using BLAS level 3. + +Returns `A`, modified in-place, `jpvt`, which represents the pivoting matrix `P`, +and `tau`, which stores the elementary reflectors. +""" function geqp3!{T<:BlasFloat}(A::StridedMatrix{T}) m,n=size(A) geqp3!(A,zeros(BlasInt,n),similar(A,T,min(m,n))) diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 5807f72b04547..c3bda14b968d9 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -1361,6 +1361,14 @@ Usually a function has 4 methods defined, one each for ``Float64``, Returns ``A`` and ``tau`` modified in-place. +.. function:: gelqf!(A) -> (A, tau) + + .. Docstring generated from Julia source + + Compute the ``LQ`` factorization of ``A``\ , ``A = LQ``\ . + + Returns ``A``\ , modified in-place, and ``tau``\ , which contains scalars which parameterize the elementary reflectors of the factorization. + .. function:: geqlf!(A, tau) .. Docstring generated from Julia source @@ -1369,6 +1377,14 @@ Usually a function has 4 methods defined, one each for ``Float64``, Returns ``A`` and ``tau`` modified in-place. +.. function:: geqlf!(A) -> (A, tau) + + .. Docstring generated from Julia source + + Compute the ``QL`` factorization of ``A``\ , ``A = QL``\ . + + Returns ``A``\ , modified in-place, and ``tau``\ , which contains scalars which parameterize the elementary reflectors of the factorization. + .. function:: geqrf!(A, tau) .. Docstring generated from Julia source @@ -1377,6 +1393,14 @@ Usually a function has 4 methods defined, one each for ``Float64``, Returns ``A`` and ``tau`` modified in-place. +.. function:: geqrf!(A) -> (A, tau) + + .. Docstring generated from Julia source + + Compute the ``QR`` factorization of ``A``\ , ``A = QR``\ . + + Returns ``A``\ , modified in-place, and ``tau``\ , which contains scalars which parameterize the elementary reflectors of the factorization. + .. function:: geqp3!(A, jpvt, tau) .. Docstring generated from Julia source @@ -1385,6 +1409,22 @@ Usually a function has 4 methods defined, one each for ``Float64``, ``A``\ , ``jpvt``\ , and ``tau`` are modified in-place. +.. function:: geqp3!(A, jpvt) -> (A, jpvt, tau) + + .. Docstring generated from Julia source + + Compute the pivoted ``QR`` factorization of ``A``\ , ``AP = QR`` using BLAS level 3. ``P`` is a pivoting matrix, represented by ``jpvt``\ . ``jpvt`` must have length greater than or equal to ``n`` if ``A`` is an ``(m x n)`` matrix. + + Returns ``A`` and ``jpvt``\ , modified in-place, and ``tau``\ , which stores the elementary reflectors. + +.. function:: geqp3!(A) -> (A, jpvt, tau) + + .. Docstring generated from Julia source + + Compute the pivoted ``QR`` factorization of ``A``\ , ``AP = QR`` using BLAS level 3. + + Returns ``A``\ , modified in-place, ``jpvt``\ , which represents the pivoting matrix ``P``\ , and ``tau``\ , which stores the elementary reflectors. + .. function:: gerqf!(A, tau) .. Docstring generated from Julia source @@ -1393,6 +1433,14 @@ Usually a function has 4 methods defined, one each for ``Float64``, Returns ``A`` and ``tau`` modified in-place. +.. function:: gerqf!(A) -> (A, tau) + + .. Docstring generated from Julia source + + Compute the ``RQ`` factorization of ``A``\ , ``A = RQ``\ . + + Returns ``A``\ , modified in-place, and ``tau``\ , which contains scalars which parameterize the elementary reflectors of the factorization. + .. function:: geqrt!(A, T) .. Docstring generated from Julia source @@ -1401,6 +1449,14 @@ Usually a function has 4 methods defined, one each for ``Float64``, Returns ``A`` and ``T`` modified in-place. +.. function:: geqrt!(A, nb) -> (A, T) + + .. Docstring generated from Julia source + + Compute the blocked ``QR`` factorization of ``A``\ , ``A = QR``\ . ``nb`` sets the block size and it must be between 1 and ``n``\ , the second dimension of ``A``\ . + + Returns ``A``\ , modified in-place, and ``T``\ , which contains upper triangular block reflectors which parameterize the elementary reflectors of the factorization. + .. function:: geqrt3!(A, T) .. Docstring generated from Julia source @@ -1409,6 +1465,14 @@ Usually a function has 4 methods defined, one each for ``Float64``, Returns ``A`` and ``T`` modified in-place. +.. function:: geqrt3!(A) -> (A, T) + + .. Docstring generated from Julia source + + Recursively computes the blocked ``QR`` factorization of ``A``\ , ``A = QR``\ . + + Returns ``A``\ , modified in-place, and ``T``\ , which contains upper triangular block reflectors which parameterize the elementary reflectors of the factorization. + .. function:: getrf!(A) -> (A, ipiv, info) .. Docstring generated from Julia source @@ -1479,12 +1543,6 @@ Usually a function has 4 methods defined, one each for ``Float64``, Computes the least norm solution of ``A * X = B`` by finding the full ``QR`` factorization of ``A``\ , then dividing-and-conquering the problem. ``B`` is overwritten with the solution ``X``\ . Singular values below ``rcond`` will be treated as zero. Returns the solution in ``B`` and the effective rank of ``A`` in ``rnk``\ . -.. function:: gelsy!(A, B, rcond) -> (B, rnk) - - .. Docstring generated from Julia source - - Computes the least norm solution of ``A * X = B`` by finding the full ``QR`` factorization of ``A``\ , then dividing-and-conquering the problem. ``B`` is overwritten with the solution ``X``\ . Singular values below ``rcond`` will be treated as zero. Returns the solution in ``B`` and the effective rank of ``A`` in ``rnk``\ . - .. function:: gglse!(A, c, B, d) -> (X,res) .. Docstring generated from Julia source From 8624221b7cda12b7d7efdc90686ffe4f28811dac Mon Sep 17 00:00:00 2001 From: Katie Hyatt Date: Sun, 27 Sep 2015 14:02:03 -0700 Subject: [PATCH 0254/1938] Added test for gebrd --- test/linalg/lapack.jl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/linalg/lapack.jl b/test/linalg/lapack.jl index 2f68d611546e7..bf0e537065579 100644 --- a/test/linalg/lapack.jl +++ b/test/linalg/lapack.jl @@ -36,7 +36,7 @@ let # Test gglse end end -let # bdsqr & throw for bdsdc +let # gebrd, bdsqr & throw for bdsdc n = 10 for elty in (Float32, Float64) d, e = convert(Vector{elty}, randn(n)), convert(Vector{elty}, randn(n - 1)) @@ -51,6 +51,13 @@ let # bdsqr & throw for bdsdc @test_throws DimensionMismatch LAPACK.bdsqr!('U', d, e, Vt, U, C[1:end - 1, :]) @test_throws ArgumentError LAPACK.bdsdc!('U','Z',d,e) + + A = rand(elty,n,n) + B = copy(A) + B, d, e, tauq, taup = LAPACK.gebrd!(B) + U, Vt, C = eye(elty, n), eye(elty, n), eye(elty, n) + s, _ = LAPACK.bdsqr!('U',d,e[1:n-1],Vt, U, C) + @test s ≈ svdvals(A) end end From 8be556ae29d90a998bb3216fc64996016721ac62 Mon Sep 17 00:00:00 2001 From: Katie Hyatt Date: Sun, 27 Sep 2015 14:02:14 -0700 Subject: [PATCH 0255/1938] Added fallback and error throwing tests --- test/linalg/matmul.jl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/linalg/matmul.jl b/test/linalg/matmul.jl index 0ccdcb0b9f917..02d3fef9f6b30 100644 --- a/test/linalg/matmul.jl +++ b/test/linalg/matmul.jl @@ -66,6 +66,18 @@ C = Array(Int, size(A, 1), size(B, 2)) @test At_mul_B!(C, A, B) == A'*B @test A_mul_Bt!(C, A, B) == A*B' @test At_mul_Bt!(C, A, B) == A'*B' +@test Base.LinAlg.Ac_mul_Bt!(C, A, B) == A'*B.' + +#test DimensionMismatch for generic_matmatmul +@test_throws DimensionMismatch Base.LinAlg.Ac_mul_Bt!(C,A,ones(Int,4,4)) +@test_throws DimensionMismatch Base.LinAlg.Ac_mul_Bt!(C,ones(Int,4,4),B) + +#and for generic_matvecmul +A = rand(5,5) +B = rand(5) +@test_throws DimensionMismatch Base.LinAlg.generic_matvecmul!(zeros(6),'N',A,B) +@test_throws DimensionMismatch Base.LinAlg.generic_matvecmul!(B,'N',A,zeros(6)) + v = [1,2,3] C = Array(Int, 3, 3) @test A_mul_Bt!(C, v, v) == v*v' @@ -73,6 +85,14 @@ vf = map(Float64,v) C = Array(Float64, 3, 3) @test A_mul_Bt!(C, v, v) == v*v' +# fallbacks & such for BlasFloats +A = rand(Float64,6,6) +B = rand(Float64,6,6) +C = zeros(Float64,6,6) +@test Base.LinAlg.At_mul_Bt!(C,A,B) == A.'*B.' +@test Base.LinAlg.A_mul_Bc!(C,A,B) == A*B.' +@test Base.LinAlg.Ac_mul_B!(C,A,B) == A.'*B + # matrix algebra with subarrays of floats (stride != 1) A = reshape(map(Float64,1:20),5,4) Aref = A[1:2:end,1:2:end] From 77c13b0bb22326be369efea357eb5900be10aa74 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sun, 27 Sep 2015 17:46:12 -0400 Subject: [PATCH 0256/1938] Initialze jl_datatype_t::haspadding. Fix #13331 --- src/alloc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/alloc.c b/src/alloc.c index 1903755c884ea..71497e94f41ba 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -543,6 +543,7 @@ jl_datatype_t *jl_new_uninitialized_datatype(size_t nfields, // corruption otherwise. t->fielddesc_type = fielddesc_type; t->nfields = nfields; + t->haspadding = 0; return t; } From bceb3305a85a669ede026efc51d579f6c3e02f13 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sun, 27 Sep 2015 18:31:29 -0400 Subject: [PATCH 0257/1938] Optionally don't skip C frames in process_backtrace Useful such that Cxx can reuse julia's backtrace showing functionality for C++ frames. --- base/replutil.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/replutil.jl b/base/replutil.jl index 07879de96b506..6e4412aaa5256 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -404,20 +404,20 @@ function show_backtrace(io::IO, top_function::Symbol, t::Vector{Any}, set) end # process the backtrace, up to (but not including) top_function -function process_backtrace(process_func::Function, top_function::Symbol, t, set) +function process_backtrace(process_func::Function, top_function::Symbol, t, set; skipC = true) n = 1 lastfile = ""; lastline = -11; lastname = symbol("#"); last_inlinedat_file = ""; last_inlinedat_line = -1 local fname, file, line count = 0 for i = 1:length(t) - lkup = ccall(:jl_lookup_code_address, Any, (Ptr{Void}, Cint), t[i]-1, true) + lkup = ccall(:jl_lookup_code_address, Any, (Ptr{Void}, Cint), t[i]-1, skipC) if lkup === nothing continue end fname, file, line, inlinedat_file, inlinedat_line, fromC = lkup - if fromC; continue; end + if fromC && skipC; continue; end if i == 1 && fname == :error; continue; end if fname == top_function; break; end count += 1 From 34c93320f9e85436bd7fba5a7f5d7b142983baac Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sat, 26 Sep 2015 19:27:49 -0400 Subject: [PATCH 0258/1938] Fix many GC related bugs * Make `jl_linenode_line` non-allocating * Fix missing GC root when calling `jl_fieldref` and `jl_get_nth_field` that can possibly allocate * Add missing write barrier when using heap allocated array as stack buffer * Remove a few allocation from `jl_static_show_x` (the general case still allocate and should be fixed) * Fix missing and extra `JL_GC_POP` * Fix misplaced `JL_GC_PUSH*` --- src/alloc.c | 4 ++-- src/array.c | 3 +++ src/ast.c | 5 +++++ src/builtins.c | 28 ++++++++++++++++++++-------- src/ccall.cpp | 2 +- src/dump.c | 4 ++-- src/interpreter.c | 9 ++++----- src/jltypes.c | 2 ++ src/julia.h | 2 +- 9 files changed, 40 insertions(+), 19 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 71497e94f41ba..5b4dd0d550ee7 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -342,8 +342,8 @@ jl_lambda_info_t *jl_new_lambda_info(jl_value_t *ast, jl_svec_t *sparams, jl_mod if (ast != NULL && jl_is_expr(ast)) { jl_value_t *body1 = skip_meta(jl_lam_body((jl_expr_t*)ast)->args); if (jl_is_linenode(body1)) { - li->file = (jl_sym_t*)jl_fieldref(body1, 0); - li->line = jl_unbox_long(jl_fieldref(body1, 1)); + li->file = jl_linenode_file(body1); + li->line = jl_linenode_line(body1); } else if (jl_is_expr(body1) && ((jl_expr_t*)body1)->head == line_sym) { li->file = (jl_sym_t*)jl_exprarg(body1, 1); li->line = jl_unbox_long(jl_exprarg(body1, 0)); diff --git a/src/array.c b/src/array.c index 5b0124eab36bb..ccd61fe027a47 100644 --- a/src/array.c +++ b/src/array.c @@ -308,9 +308,12 @@ jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_value_t *dims, } else { size_t *adims = &a->nrows; + // jl_fieldref can allocate + JL_GC_PUSH1(&a); for(i=0; i < ndims; i++) { adims[i] = jl_unbox_long(jl_fieldref(dims, i)); } + JL_GC_POP(); } return a; } diff --git a/src/ast.c b/src/ast.c index e9454c8f6ecdb..8242bc3e17932 100644 --- a/src/ast.c +++ b/src/ast.c @@ -482,6 +482,8 @@ static value_t julia_to_scm_(jl_value_t *v) return scmv; } if (jl_typeis(v, jl_linenumbernode_type)) { + // GC Note: jl_fieldref(v, 1) allocates but neither jl_fieldref(v, 0) + // or julia_to_list2 should allocate here value_t args = julia_to_list2(jl_fieldref(v,1), jl_fieldref(v,0)); fl_gc_handle(&args); value_t hd = julia_to_scm_((jl_value_t*)line_sym); @@ -489,6 +491,9 @@ static value_t julia_to_scm_(jl_value_t *v) fl_free_gc_handles(1); return scmv; } + // GC Note: jl_fieldref(v, 0) allocate for LabelNode, GotoNode + // but we don't need a GC root here because julia_to_list2 + // shouldn't allocate in this case. if (jl_typeis(v, jl_labelnode_type)) return julia_to_list2((jl_value_t*)label_sym, jl_fieldref(v,0)); if (jl_typeis(v, jl_gotonode_type)) diff --git a/src/builtins.c b/src/builtins.c index 7dc6319a7aa1d..5519beeaf8a34 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -452,12 +452,17 @@ JL_CALLABLE(jl_f_apply) jl_value_t **newargs; int onstack = (n < jl_page_size/sizeof(jl_value_t*)); JL_GC_PUSHARGS(newargs, onstack ? n : 1); + jl_svec_t *arg_heap = NULL; if (!onstack) { // put arguments on the heap if there are too many - jl_value_t *argarr = (jl_value_t*)jl_alloc_cell_1d(n); - newargs[0] = argarr; - newargs = jl_cell_data(argarr); - } + arg_heap = jl_alloc_svec(n); + newargs[0] = (jl_value_t*)arg_heap; + newargs = jl_svec_data(arg_heap); + } + // GC Note: here we assume that the the return value of `jl_svecref`, + // `jl_cellref` will not be young if `arg_heap` becomes old + // since they are allocated before `arg_heap`. Otherwise, + // we need to add write barrier for !onstack n = 0; for(i=1; i < nargs; i++) { if (jl_is_svec(args[i])) { @@ -468,8 +473,13 @@ JL_CALLABLE(jl_f_apply) } else if (jl_is_tuple(args[i])) { size_t al = jl_nfields(args[i]); - for(j=0; j < al; j++) + for(j=0; j < al; j++) { + // jl_fieldref may allocate. newargs[n++] = jl_fieldref(args[i], j); + if (arg_heap) { + jl_gc_wb(arg_heap, newargs[n - 1]); + } + } } else { size_t al = jl_array_len(args[i]); @@ -1502,7 +1512,8 @@ size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth) n += jl_printf(out, ")"); } else if (jl_is_linenode(v)) { - n += jl_printf(out, "# line %"PRIuPTR" %s", jl_linenode_line(v), jl_linenode_file(v)->name); + n += jl_printf(out, "# line %"PRIuPTR" %s", + jl_linenode_line(v), jl_linenode_file(v)->name); } else if (jl_is_expr(v)) { jl_expr_t *e = (jl_expr_t*)v; @@ -1545,8 +1556,8 @@ size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth) else if (jl_typeis(v,jl_loaderror_type)) { n += jl_printf(out, "LoadError(at "); n += jl_static_show_x(out, jl_fieldref(v, 0), depth); - n += jl_printf(out, " line "); - n += jl_static_show_x(out, jl_fieldref(v, 1), depth); + // Access the field directly to avoid allocation + n += jl_printf(out, " line %" PRIdPTR, ((intptr_t*)v)[1]); n += jl_printf(out, ": "); n += jl_static_show_x(out, jl_fieldref(v, 2), depth); n += jl_printf(out, ")"); @@ -1579,6 +1590,7 @@ size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth) //jl_fielddesc_t f = t->fields[i]; n += jl_printf(out, "="); } + // FIXME: this line can allocate fldval = jl_get_nth_field(v, i); n += jl_static_show_x(out, fldval, depth); if (istuple && tlen==1) diff --git a/src/ccall.cpp b/src/ccall.cpp index e460462059149..da7d4a86418c7 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -883,7 +883,6 @@ static std::string generate_func_sig( t = julia_struct_to_llvm(tti); if (t == NULL || t == T_void) { - JL_GC_POP(); std::stringstream msg; msg << "ccall: the type of argument "; msg << i+1; @@ -1283,6 +1282,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) bool needroot = false; if (jl_is_abstract_ref_type(jargty)) { if (addressOf) { + JL_GC_POP(); emit_error("ccall: & on a Ref{T} argument is invalid", ctx); return jl_cgval_t(); } diff --git a/src/dump.c b/src/dump.c index 91021482bcec0..e87b27aac7184 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1027,8 +1027,8 @@ void jl_serialize_dependency_list(ios_t *s) unique_func = jl_get_global(jl_base_module, jl_symbol("unique")); jl_array_t *udeps = deps && unique_func ? (jl_array_t *) jl_apply((jl_function_t*)unique_func, (jl_value_t**)&deps, 1) : NULL; + JL_GC_PUSH1(&udeps); if (udeps) { - JL_GC_PUSH1(&udeps); size_t l = jl_array_len(udeps); for (size_t i=0; i < l; i++) { jl_value_t *dep = jl_fieldref(jl_cellref(udeps, i), 0); @@ -1051,8 +1051,8 @@ void jl_serialize_dependency_list(ios_t *s) write_float64(s, jl_unbox_float64(jl_fieldref(deptuple, 1))); } write_int32(s, 0); // terminator, for ease of reading - JL_GC_POP(); } + JL_GC_POP(); } // --- deserialize --- diff --git a/src/interpreter.c b/src/interpreter.c index e4e12e0aa5495..df6da9788cefe 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -497,10 +497,9 @@ static jl_value_t *eval(jl_value_t *e, jl_value_t **locals, size_t nl, size_t ng return (jl_value_t*)jl_nothing; } -static int label_idx(jl_value_t *tgt, jl_array_t *stmts) +static int label_idx(long ltgt, jl_array_t *stmts) { size_t j; - long ltgt = jl_unbox_long(tgt); for(j=0; j < stmts->nrows; j++) { jl_value_t *l = jl_cellref(stmts,j); if (jl_is_labelnode(l) && jl_labelnode_label(l)==ltgt) @@ -538,7 +537,7 @@ static jl_value_t *eval_body(jl_array_t *stmts, jl_value_t **locals, size_t nl, while (1) { jl_value_t *stmt = jl_cellref(stmts,i); if (jl_is_gotonode(stmt)) { - i = label_idx(jl_fieldref(stmt,0), stmts); + i = label_idx(jl_gotonode_label(stmt), stmts); continue; } if (jl_is_expr(stmt)) { @@ -546,7 +545,7 @@ static jl_value_t *eval_body(jl_array_t *stmts, jl_value_t **locals, size_t nl, if (head == goto_ifnot_sym) { jl_value_t *cond = eval(jl_exprarg(stmt,0), locals, nl, ngensym); if (cond == jl_false) { - i = label_idx(jl_exprarg(stmt,1), stmts); + i = label_idx(jl_unbox_long(jl_exprarg(stmt, 1)), stmts); continue; } else if (cond != jl_true) { @@ -571,7 +570,7 @@ static jl_value_t *eval_body(jl_array_t *stmts, jl_value_t **locals, size_t nl, if (jl_exception_in_transit == jl_stackovf_exception) _resetstkoflw(); #endif - i = label_idx(jl_exprarg(stmt,0), stmts); + i = label_idx(jl_unbox_long(jl_exprarg(stmt,0)), stmts); continue; } } diff --git a/src/jltypes.c b/src/jltypes.c index 7e62e8d383bdb..9c76d07c20414 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2217,6 +2217,8 @@ static jl_value_t *inst_tuple_w_(jl_value_t *t, jl_value_t **env, size_t n, for(i=0; i < ntp; i++) { jl_value_t *elt = jl_svecref(tp, i); iparams[i] = (jl_value_t*)inst_type_w_(elt, env, n, stack, 0); + if (ip_heap) + jl_gc_wb(ip_heap, iparams[i]); jl_value_t *pi = iparams[i]; check_tuple_parameter(pi, i, ntp); if (!isabstract && !jl_is_leaf_type(pi)) { diff --git a/src/julia.h b/src/julia.h index ea9416c66e946..f3b224edc36f6 100644 --- a/src/julia.h +++ b/src/julia.h @@ -682,7 +682,7 @@ STATIC_INLINE jl_value_t *jl_cellset(void *a, size_t i, void *x) #define jl_symbolnode_sym(s) ((jl_sym_t*)jl_fieldref(s,0)) #define jl_symbolnode_type(s) (jl_fieldref(s,1)) #define jl_linenode_file(x) ((jl_sym_t*)jl_fieldref(x,0)) -#define jl_linenode_line(x) (((ptrint_t*)jl_fieldref(x,1))[0]) +#define jl_linenode_line(x) (((ptrint_t*)x)[1]) #define jl_labelnode_label(x) (((ptrint_t*)x)[0]) #define jl_gotonode_label(x) (((ptrint_t*)x)[0]) #define jl_globalref_mod(s) ((jl_module_t*)jl_fieldref(s,0)) From d8f5bf271c753647671ad2609b15ec745051af4b Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sat, 26 Sep 2015 23:49:13 -0400 Subject: [PATCH 0259/1938] Make jl_static_show non-allocating --- src/builtins.c | 226 +++++++++++++++++++++++++++---------------------- src/julia.h | 11 +-- 2 files changed, 130 insertions(+), 107 deletions(-) diff --git a/src/builtins.c b/src/builtins.c index 5519beeaf8a34..0dd497981bc6a 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1328,32 +1328,36 @@ static size_t jl_show_svec(JL_STREAM *out, jl_svec_t *t, char *head, char *opn, #define MAX_DEPTH 25 -size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth) +static size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth); + +// `v` might be pointing to a field inlined in a structure therefore +// `jl_typeof(v)` may not be the same with `vt` and only `vt` should be +// used to determine the type of the value. +// This is necessary to make sure that this function doesn't allocate any +// memory through the Julia GC +static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, + jl_datatype_t *vt, int depth) { - // mimic jl_show, but never calling a julia method - size_t n = 0; if (depth > MAX_DEPTH) { // cheap way of bailing out of cycles return jl_printf(out, "•"); } + size_t n = 0; depth++; - if (v == NULL) { - n += jl_printf(out, "#"); + if ((uintptr_t)vt < 4096U) { + n += jl_printf(out, "", v, vt); } else if ((uintptr_t)v < 4096U) { - n += jl_printf(out, "#<%d>", (int)(uintptr_t)v); - } - else if (jl_typeof(v) == NULL) { - n += jl_printf(out, ""); - } - else if (jl_astaggedvalue(v)->type_bits < 4096U) { - n += jl_printf(out, "", (int)jl_astaggedvalue(v)->type_bits); + n += jl_printf(out, ""); } - else if (jl_is_lambda_info(v)) { + else if (vt == jl_lambda_info_type) { jl_lambda_info_t *li = (jl_lambda_info_t*)v; n += jl_static_show_x(out, (jl_value_t*)li->module, depth); if (li->specTypes) { n += jl_printf(out, "."); - n += jl_show_svec(out, li->specTypes->parameters, li->name->name, "(", ")"); + n += jl_show_svec(out, li->specTypes->parameters, + li->name->name, "(", ")"); } else { n += jl_printf(out, ".%s(?)", li->name->name); @@ -1363,14 +1367,14 @@ size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth) //jl_printf(out, " -> "); //jl_static_show(out, !jl_is_expr(li->ast) ? jl_uncompress_ast(li, li->ast) : li->ast); } - else if (jl_is_svec(v)) { + else if (vt == jl_simplevector_type) { n += jl_show_svec(out, (jl_svec_t*)v, "svec", "(", ")"); } - else if (jl_is_datatype(v)) { + else if (vt == jl_datatype_type) { jl_datatype_t *dv = (jl_datatype_t*)v; if (dv->name->module != jl_core_module) { n += jl_static_show_x(out, (jl_value_t*)dv->name->module, depth); - jl_printf(out, "."); n += 1; + n += jl_printf(out, "."); } n += jl_printf(out, "%s", dv->name->name->name); if (dv->parameters && (jl_value_t*)dv != dv->name->primary && @@ -1386,12 +1390,12 @@ size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth) } n += jl_printf(out, "}"); } - else if (jl_is_tuple_type(dv)) { + else if (dv->name == jl_tuple_typename) { n += jl_printf(out, "{}"); } } } - else if (jl_is_func(v)) { + else if (vt == jl_function_type) { if (jl_is_gf(v)) { n += jl_printf(out, "%s", jl_gf_name(v)->name); } @@ -1399,66 +1403,63 @@ size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth) n += jl_printf(out, "#"); } } - else if (jl_typeis(v, jl_intrinsic_type)) { - n += jl_printf(out, "#", *(uint32_t*)jl_data_ptr(v)); + else if (vt == jl_intrinsic_type) { + n += jl_printf(out, "#", + *(uint32_t*)jl_data_ptr(v)); } - else if (jl_is_int64(v)) { - n += jl_printf(out, "%" PRId64, jl_unbox_int64(v)); + else if (vt == jl_int64_type) { + n += jl_printf(out, "%" PRId64, *(int64_t*)v); } - else if (jl_is_int32(v)) { - n += jl_printf(out, "%" PRId32, jl_unbox_int32(v)); + else if (vt == jl_int32_type) { + n += jl_printf(out, "%" PRId32, *(int32_t*)v); } - else if (jl_typeis(v,jl_int16_type)) { - n += jl_printf(out, "%" PRId16, jl_unbox_int16(v)); + else if (vt == jl_int16_type) { + n += jl_printf(out, "%" PRId16, *(int16_t*)v); } - else if (jl_typeis(v,jl_int8_type)) { - n += jl_printf(out, "%" PRId8, jl_unbox_int8(v)); + else if (vt == jl_int8_type) { + n += jl_printf(out, "%" PRId8, *(int8_t*)v); } - else if (jl_is_uint64(v)) { - n += jl_printf(out, "0x%016" PRIx64, jl_unbox_uint64(v)); + else if (vt == jl_uint64_type) { + n += jl_printf(out, "0x%016" PRIx64, *(uint64_t*)v); } - else if (jl_is_uint32(v)) { - n += jl_printf(out, "0x%08" PRIx32, jl_unbox_uint32(v)); + else if (vt == jl_uint32_type) { + n += jl_printf(out, "0x%08" PRIx32, *(uint32_t*)v); } - else if (jl_typeis(v,jl_uint16_type)) { - n += jl_printf(out, "0x%04" PRIx16, jl_unbox_uint16(v)); + else if (vt == jl_uint16_type) { + n += jl_printf(out, "0x%04" PRIx16, *(uint16_t*)v); } - else if (jl_typeis(v,jl_uint8_type)) { - n += jl_printf(out, "0x%02" PRIx8, jl_unbox_uint8(v)); + else if (vt == jl_uint8_type) { + n += jl_printf(out, "0x%02" PRIx8, *(uint8_t*)v); } - else if (jl_is_cpointer(v)) { + else if (jl_is_cpointer_type((jl_value_t*)vt)) { #ifdef _P64 - n += jl_printf(out, "0x%016" PRIx64, - (int64_t)jl_unbox_voidpointer(v)); + n += jl_printf(out, "0x%016" PRIx64, *(uint64_t*)v); #else - n += jl_printf(out, "0x%08" PRIx32, (int32_t)jl_unbox_voidpointer(v)); + n += jl_printf(out, "0x%08" PRIx32, *(uint32_t*)v); #endif } - else if (jl_is_float32(v)) { - n += jl_printf(out, "%g", jl_unbox_float32(v)); - } - else if (jl_is_float64(v)) { - n += jl_printf(out, "%g", jl_unbox_float64(v)); + else if (vt == jl_float32_type) { + n += jl_printf(out, "%g", *(float*)v); } - else if (v == jl_true) { - n += jl_printf(out, "true"); + else if (vt == jl_float64_type) { + n += jl_printf(out, "%g", *(double*)v); } - else if (v == jl_false) { - n += jl_printf(out, "false"); + else if (vt == jl_bool_type) { + n += jl_printf(out, "%s", *(uint8_t*)v ? "true" : "false"); } - else if (v == jl_nothing) { + else if ((jl_value_t*)vt == jl_typeof(jl_nothing)) { n += jl_printf(out, "nothing"); } - else if (jl_is_byte_string(v)) { + else if (vt == jl_ascii_string_type || vt == jl_utf8_string_type) { n += jl_printf(out, "\"%s\"", jl_iostr_data(v)); } - else if (jl_is_uniontype(v)) { + else if (vt == jl_uniontype_type) { n += jl_show_svec(out, ((jl_uniontype_t*)v)->types, "Union", "{", "}"); } - else if (jl_is_typector(v)) { + else if (vt == jl_typector_type) { n += jl_static_show_x(out, ((jl_typector_t*)v)->body, depth); } - else if (jl_is_typevar(v)) { + else if (vt == jl_tvar_type) { if (((jl_tvar_t*)v)->lb != jl_bottom_type) { n += jl_static_show(out, ((jl_tvar_t*)v)->lb); n += jl_printf(out, "<:"); @@ -1466,7 +1467,7 @@ size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth) n += jl_printf(out, "%s%s<:", (((jl_tvar_t*)v)->bound)?"#":"", ((jl_tvar_t*)v)->name->name); n += jl_static_show(out, ((jl_tvar_t*)v)->ub); } - else if (jl_is_module(v)) { + else if (vt == jl_module_type) { jl_module_t *m = (jl_module_t*)v; if (m->parent != m && m->parent != jl_main_module) { n += jl_static_show_x(out, (jl_value_t*)m->parent, depth); @@ -1474,48 +1475,52 @@ size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth) } n += jl_printf(out, "%s", m->name->name); } - else if (jl_is_symbol(v)) { + else if (vt == jl_sym_type) { n += jl_printf(out, ":%s", ((jl_sym_t*)v)->name); } - else if (jl_is_gensym(v)) { + else if (vt == jl_gensym_type) { n += jl_printf(out, "GenSym(%" PRIuPTR ")", (uintptr_t)((jl_gensym_t*)v)->id); } - else if (jl_is_symbolnode(v)) { + else if (vt == jl_symbolnode_type) { n += jl_printf(out, "%s::", jl_symbolnode_sym(v)->name); n += jl_static_show_x(out, jl_symbolnode_type(v), depth); } - else if (jl_is_globalref(v)) { + else if (vt == jl_globalref_type) { n += jl_static_show_x(out, (jl_value_t*)jl_globalref_mod(v), depth); n += jl_printf(out, ".%s", jl_globalref_name(v)->name); } - else if (jl_is_labelnode(v)) { + else if (vt == jl_labelnode_type) { n += jl_printf(out, "%" PRIuPTR ":", jl_labelnode_label(v)); } - else if (jl_is_gotonode(v)) { + else if (vt == jl_gotonode_type) { n += jl_printf(out, "goto %" PRIuPTR, jl_gotonode_label(v)); } - else if (jl_is_quotenode(v)) { - jl_value_t *qv = jl_fieldref(v,0); - if (!jl_is_symbol(qv)) { n += jl_printf(out, "quote "); } - n += jl_static_show_x(out, jl_fieldref(v,0), depth); - if (!jl_is_symbol(qv)) { n += jl_printf(out, " end"); } + else if (vt == jl_quotenode_type) { + jl_value_t *qv = *(jl_value_t**)v; + if (!jl_is_symbol(qv)) { + n += jl_printf(out, "quote "); + } + n += jl_static_show_x(out, qv, depth); + if (!jl_is_symbol(qv)) { + n += jl_printf(out, " end"); + } } - else if (jl_is_newvarnode(v)) { + else if (vt == jl_newvarnode_type) { n += jl_printf(out, ""); } - else if (jl_is_topnode(v)) { + else if (vt == jl_topnode_type) { n += jl_printf(out, "top("); - n += jl_static_show_x(out, jl_fieldref(v,0), depth); + n += jl_static_show_x(out, *(jl_value_t**)v, depth); n += jl_printf(out, ")"); } - else if (jl_is_linenode(v)) { - n += jl_printf(out, "# line %"PRIuPTR" %s", + else if (vt == jl_linenumbernode_type) { + n += jl_printf(out, "# line %" PRIuPTR " %s", jl_linenode_line(v), jl_linenode_file(v)->name); } - else if (jl_is_expr(v)) { + else if (vt == jl_expr_type) { jl_expr_t *e = (jl_expr_t*)v; if (e->head == assign_sym && jl_array_len(e->args) == 2) { n += jl_static_show_x(out, jl_exprarg(e,0), depth); @@ -1536,45 +1541,46 @@ size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth) n += jl_static_show_x(out, e->etype, depth); } } - else if (jl_is_array(v)) { - n += jl_static_show_x(out, jl_typeof(v), depth); + else if (jl_is_array_type(vt)) { + n += jl_static_show_x(out, (jl_value_t*)vt, depth); n += jl_printf(out, "["); size_t j, tlen = jl_array_len(v); + jl_array_t *av = (jl_array_t*)v; + jl_datatype_t *el_type = (jl_datatype_t*)jl_tparam0(vt); for (j = 0; j < tlen; j++) { - jl_value_t *elt; - if (((jl_array_t*)v)->ptrarray) - elt = jl_cellref(v, j); - else - elt = jl_arrayref((jl_array_t*)v,j); - n += jl_static_show_x(out, elt, depth); + if (av->ptrarray) { + n += jl_static_show_x(out, jl_cellref(v, j), depth); + } else { + char *ptr = ((char*)av->data) + j * av->elsize; + n += jl_static_show_x_(out, (jl_value_t*)ptr, el_type, depth); + } if (j != tlen-1) n += jl_printf(out, ", "); } if (j < tlen) n += jl_printf(out, " ..."); n += jl_printf(out, "]"); } - else if (jl_typeis(v,jl_loaderror_type)) { + else if (vt == jl_loaderror_type) { n += jl_printf(out, "LoadError(at "); - n += jl_static_show_x(out, jl_fieldref(v, 0), depth); + n += jl_static_show_x(out, *(jl_value_t**)v, depth); // Access the field directly to avoid allocation n += jl_printf(out, " line %" PRIdPTR, ((intptr_t*)v)[1]); n += jl_printf(out, ": "); - n += jl_static_show_x(out, jl_fieldref(v, 2), depth); + n += jl_static_show_x(out, ((jl_value_t**)v)[2], depth); n += jl_printf(out, ")"); } - else if (jl_typeis(v,jl_errorexception_type)) { + else if (vt == jl_errorexception_type) { n += jl_printf(out, "ErrorException("); - n += jl_static_show_x(out, jl_fieldref(v, 0), depth); + n += jl_static_show_x(out, *(jl_value_t**)v, depth); n += jl_printf(out, ")"); } - else if (jl_is_datatype(jl_typeof(v))) { - jl_datatype_t *t = (jl_datatype_t*)jl_typeof(v); - int istuple = jl_is_tuple_type(t); + else if (jl_is_datatype(vt)) { + int istuple = jl_is_tuple_type(vt); if (!istuple) - n += jl_static_show_x(out, (jl_value_t*)t, depth); + n += jl_static_show_x(out, (jl_value_t*)vt, depth); n += jl_printf(out, "("); - size_t nb = jl_datatype_size(t); - size_t tlen = jl_datatype_nfields(t); + size_t nb = jl_datatype_size(vt); + size_t tlen = jl_datatype_nfields(vt); if (nb > 0 && tlen == 0) { char *data = (char*)jl_data_ptr(v); n += jl_printf(out, "0x"); @@ -1582,34 +1588,50 @@ size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth) n += jl_printf(out, "%02" PRIx8, data[i]); } else { - jl_value_t *fldval=NULL; - JL_GC_PUSH1(&fldval); for (size_t i = 0; i < tlen; i++) { if (!istuple) { - n += jl_printf(out, "%s", ((jl_sym_t*)jl_svecref(t->name->names, i))->name); + n += jl_printf(out, "%s", ((jl_sym_t*)jl_svecref(vt->name->names, i))->name); //jl_fielddesc_t f = t->fields[i]; n += jl_printf(out, "="); } - // FIXME: this line can allocate - fldval = jl_get_nth_field(v, i); - n += jl_static_show_x(out, fldval, depth); + size_t offs = jl_field_offset(vt, i); + char *fld_ptr = (char*)v + offs; + if (jl_field_isptr(vt, i)) { + n += jl_static_show_x(out, *(jl_value_t**)fld_ptr, depth); + } else { + n += jl_static_show_x_(out, (jl_value_t*)fld_ptr, + (jl_datatype_t*)jl_field_type(vt, i), + depth); + } if (istuple && tlen==1) n += jl_printf(out, ","); else if (i != tlen-1) n += jl_printf(out, ", "); } - JL_GC_POP(); } n += jl_printf(out, ")"); } else { - n += jl_printf(out, ""); } return n; } +static size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth) +{ + // mimic jl_show, but never calling a julia method and + // (hopefully) never allocate through julia gc + if (v == NULL) { + return jl_printf(out, "#"); + } + else if ((uintptr_t)v < 4096U) { + return jl_printf(out, "#<%d>", (int)(uintptr_t)v); + } + return jl_static_show_x_(out, v, (jl_datatype_t*)jl_typeof(v), depth); +} + DLLEXPORT size_t jl_static_show(JL_STREAM *out, jl_value_t *v) { return jl_static_show_x(out, v, 0); diff --git a/src/julia.h b/src/julia.h index f3b224edc36f6..0b8e000a40a9e 100644 --- a/src/julia.h +++ b/src/julia.h @@ -679,14 +679,15 @@ STATIC_INLINE jl_value_t *jl_cellset(void *a, size_t i, void *x) #define jl_fieldref(s,i) jl_get_nth_field(((jl_value_t*)s),i) #define jl_nfields(v) jl_datatype_nfields(jl_typeof(v)) -#define jl_symbolnode_sym(s) ((jl_sym_t*)jl_fieldref(s,0)) -#define jl_symbolnode_type(s) (jl_fieldref(s,1)) -#define jl_linenode_file(x) ((jl_sym_t*)jl_fieldref(x,0)) +// Not using jl_fieldref to avoid allocations +#define jl_symbolnode_sym(s) (*(jl_sym_t**)s) +#define jl_symbolnode_type(s) (((jl_value_t**)s)[1]) +#define jl_linenode_file(x) (*(jl_sym_t**)x) #define jl_linenode_line(x) (((ptrint_t*)x)[1]) #define jl_labelnode_label(x) (((ptrint_t*)x)[0]) #define jl_gotonode_label(x) (((ptrint_t*)x)[0]) -#define jl_globalref_mod(s) ((jl_module_t*)jl_fieldref(s,0)) -#define jl_globalref_name(s) ((jl_sym_t*)jl_fieldref(s,1)) +#define jl_globalref_mod(s) (*(jl_module_t**)s) +#define jl_globalref_name(s) (((jl_sym_t**)s)[1]) #define jl_nparams(t) jl_svec_len(((jl_datatype_t*)(t))->parameters) #define jl_tparam0(t) jl_svecref(((jl_datatype_t*)(t))->parameters, 0) From 2fb0433d5c3b8ecb0b55a29ef75158dfdaaacee2 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Sun, 27 Sep 2015 18:43:59 -0700 Subject: [PATCH 0260/1938] Added note about support --- doc/stdlib/linalg.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index c3bda14b968d9..9ab68658b67cf 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -1321,6 +1321,10 @@ arrays have names ending in ``'!'``. Usually a function has 4 methods defined, one each for ``Float64``, ``Float32``, ``Complex128`` and ``Complex64`` arrays. +Note that the LAPACK API provided by Julia can and will change in the future. Since +this API is not user-facing, there is no commitment to support/deprecate this specific +set of functions in future releases. + .. currentmodule:: Base.LinAlg.LAPACK .. function:: gbtrf!(kl, ku, m, AB) -> (AB, ipiv) From bc43fe6321f57259f1576550880684d15882d03c Mon Sep 17 00:00:00 2001 From: Greg Peairs Date: Sun, 27 Sep 2015 21:01:24 -0700 Subject: [PATCH 0261/1938] Added tests for sparsevec --- test/sparsedir/sparse.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 776b4b04b92a9..1504d6d3e6c79 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -904,6 +904,8 @@ A = sprandbool(5,5,0.2) # test sparsevec A = sparse(ones(5,5)) @test all(full(sparsevec(A)) .== ones(25)) +@test all(full(sparsevec([1:5;],1)) .== ones(5)) +@test_throws ArgumentError sparsevec([1:5;], [1:4;]) #test sparse @test sparse(A) == A From 9064fe768a5ebbf1dba1a8b45c2cb3bbc386b8e4 Mon Sep 17 00:00:00 2001 From: Mark Dewing Date: Mon, 28 Sep 2015 10:10:24 -0500 Subject: [PATCH 0262/1938] Profile JIT events with OProfile This assumes that OProfile is installed in /usr. Adjust deps/Makefile if it is installed elsewhere (the line '--with-oprofile=/usr'). In the current default version of LLVM (3.3), the check to see if the execution is being profiled only looks for the older style of running OProfile. (It checks if the 'oprofiled' daemon is running.) The check is in llvm-3.3/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp This is fixed to recognize 'operf' in newer versions of LLVM. --- Make.inc | 8 ++++++++ deps/Makefile | 4 ++++ src/codegen.cpp | 6 ++++++ src/init.c | 11 +++++++++++ src/julia_internal.h | 3 +++ 5 files changed, 32 insertions(+) diff --git a/Make.inc b/Make.inc index d11567b3100bd..c571d0d61641a 100644 --- a/Make.inc +++ b/Make.inc @@ -59,6 +59,9 @@ $(warning "The julia make variable USE_MKL has been renamed to USE_INTEL_MKL") USE_INTEL_MKL := 1 endif +# Set to 1 to enable profiling with OProfile +USE_OPROFILE_JITEVENTS ?= 0 + # libc++ is standard on OS X 10.9, but not for earlier releases USE_LIBCPP := 0 @@ -839,6 +842,11 @@ ifeq ($(USE_INTEL_JITEVENTS), 1) JCPPFLAGS += -DJL_USE_INTEL_JITEVENTS endif +# OProfile +ifeq ($(USE_OPROFILE_JITEVENTS), 1) +JCPPFLAGS += -DJL_USE_OPROFILE_JITEVENTS +endif + # Intel libraries ifeq ($(USE_INTEL_LIBM), 1) diff --git a/deps/Makefile b/deps/Makefile index aebfd9d74456b..51c2389c376d4 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -424,6 +424,10 @@ else LLVM_FLAGS += --disable-threads endif # USE_INTEL_JITEVENTS +ifeq ($(USE_OPROFILE_JITEVENTS), 1) +LLVM_FLAGS += --with-oprofile=/usr/ +endif # USE_OPROFILE_JITEVENTS + ifeq ($(BUILD_LLDB),1) ifeq ($(USECLANG),1) LLVM_FLAGS += --enable-cxx11 diff --git a/src/codegen.cpp b/src/codegen.cpp index e178a4597c861..3edcd2c6c974f 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6043,6 +6043,12 @@ extern "C" void jl_init_codegen(void) JITEventListener::createIntelJITEventListener()); #endif // JL_USE_INTEL_JITEVENTS +#ifdef JL_USE_OPROFILE_JITEVENTS + if (jl_using_oprofile_jitevents) + jl_ExecutionEngine->RegisterJITEventListener( + JITEventListener::createOProfileJITEventListener()); +#endif // JL_USE_OPROFILE_JITEVENTS + BOX_F(int8,int8); UBOX_F(uint8,uint8); BOX_F(int16,int16); UBOX_F(uint16,uint16); BOX_F(int32,int32); UBOX_F(uint32,uint32); diff --git a/src/init.c b/src/init.c index 5debfbbe0176b..05479077a80cb 100644 --- a/src/init.c +++ b/src/init.c @@ -346,6 +346,10 @@ void init_stdio() char jl_using_intel_jitevents; // Non-zero if running under Intel VTune Amplifier #endif +#ifdef JL_USE_OPROFILE_JITEVENTS +char jl_using_oprofile_jitevents = 0; // Non-zero if running under OProfile +#endif + int isabspath(const char *in) { #ifdef _OS_WINDOWS_ @@ -520,6 +524,13 @@ void _julia_init(JL_IMAGE_SEARCH rel) } #endif +#if defined(JL_USE_OPROFILE_JITEVENTS) + const char *jit_profiling = getenv("ENABLE_JITPROFILING"); + if (jit_profiling && atoi(jit_profiling)) { + jl_using_oprofile_jitevents = 1; + } +#endif + jl_gc_init(); jl_gc_enable(0); jl_init_frontend(); diff --git a/src/julia_internal.h b/src/julia_internal.h index 4a230bf7d31d9..ea14b50079e83 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -120,6 +120,9 @@ extern jl_array_t *jl_module_init_order; #ifdef JL_USE_INTEL_JITEVENTS extern char jl_using_intel_jitevents; #endif +#ifdef JL_USE_OPROFILE_JITEVENTS +extern char jl_using_oprofile_jitevents; +#endif extern size_t jl_arr_xtralloc_limit; void jl_init_types(void); From fbfc811f7023b5f3237506e4ab2d82d65920daeb Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Sun, 27 Sep 2015 23:50:06 -0400 Subject: [PATCH 0263/1938] Make function argument the first in remotecallX and remote_do to ease the use of do block syntax. --- NEWS.md | 24 ++++++ base/client.jl | 2 +- base/deprecated.jl | 8 ++ base/docs/helpdb.jl | 8 +- base/loading.jl | 8 +- base/managers.jl | 2 +- base/multi.jl | 72 +++++++++--------- base/require.jl | 4 +- base/sharedarray.jl | 39 ++++++---- doc/stdlib/parallel.rst | 8 +- examples/clustermanager/simple/test_simple.jl | 2 +- examples/hpl.jl | 10 +-- test/examples.jl | 5 +- test/netload/memtest.jl | 2 +- test/parallel.jl | 74 +++++++++++++------ test/topology.jl | 10 ++- 16 files changed, 177 insertions(+), 101 deletions(-) diff --git a/NEWS.md b/NEWS.md index b989233f52dd7..ecf85b6cdba40 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,27 @@ +Julia v0.5.0 Release Notes +========================== + +New language features +--------------------- + +Language changes +---------------- + +Command line option changes +--------------------------- + +Compiler/Runtime improvements +----------------------------- + +Library improvements +-------------------- + +Deprecated or removed +--------------------- + + * The function `remotecall`, `remotecall_fetch`, and `remotecall_wait` now have the + the function argument as the first argument to allow for do-block syntax. [#13338] + Julia v0.4.0 Release Notes ========================== diff --git a/base/client.jl b/base/client.jl index 8303bbdb6a204..a83893ee32ccf 100644 --- a/base/client.jl +++ b/base/client.jl @@ -243,7 +243,7 @@ function process_options(opts::JLOptions, args::Vector{UTF8String}) # load file immediately on all processors if opts.load != C_NULL @sync for p in procs() - @async remotecall_fetch(p, include, bytestring(opts.load)) + @async remotecall_fetch(include, p, bytestring(opts.load)) end end # eval expression diff --git a/base/deprecated.jl b/base/deprecated.jl index f72e9ff61e46d..df4a5a144f831 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -834,3 +834,11 @@ end # 12839 const AsyncStream = IO deprecate(:AsyncStream) + +for f in (:remotecall, :remotecall_fetch, :remotecall_wait) + @eval begin + @deprecate ($f)(w::LocalProcess, f::Function, args...) ($f)(f, w::LocalProcess, args...) + @deprecate ($f)(w::Worker, f::Function, args...) ($f)(f, w::Worker, args...) + @deprecate ($f)(id::Integer, f::Function, args...) ($f)(f, id::Integer, args...) + end +end \ No newline at end of file diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 23dbcac1673c6..9392455e1d9a7 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -3609,7 +3609,7 @@ Returns the index of the current worker into the `pids` vector, i.e., the list o indexpids doc""" - remotecall_wait(id, func, args...) + remotecall_wait(func, id, args...) Perform `wait(remotecall(...))` in one message. """ @@ -6259,7 +6259,7 @@ Search for the first occurrence of the given characters within the given string. search doc""" - remotecall_fetch(id, func, args...) + remotecall_fetch(func, id, args...) Perform `fetch(remotecall(...))` in one message. Any remote exceptions are captured in a `RemoteException` and thrown. """ @@ -7370,7 +7370,7 @@ Determine whether a `RemoteRef` has a value stored to it. Note that this functio If the argument `RemoteRef` is owned by a different node, this call will block to wait for the answer. It is recommended to wait for `r` in a separate task instead, or to use a local `RemoteRef` as a proxy: rr = RemoteRef() - @async put!(rr, remotecall_fetch(p, long_computation)) + @async put!(rr, remotecall_fetch(long_computation, p)) isready(rr) # will not block """ isready @@ -9490,7 +9490,7 @@ Read all available data on the stream, blocking the task only if no data is avai readavailable doc""" - remotecall(id, func, args...) + remotecall(func, id, args...) Call a function asynchronously on the given arguments on the specified process. Returns a `RemoteRef`. """ diff --git a/base/loading.jl b/base/loading.jl index fd6ab8755f89a..b5b05e23b9ef5 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -30,7 +30,7 @@ function find_in_node_path(name, srcpath, node::Int=1) if myid() == node find_in_path(name, srcpath) else - remotecall_fetch(node, find_in_path, name, srcpath) + remotecall_fetch(find_in_path, node, name, srcpath) end end @@ -67,7 +67,7 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::ByteStrin if node == myid() content = open(readbytes, path_to_try) else - content = remotecall_fetch(node, open, readbytes, path_to_try) + content = remotecall_fetch(open, node, readbytes, path_to_try) end restored = _include_from_serialized(content) if restored !== nothing @@ -83,7 +83,7 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::ByteStrin myid() == 1 && recompile_stale(mod, path_to_try) restored = ccall(:jl_restore_incremental, Any, (Ptr{UInt8},), path_to_try) else - content = remotecall_fetch(node, open, readbytes, path_to_try) + content = remotecall_fetch(open, node, readbytes, path_to_try) restored = _include_from_serialized(content) end # otherwise, continue search @@ -304,7 +304,7 @@ function include_from_node1(_path::AbstractString) result = Core.include(path) nprocs()>1 && sleep(0.005) else - result = include_string(remotecall_fetch(1, readall, path), path) + result = include_string(remotecall_fetch(readall, 1, path), path) end finally if prev === nothing diff --git a/base/managers.jl b/base/managers.jl index 5102eecb70097..4a5a8f9730c72 100644 --- a/base/managers.jl +++ b/base/managers.jl @@ -357,7 +357,7 @@ function connect_to_worker(host::AbstractString, bind_addr::AbstractString, port end function kill(manager::ClusterManager, pid::Int, config::WorkerConfig) - remote_do(pid, exit) # For TCP based transports this will result in a close of the socket + remote_do(exit, pid) # For TCP based transports this will result in a close of the socket # at our end, which will result in a cleanup of the worker. nothing end diff --git a/base/multi.jl b/base/multi.jl index 3378654457d59..5f977eccd0f03 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -185,14 +185,14 @@ function flush_gc_msgs(w::Worker) msgs = copy(w.add_msgs) if !isempty(msgs) empty!(w.add_msgs) - remote_do(w, add_clients, msgs) + remote_do(add_clients, w, msgs) end msgs = copy(w.del_msgs) if !isempty(msgs) empty!(w.del_msgs) #print("sending delete of $msgs\n") - remote_do(w, del_clients, msgs) + remote_do(del_clients, w, msgs) end end @@ -293,7 +293,7 @@ get_bind_addr(w::LocalProcess) = LPROC.bind_addr function get_bind_addr(w::Worker) if isnull(w.config.bind_addr) if w.id != myid() - w.config.bind_addr = remotecall_fetch(w.id, get_bind_addr, w.id) + w.config.bind_addr = remotecall_fetch(get_bind_addr, w.id, w.id) end end get(w.config.bind_addr) @@ -317,7 +317,7 @@ function procs(pid::Integer) Int[x.id for x in filter(w -> get_bind_addr(w) == ipatpid, PGRP.workers)] end else - remotecall_fetch(1, procs, pid) + remotecall_fetch(procs, 1, pid) end end @@ -425,7 +425,7 @@ function deregister_worker(pg, pid) if PGRP.topology != :all_to_all for rpid in workers() try - remote_do(rpid, deregister_worker, pid) + remote_do(deregister_worker, rpid, pid) catch end end @@ -494,10 +494,10 @@ function RemoteRef(pid::Integer=myid()) end function RemoteRef(f::Function, pid::Integer=myid()) - remotecall_fetch(pid, (f, rrid) -> begin + remotecall_fetch(pid, f, next_rrid_tuple()) do f, rrid rv=lookup_ref(rrid, f) RemoteRef{typeof(rv.c)}(myid(), rrid[1], rrid[2]) - end, f, next_rrid_tuple()) + end end hash(r::RemoteRef, h::UInt) = hash(r.whence, hash(r.id, h)) @@ -522,7 +522,7 @@ function isready(rr::RemoteRef, args...) if rr.where == myid() isready(lookup_ref(rid).c, args...) else - remotecall_fetch(rr.where, id->isready(lookup_ref(rid).c, args...), rid) + remotecall_fetch(id->isready(lookup_ref(rid).c, args...), rr.where, rid) end end @@ -698,28 +698,28 @@ function local_remotecall_thunk(f, args) # f(map(localize_ref,args)...) end -function remotecall(w::LocalProcess, f, args...) +function remotecall(f, w::LocalProcess, args...) rr = RemoteRef(w) schedule_call(rr2id(rr), local_remotecall_thunk(f,args)) rr end -function remotecall(w::Worker, f, args...) +function remotecall(f, w::Worker, args...) rr = RemoteRef(w) #println("$(myid()) asking for $rr") send_msg(w, CallMsg{:call}(f, args, rr2id(rr))) rr end -remotecall(id::Integer, f, args...) = remotecall(worker_from_id(id), f, args...) +remotecall(f, id::Integer, args...) = remotecall(f, worker_from_id(id), args...) # faster version of fetch(remotecall(...)) -function remotecall_fetch(w::LocalProcess, f, args...) +function remotecall_fetch(f, w::LocalProcess, args...) v=run_work_thunk(local_remotecall_thunk(f,args), false) isa(v, RemoteException) ? throw(v) : v end -function remotecall_fetch(w::Worker, f, args...) +function remotecall_fetch(f, w::Worker, args...) # can be weak, because the program will have no way to refer to the Ref # itself, it only gets the result. oid = next_rrid_tuple() @@ -731,13 +731,13 @@ function remotecall_fetch(w::Worker, f, args...) isa(v, RemoteException) ? throw(v) : v end -remotecall_fetch(id::Integer, f, args...) = - remotecall_fetch(worker_from_id(id), f, args...) +remotecall_fetch(f, id::Integer, args...) = + remotecall_fetch(f, worker_from_id(id), args...) # faster version of wait(remotecall(...)) -remotecall_wait(w::LocalProcess, f, args...) = wait(remotecall(w,f,args...)) +remotecall_wait(f, w::LocalProcess, args...) = wait(remotecall(f, w, args...)) -function remotecall_wait(w::Worker, f, args...) +function remotecall_wait(f, w::Worker, args...) prid = next_rrid_tuple() rv = lookup_ref(prid) rv.waitingfor = w.id @@ -748,10 +748,10 @@ function remotecall_wait(w::Worker, f, args...) rr end -remotecall_wait(id::Integer, f, args...) = - remotecall_wait(worker_from_id(id), f, args...) +remotecall_wait(f, id::Integer, args...) = + remotecall_wait(f, worker_from_id(id), args...) -function remote_do(w::LocalProcess, f, args...) +function remote_do(f, w::LocalProcess, args...) # the LocalProcess version just performs in local memory what a worker # does when it gets a :do message. # same for other messages on LocalProcess. @@ -760,12 +760,12 @@ function remote_do(w::LocalProcess, f, args...) nothing end -function remote_do(w::Worker, f, args...) +function remote_do(f, w::Worker, args...) send_msg(w, RemoteDoMsg(f, args)) nothing end -remote_do(id::Integer, f, args...) = remote_do(worker_from_id(id), f, args...) +remote_do(f, id::Integer, args...) = remote_do(f, worker_from_id(id), args...) # have the owner of rr call f on it function call_on_owner(f, rr::RemoteRef, args...) @@ -773,7 +773,7 @@ function call_on_owner(f, rr::RemoteRef, args...) if rr.where == myid() f(rid, args...) else - remotecall_fetch(rr.where, f, rid, args...) + remotecall_fetch(f, rr.where, rid, args...) end end @@ -818,7 +818,7 @@ function deliver_result(sock::IO, msg, oid, value) elseif wid == 1 exit(1) else - remote_do(1, rmprocs, wid) + remote_do(rmprocs, 1, wid) end end end @@ -1125,7 +1125,7 @@ function addprocs(manager::ClusterManager; kwargs...) # function returns to the caller. all_w = workers() for pid in all_w - remote_do(pid, set_valid_processes, all_w) + remote_do(set_valid_processes, pid, all_w) end sort!(launched_q) @@ -1169,7 +1169,7 @@ function launch_n_additional_processes(manager, frompid, fromconfig, cnt, launch exeflags = get(fromconfig.exeflags, ``) cmd = `$exename $exeflags` - new_addresses = remotecall_fetch(frompid, launch_additional, cnt, cmd) + new_addresses = remotecall_fetch(launch_additional, frompid, cnt, cmd) for address in new_addresses (bind_addr, port) = address @@ -1183,7 +1183,7 @@ function launch_n_additional_processes(manager, frompid, fromconfig, cnt, launch let wconfig=wconfig @async begin pid = create_worker(manager, wconfig) - remote_do(frompid, redirect_output_from_additional_worker, pid, port) + remote_do(redirect_output_from_additional_worker, frompid, pid, port) push!(launched_q, pid) end end @@ -1317,7 +1317,7 @@ let nextidx = 0 end end -spawnat(p, thunk) = sync_add(remotecall(p, thunk)) +spawnat(p, thunk) = sync_add(remotecall(thunk, p)) spawn_somewhere(thunk) = spawnat(chooseproc(thunk),thunk) @@ -1335,13 +1335,13 @@ macro fetch(expr) expr = localize_vars(:(()->($expr)), false) quote thunk = $(esc(expr)) - remotecall_fetch(chooseproc(thunk), thunk) + remotecall_fetch(thunk, chooseproc(thunk)) end end macro fetchfrom(p, expr) expr = localize_vars(:(()->($expr)), false) - :(remotecall_fetch($(esc(p)), $(esc(expr)))) + :(remotecall_fetch($(esc(expr)), $(esc(p)))) end macro everywhere(ex) @@ -1349,7 +1349,7 @@ macro everywhere(ex) sync_begin() thunk = ()->(eval(Main,$(Expr(:quote,ex))); nothing) for pid in workers() - async_run_thunk(()->remotecall_fetch(pid, thunk)) + async_run_thunk(()->remotecall_fetch(thunk, pid)) yield() # ensure that the remotecall_fetch has been started end @@ -1365,7 +1365,7 @@ end function pmap_static(f, lsts...) np = nprocs() n = length(lsts[1]) - Any[ remotecall(PGRP.workers[(i-1)%np+1].id, f, map(L->L[i], lsts)...) for i = 1:n ] + Any[ remotecall(f, PGRP.workers[(i-1)%np+1].id, map(L->L[i], lsts)...) for i = 1:n ] end pmap(f) = f() @@ -1427,7 +1427,7 @@ function pmap(f, lsts...; err_retry=true, err_stop=false, pids = workers()) (idx, fvals) = tasklet busy_workers[pididx] = true try - results[idx] = remotecall_fetch(wpid, f, fvals...) + results[idx] = remotecall_fetch(f, wpid, fvals...) catch ex if err_retry push!(retryqueue, (idx,fvals, ex)) @@ -1482,7 +1482,7 @@ function preduce(reducer, f, N::Int) w_exec = Task[] for (idx,pid) in enumerate(all_w) - t = Task(()->remotecall_fetch(pid, f, first(chunks[idx]), last(chunks[idx]))) + t = Task(()->remotecall_fetch(f, pid, first(chunks[idx]), last(chunks[idx]))) schedule(t) push!(w_exec, t) end @@ -1625,7 +1625,7 @@ end function check_same_host(pids) if myid() != 1 - return remotecall_fetch(1, check_same_host, pids) + return remotecall_fetch(check_same_host, 1, pids) else # We checkfirst if all test pids have been started using the local manager, # else we check for the same bind_to addr. This handles the special case @@ -1663,5 +1663,5 @@ function getindex(r::RemoteRef, args...) if r.where == myid() return getindex(fetch(r), args...) end - return remotecall_fetch(r.where, getindex, r, args...) + return remotecall_fetch(getindex, r.where, r, args...) end diff --git a/base/require.jl b/base/require.jl index d8dcff0f9e9cb..d852239225e28 100644 --- a/base/require.jl +++ b/base/require.jl @@ -23,7 +23,7 @@ function find_in_path(name::AbstractString) end find_in_node1_path(name) = myid()==1 ? - find_in_path(name) : remotecall_fetch(1, find_in_path, name) + find_in_path(name) : remotecall_fetch(find_in_path, 1, name) # Store list of files and their load time package_list = (ByteString=>Float64)[] @@ -94,7 +94,7 @@ function include_from_node1(path::AbstractString) result = Core.include(path) nprocs()>1 && sleep(0.005) else - result = include_string(remotecall_fetch(1, readall, path), path) + result = include_string(remotecall_fetch(readall, 1, path), path) end finally if prev == nothing diff --git a/base/sharedarray.jl b/base/sharedarray.jl index ab29e05e8b47b..24b1679bbe855 100644 --- a/base/sharedarray.jl +++ b/base/sharedarray.jl @@ -44,14 +44,17 @@ function SharedArray(T::Type, dims::NTuple; init=false, pids=Int[]) else # The shared array is created on a remote machine.... shmmem_create_pid = pids[1] - remotecall_fetch(pids[1], () -> begin shm_mmap_array(T, dims, shm_seg_name, JL_O_CREAT | JL_O_RDWR); nothing end) + remotecall_fetch(pids[1]) do + shm_mmap_array(T, dims, shm_seg_name, JL_O_CREAT | JL_O_RDWR) + nothing + end end func_mapshmem = () -> shm_mmap_array(T, dims, shm_seg_name, JL_O_RDWR) refs = Array(RemoteRef, length(pids)) for (i, p) in enumerate(pids) - refs[i] = remotecall(p, func_mapshmem) + refs[i] = remotecall(func_mapshmem, p) end # Wait till all the workers have mapped the segment @@ -64,7 +67,7 @@ function SharedArray(T::Type, dims::NTuple; init=false, pids=Int[]) if onlocalhost rc = shm_unlink(shm_seg_name) else - rc = remotecall_fetch(shmmem_create_pid, shm_unlink, shm_seg_name) + rc = remotecall_fetch(shm_unlink, shmmem_create_pid, shm_seg_name) end systemerror("Error unlinking shmem segment " * shm_seg_name, rc != 0) end @@ -85,14 +88,14 @@ function SharedArray(T::Type, dims::NTuple; init=false, pids=Int[]) if isa(init, Function) @sync begin for p in pids - @async remotecall_wait(p, init, S) + @async remotecall_wait(init, p, S) end end end finally if shm_seg_name != "" - remotecall_fetch(shmmem_create_pid, shm_unlink, shm_seg_name) + remotecall_fetch(shm_unlink, shmmem_create_pid, shm_seg_name) end end S @@ -108,7 +111,7 @@ function SharedArray{T,N}(filename::AbstractString, ::Type{T}, dims::NTuple{N,In pids, onlocalhost = shared_pids(pids) # If not supplied, determine the appropriate mode - have_file = onlocalhost ? isfile(filename) : remotecall_fetch(pids[1], isfile, filename) + have_file = onlocalhost ? isfile(filename) : remotecall_fetch(isfile, pids[1], filename) if mode == nothing mode = have_file ? "r+" : "w+" end @@ -127,14 +130,20 @@ function SharedArray{T,N}(filename::AbstractString, ::Type{T}, dims::NTuple{N,In local s if onlocalhost s = func_mmap(mode) - refs[1] = remotecall(pids[1], () -> func_mmap(workermode)) + refs[1] = remotecall(pids[1]) do + func_mmap(workermode) + end else - refs[1] = remotecall_wait(pids[1], () -> func_mmap(mode)) + refs[1] = remotecall_wait(pids[1]) do + func_mmap(mode) + end end # Populate the rest of the workers for i = 2:length(pids) - refs[i] = remotecall(pids[i], () -> func_mmap(workermode)) + refs[i] = remotecall(pids[i]) do + func_mmap(workermode) + end end # Wait till all the workers have mapped the segment @@ -157,7 +166,7 @@ function SharedArray{T,N}(filename::AbstractString, ::Type{T}, dims::NTuple{N,In if isa(init, Function) @sync begin for p in pids - @async remotecall_wait(p, init, S) + @async remotecall_wait(init, p, S) end end end @@ -175,7 +184,9 @@ function reshape{T,N}(a::SharedArray{T}, dims::NTuple{N,Int}) (length(a) != prod(dims)) && throw(DimensionMismatch("dimensions must be consistent with array size")) refs = Array(RemoteRef, length(a.pids)) for (i, p) in enumerate(a.pids) - refs[i] = remotecall(p, (r,d)->reshape(fetch(r),d), a.refs[i], dims) + refs[i] = remotecall(p, a.refs[i], dims) do r,d + reshape(fetch(r),d) + end end A = SharedArray{T,N}(dims, a.pids, refs, a.segname) @@ -288,7 +299,7 @@ function fill!(S::SharedArray, v) vT = convert(eltype(S), v) f = S->fill!(S.loc_subarr_1d, vT) @sync for p in procs(S) - @async remotecall_wait(p, f, S) + @async remotecall_wait(f, p, S) end return S end @@ -296,7 +307,7 @@ end function rand!{T}(S::SharedArray{T}) f = S->map!(x->rand(T), S.loc_subarr_1d) @sync for p in procs(S) - @async remotecall_wait(p, f, S) + @async remotecall_wait(f, p, S) end return S end @@ -304,7 +315,7 @@ end function randn!(S::SharedArray) f = S->map!(x->randn(), S.loc_subarr_1d) @sync for p in procs(S) - @async remotecall_wait(p, f, S) + @async remotecall_wait(f, p, S) end return S end diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 6075736808aa8..8a9652ef42c11 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -264,7 +264,7 @@ General Parallel Computing Support If ``err_retry`` is ``true``, it retries a failed application of ``f`` on a different worker. If ``err_stop`` is ``true``, it takes precedence over the value of ``err_retry`` and ``pmap`` stops execution on the first error. -.. function:: remotecall(id, func, args...) +.. function:: remotecall(func, id, args...) .. Docstring generated from Julia source @@ -296,13 +296,13 @@ General Parallel Computing Support * ``RemoteRef``\ : Wait for and get the value of a remote reference. If the remote value is an exception, throws a ``RemoteException`` which captures the remote exception and backtrace. * ``Channel`` : Wait for and get the first available item from the channel. -.. function:: remotecall_wait(id, func, args...) +.. function:: remotecall_wait(func, id, args...) .. Docstring generated from Julia source Perform ``wait(remotecall(...))`` in one message. -.. function:: remotecall_fetch(id, func, args...) +.. function:: remotecall_fetch(func, id, args...) .. Docstring generated from Julia source @@ -343,7 +343,7 @@ General Parallel Computing Support .. code-block:: julia rr = RemoteRef() - @async put!(rr, remotecall_fetch(p, long_computation)) + @async put!(rr, remotecall_fetch(long_computation, p)) isready(rr) # will not block .. function:: close(Channel) diff --git a/examples/clustermanager/simple/test_simple.jl b/examples/clustermanager/simple/test_simple.jl index 42268d58514af..faca7e168fc98 100644 --- a/examples/clustermanager/simple/test_simple.jl +++ b/examples/clustermanager/simple/test_simple.jl @@ -5,7 +5,7 @@ include(cmanpath) npids = addprocs(UnixDomainCM(2)) assert(length(npids) == 2) -test_pids = [remotecall_fetch(x, myid) for x in npids] +test_pids = [remotecall_fetch(myid, x) for x in npids] assert(npids == test_pids) rmprocs(npids; waitfor=1.0) diff --git a/examples/hpl.jl b/examples/hpl.jl index 463da04d76c00..e67c085452716 100644 --- a/examples/hpl.jl +++ b/examples/hpl.jl @@ -251,15 +251,15 @@ function hpl_par2(A::Matrix, b::Vector) for i = 1:nB #println("C=$(convert(Array, C))") ##### ##panel factorization - panel_p = remotecall_fetch(C.pmap[i], panel_factor_par2, C, i, n) + panel_p = remotecall_fetch(panel_factor_par2, C.pmap[i], C, i, n) ## Apply permutation from pivoting for j = (i+1):nB - depend[i,j] = remotecall(C.pmap[j], permute, C, i, j, panel_p, n, false) + depend[i,j] = remotecall(permute, C.pmap[j], C, i, j, panel_p, n, false) end ## Special case for last column if i == nB - depend[nB,nB] = remotecall(C.pmap[nB], permute, C, i, nB+1, panel_p, n, true) + depend[nB,nB] = remotecall(permute, C.pmap[nB], C, i, nB+1, panel_p, n, true) end ##Trailing updates @@ -276,13 +276,13 @@ function hpl_par2(A::Matrix, b::Vector) for j=(i+1):nB dep = depend[i,j] - depend[j,i] = remotecall(C.pmap[j], trailing_update_par2, C, L_II, C_KI, i, j, n, false, dep) + depend[j,i] = remotecall(trailing_update_par2, C.pmap[j], C, L_II, C_KI, i, j, n, false, dep) end ## Special case for last column if i == nB dep = depend[nB,nB] - remotecall_fetch(C.pmap[nB], trailing_update_par2, C, L_II, C_KI, i, nB+1, n, true, dep) + remotecall_fetch(trailing_update_par2, C.pmap[nB], C, L_II, C_KI, i, nB+1, n, true, dep) else #enforce dependencies for nonspecial case for j=(i+1):nB diff --git a/test/examples.jl b/test/examples.jl index 0425fa15aace0..764d0e1d82666 100644 --- a/test/examples.jl +++ b/test/examples.jl @@ -55,7 +55,10 @@ include(dc_path) w_set=filter!(x->x != myid(), workers()) pid = length(w_set) > 0 ? w_set[1] : myid() -remotecall_fetch(pid, f->(include(f); nothing), dc_path) +remotecall_fetch(pid, dc_path) do f + include(f) + nothing +end dc=RemoteRef(()->DictChannel(), pid) @test typeof(dc) == RemoteRef{DictChannel} diff --git a/test/netload/memtest.jl b/test/netload/memtest.jl index aea2239a06f18..627b03ea2a215 100644 --- a/test/netload/memtest.jl +++ b/test/netload/memtest.jl @@ -54,7 +54,7 @@ end function mtest_remotecall_fetch() for i in 1:10^5 - remotecall_fetch(1, myid) + remotecall_fetch(myid, 1) end gc() end diff --git a/test/parallel.jl b/test/parallel.jl index e0a15ed78af98..8d5d14cfa9b1a 100644 --- a/test/parallel.jl +++ b/test/parallel.jl @@ -4,7 +4,9 @@ using Base.Test if nworkers() < 3 - remotecall_fetch(1, () -> addprocs(3 - nworkers())) + remotecall_fetch(1) do + addprocs(3 - nworkers()) + end end id_me = myid() @@ -47,7 +49,9 @@ end function check_pids_all(S::SharedArray) pidtested = falses(size(S)) for p in procs(S) - idxes_in_p = remotecall_fetch(p, D -> parentindexes(D.loc_subarr_1d)[1], S) + idxes_in_p = remotecall_fetch(p, S) do D + parentindexes(D.loc_subarr_1d)[1] + end @test all(sdata(S)[idxes_in_p] .== p) pidtested[idxes_in_p] = true end @@ -60,18 +64,26 @@ a = convert(Array, d) partsums = Array(Int, length(procs(d))) @sync begin for (i, p) in enumerate(procs(d)) - @async partsums[i] = remotecall_fetch(p, D->sum(D.loc_subarr_1d), d) + @async partsums[i] = remotecall_fetch(p, d) do D + sum(D.loc_subarr_1d) + end end end @test sum(a) == sum(partsums) d = Base.shmem_rand(dims) for p in procs(d) - idxes_in_p = remotecall_fetch(p, D -> parentindexes(D.loc_subarr_1d)[1], d) + idxes_in_p = remotecall_fetch(p, d) do D + parentindexes(D.loc_subarr_1d)[1] + end idxf = first(idxes_in_p) idxl = last(idxes_in_p) d[idxf] = Float64(idxf) - rv = remotecall_fetch(p, (D,idxf,idxl) -> begin assert(D[idxf] == Float64(idxf)); D[idxl] = Float64(idxl); D[idxl]; end, d,idxf,idxl) + rv = remotecall_fetch(p, d,idxf,idxl) do D,idxf,idxl + assert(D[idxf] == Float64(idxf)) + D[idxl] = Float64(idxl) + D[idxl] + end @test d[idxl] == rv end @@ -90,7 +102,9 @@ a = rand(dims) d = SharedArray(Int, dims; init = D->fill!(D.loc_subarr_1d, myid())) for p in procs(d) - idxes_in_p = remotecall_fetch(p, D -> parentindexes(D.loc_subarr_1d)[1], d) + idxes_in_p = remotecall_fetch(p, d) do D + parentindexes(D.loc_subarr_1d)[1] + end idxf = first(idxes_in_p) idxl = last(idxes_in_p) @test d[idxf] == p @@ -115,7 +129,9 @@ S = SharedArray(fn, Int, sz) @test length(procs(S)) > 1 @sync begin for p in procs(S) - @async remotecall_wait(p, D->fill!(D.loc_subarr_1d, myid()), S) + @async remotecall_wait(p, S) do D + fill!(D.loc_subarr_1d, myid()) + end end end check_pids_all(S) @@ -187,7 +203,7 @@ s = copy(sdata(d)) ds = deepcopy(d) @test ds == d pids_d = procs(d) -remotecall_fetch(pids_d[findfirst(id->(id != myid()), pids_d)], setindex!, d, 1.0, 1:10) +remotecall_fetch(setindex!, pids_d[findfirst(id->(id != myid()), pids_d)], d, 1.0, 1:10) @test ds != d @test s != d @@ -220,8 +236,8 @@ map!(x->1, d) @test d[1,:] == fill(2, 1, 10) # Boundary cases where length(S) <= length(pids) -@test 2.0 == remotecall_fetch(id_other, D->D[2], Base.shmem_fill(2.0, 2; pids=[id_me, id_other])) -@test 3.0 == remotecall_fetch(id_other, D->D[1], Base.shmem_fill(3.0, 1; pids=[id_me, id_other])) +@test 2.0 == remotecall_fetch(D->D[2], id_other, Base.shmem_fill(2.0, 2; pids=[id_me, id_other])) +@test 3.0 == remotecall_fetch(D->D[1], id_other, Base.shmem_fill(3.0, 1; pids=[id_me, id_other])) # Test @parallel load balancing - all processors should get either M or M+1 # iterations out of the loop range for some M. @@ -265,15 +281,15 @@ end # Testing buffered and unbuffered reads # This large array should write directly to the socket a = ones(10^6) -@test a == remotecall_fetch(id_other, (x)->x, a) +@test a == remotecall_fetch((x)->x, id_other, a) # Not a bitstype, should be buffered s = [randstring() for x in 1:10^5] -@test s == remotecall_fetch(id_other, (x)->x, s) +@test s == remotecall_fetch((x)->x, id_other, s) #large number of small requests num_small_requests = 10000 -@test fill(id_other, num_small_requests) == [remotecall_fetch(id_other, myid) for i in 1:num_small_requests] +@test fill(id_other, num_small_requests) == [remotecall_fetch(myid, id_other) for i in 1:num_small_requests] # test parallel sends of large arrays from multiple tasks to the same remote worker ntasks = 10 @@ -283,7 +299,7 @@ for rr in rr_list @async let rr=rr try for i in 1:10 - @test a == remotecall_fetch(id_other, (x)->x, a) + @test a == remotecall_fetch((x)->x, id_other, a) yield() end put!(rr, :OK) @@ -377,7 +393,7 @@ catch ex end try - remotecall_fetch(id_other, ()->throw(ErrorException("foobar"))) + remotecall_fetch(()->throw(ErrorException("foobar")), id_other) catch ex @test typeof(ex) == RemoteException @test typeof(ex.captured) == CapturedException @@ -396,7 +412,7 @@ DoFullTest = Bool(parse(Int,(get(ENV, "JULIA_TESTFULL", "0")))) if DoFullTest # pmap tests # needs at least 4 processors dedicated to the below tests - ppids = remotecall_fetch(1, ()->addprocs(4)) + ppids = remotecall_fetch(()->addprocs(4), 1) s = "abcdefghijklmnopqrstuvwxyz"; ups = uppercase(s); @test ups == bytestring(UInt8[UInt8(c) for c in pmap(x->uppercase(x), s)]) @@ -438,13 +454,15 @@ if DoFullTest println("Testing exception printing on remote worker from a `remote_do` call") println("Please ensure the remote error and backtrace is displayed on screen") - Base.remote_do(id_other, ()->throw(ErrorException("TESTING EXCEPTION ON REMOTE DO. PLEASE IGNORE"))) + Base.remote_do(id_other) do + throw(ErrorException("TESTING EXCEPTION ON REMOTE DO. PLEASE IGNORE")) + end sleep(0.5) # Give some time for the above error to be printed @unix_only begin function test_n_remove_pids(new_pids) for p in new_pids - w_in_remote = sort(remotecall_fetch(p, workers)) + w_in_remote = sort(remotecall_fetch(workers, p)) try @test intersect(new_pids, w_in_remote) == new_pids catch e @@ -456,7 +474,9 @@ if DoFullTest end end - @test :ok == remotecall_fetch(1, (p)->rmprocs(p; waitfor=5.0), new_pids) + @test :ok == remotecall_fetch(1, new_pids) do p + rmprocs(p; waitfor=5.0) + end end print("\n\nTesting SSHManager. A minimum of 4GB of RAM is recommended.\n") @@ -473,22 +493,30 @@ if DoFullTest end print("\nTesting SSH addprocs with $(length(hosts)) workers...\n") - new_pids = remotecall_fetch(1, (h, sf) -> addprocs(h; sshflags=sf), hosts, sshflags) + new_pids = remotecall_fetch(1, hosts, sshflags) do h, sf + addprocs(h; sshflags=sf) + end @test length(new_pids) == length(hosts) test_n_remove_pids(new_pids) print("\nMixed ssh addprocs with :auto\n") - new_pids = sort(remotecall_fetch(1, (h, sf) -> addprocs(h; sshflags=sf), ["localhost", ("127.0.0.1", :auto), "localhost"], sshflags)) + new_pids = sort(remotecall_fetch(1, ["localhost", ("127.0.0.1", :auto), "localhost"], sshflags) do h, sf + addprocs(h; sshflags=sf) + end) @test length(new_pids) == (2 + Sys.CPU_CORES) test_n_remove_pids(new_pids) print("\nMixed ssh addprocs with numeric counts\n") - new_pids = sort(remotecall_fetch(1, (h, sf) -> addprocs(h; sshflags=sf), [("localhost", 2), ("127.0.0.1", 2), "localhost"], sshflags)) + new_pids = sort(remotecall_fetch(1, [("localhost", 2), ("127.0.0.1", 2), "localhost"], sshflags) do h, sf + addprocs(h; sshflags=sf) + end) @test length(new_pids) == 5 test_n_remove_pids(new_pids) print("\nssh addprocs with tunnel\n") - new_pids = sort(remotecall_fetch(1, (h, sf) -> addprocs(h; tunnel=true, sshflags=sf), [("localhost", num_workers)], sshflags)) + new_pids = sort(remotecall_fetch(1, [("localhost", num_workers)], sshflags) do h, sf + addprocs(h; tunnel=true, sshflags=sf) + end) @test length(new_pids) == num_workers test_n_remove_pids(new_pids) diff --git a/test/topology.jl b/test/topology.jl index 70a5e9792060c..0c3015c33901c 100644 --- a/test/topology.jl +++ b/test/topology.jl @@ -2,7 +2,7 @@ include("testdefs.jl") addprocs(4; topology="master_slave") -@test_throws RemoteException remotecall_fetch(2, ()->remotecall_fetch(3,myid)) +@test_throws RemoteException remotecall_fetch(()->remotecall_fetch(3,myid), 2) function test_worker_counts() # check if the nprocs/nworkers/workers are the same on the remaining workers @@ -11,7 +11,9 @@ function test_worker_counts() ws=sort(workers()) for p in workers() - @test (true, true, true) == remotecall_fetch(p, (x,y,z)->(x==nprocs(), y==nworkers(), z==sort(workers())), np, nw, ws) + @test (true, true, true) == remotecall_fetch(p, np, nw, ws) for (x,y,z) + (x==nprocs(), y==nworkers(), z==sort(workers())) + end end end @@ -75,9 +77,9 @@ for p1 in workers() i1 = map_pid_ident[p1] i2 = map_pid_ident[p2] if (iseven(i1) && iseven(i2)) || (isodd(i1) && isodd(i2)) - @test p2 == remotecall_fetch(p1, p->remotecall_fetch(p,myid), p2) + @test p2 == remotecall_fetch(p->remotecall_fetch(p,myid), p1, p2) else - @test_throws RemoteException remotecall_fetch(p1, p->remotecall_fetch(p,myid), p2) + @test_throws RemoteException remotecall_fetch(p->remotecall_fetch(p,myid), p1, p2) end end end From 84204a8b3bd1e80217e600453760fa1f822257e3 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Mon, 28 Sep 2015 14:38:56 -0400 Subject: [PATCH 0264/1938] docs: fix wrong indentation causing accidental literal quote blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReStructured is very finicky about this – bullets, etc. must be fully flush left relative to their parent block or Sphinx wraps them in a literal quote block. This commit also fixes a few other broken RST syntax usages. --- doc/devdocs/llvm.rst | 4 +- doc/devdocs/stdio.rst | 14 +-- doc/devdocs/types.rst | 28 +++--- doc/manual/calling-c-and-fortran-code.rst | 2 +- doc/manual/packages.rst | 43 ++++----- doc/manual/parallel-computing.rst | 110 +++++++++++----------- doc/manual/types.rst | 4 +- doc/manual/workflow-tips.rst | 58 ++++++------ doc/stdlib/base.rst | 10 +- doc/stdlib/collections.rst | 2 +- doc/stdlib/linalg.rst | 104 ++++++++++---------- doc/stdlib/math.rst | 6 +- doc/stdlib/numbers.rst | 4 +- doc/stdlib/parallel.rst | 45 ++++----- doc/stdlib/punctuation.rst | 85 +++++++++-------- doc/stdlib/strings.rst | 28 +++--- 16 files changed, 275 insertions(+), 272 deletions(-) diff --git a/doc/devdocs/llvm.rst b/doc/devdocs/llvm.rst index addc45f36742e..101122568ca4a 100644 --- a/doc/devdocs/llvm.rst +++ b/doc/devdocs/llvm.rst @@ -58,7 +58,9 @@ Building Julia with a different version of LLVM The default version of LLVM is specified in ``deps/Versions.make``. You can override it by creating a file called ``Make.user`` in the top-level directory and adding a line to it such as: - ``LLVM_VER = 3.5.0`` +```make +LLVM_VER = 3.5.0 +``` Besides the LLVM release numerals, you can also use ``LLVM_VER = svn`` to bulid against the latest development version diff --git a/doc/devdocs/stdio.rst b/doc/devdocs/stdio.rst index c2f1599a08ef2..6e5560f26a21b 100644 --- a/doc/devdocs/stdio.rst +++ b/doc/devdocs/stdio.rst @@ -101,9 +101,9 @@ It provides cross-platform buffered file IO and in-memory temporary buffers. :code:`ios.c` is still used by: - - :code:`julia/src/flisp/*.c` - - :code:`julia/src/dump.c` -- for serialisation file IO and for memory buffers. - - :code:`base/iostream.jl` -- for file IO (see :code:`base/fs.jl` for ``libuv`` equivalent). +- :code:`julia/src/flisp/*.c` +- :code:`julia/src/dump.c` -- for serialisation file IO and for memory buffers. +- :code:`base/iostream.jl` -- for file IO (see :code:`base/fs.jl` for ``libuv`` equivalent). Use of :code:`ios.c` in these modules is mostly self-contained and separated from the ``libuv`` I/O system. However, there is `one place @@ -120,6 +120,8 @@ This is needed because :c:func:`jl_printf` caller :c:func:`jl_static_show` is passed an :code:`ios_t` stream by femtolisp's :c:func:`fl_print` function. Julia's :c:func:`jl_write` function has special handling for this:: - if (stream->type > UV_HANDLE_TYPE_MAX) { - return ios_write((ios_t*)stream, str, n); - } +```c +if (stream->type > UV_HANDLE_TYPE_MAX) { + return ios_write((ios_t*)stream, str, n); +} +``` diff --git a/doc/devdocs/types.rst b/doc/devdocs/types.rst index cfea8aff7155a..e4470fbdc521c 100644 --- a/doc/devdocs/types.rst +++ b/doc/devdocs/types.rst @@ -578,20 +578,20 @@ constraints that we just saw in ``typeintersect``. ``jltypes.c`` contains three closely related collections of functions for testing how types ``a`` and ``b`` are ordered: - - The ``subtype`` functions implement ``a <: b``. Among other uses, they - serve in matching function arguments against method signatures in - the function cache. - - - The ``type_morespecific`` functions are used for imposing a partial - order on functions in method tables (from most-to-least - specific). Note that ``jl_type_morespecific(a,b,0)`` really means "is ``a`` - at least as specific as ``b``?" and not "is ``a`` strictly more specific - than ``b``?" - - - The ``type_match`` functions are similar to ``type_morespecific``, but - additionally accept (and employ) an environment to constrain - typevars. The related ``type_match_morespecific`` functions call - ``type_match`` with an argument ``morespecific=1`` +- The ``subtype`` functions implement ``a <: b``. Among other uses, they + serve in matching function arguments against method signatures in + the function cache. + +- The ``type_morespecific`` functions are used for imposing a partial + order on functions in method tables (from most-to-least + specific). Note that ``jl_type_morespecific(a,b,0)`` really means "is ``a`` + at least as specific as ``b``?" and not "is ``a`` strictly more specific + than ``b``?" + +- The ``type_match`` functions are similar to ``type_morespecific``, but + additionally accept (and employ) an environment to constrain + typevars. The related ``type_match_morespecific`` functions call + ``type_match`` with an argument ``morespecific=1`` All three of these take an argument, ``invariant``, which is set to 1 when comparing type parameters and otherwise is 0. diff --git a/doc/manual/calling-c-and-fortran-code.rst b/doc/manual/calling-c-and-fortran-code.rst index 0226c73f6aae4..d12ed5910d576 100644 --- a/doc/manual/calling-c-and-fortran-code.rst +++ b/doc/manual/calling-c-and-fortran-code.rst @@ -678,7 +678,7 @@ they act like zero-dimensional arrays. Special Reference Syntax for ccall (deprecated): ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - The ``&`` syntax is deprecated, use the ``Ref{T}`` argument type instead +The ``&`` syntax is deprecated, use the ``Ref{T}`` argument type instead. A prefix ``&`` is used on an argument to ccall to indicate that a pointer to a scalar argument should be passed instead of the scalar value itself diff --git a/doc/manual/packages.rst b/doc/manual/packages.rst index 5432c1fe9bc60..49d77e52e338a 100644 --- a/doc/manual/packages.rst +++ b/doc/manual/packages.rst @@ -626,50 +626,51 @@ Here are some guidelines to follow in naming your package: 1. Avoid jargon. In particular, avoid acronyms unless there is minimal possibility of confusion. - * It's ok to say ``USA`` if you're talking about the USA. + * It's ok to say ``USA`` if you're talking about the USA. - * It's not ok to say ``PMA``, even if you're talking about positive mental - attitude. + * It's not ok to say ``PMA``, even if you're talking about positive mental + attitude. 2. Avoid using ``Julia`` in your package name. - * It is usually clear from context and to your users that the package is a - Julia package. - * Having Julia in the name can imply that the package is connected to, or - endorsed by, contributors to the Julia language itself. + * It is usually clear from context and to your users that the package is a + Julia package. + + * Having Julia in the name can imply that the package is connected to, or + endorsed by, contributors to the Julia language itself. 3. Packages that provide most of their functionality in association with a new type should have pluralized names. - * ``DataFrames`` provides the ``DataFrame`` type. + * ``DataFrames`` provides the ``DataFrame`` type. - * ``BloomFilters`` provides the ``BloomFilter`` type. + * ``BloomFilters`` provides the ``BloomFilter`` type. - * In contrast, ``JuliaParser`` provides no new type, but instead new - functionality in the ``JuliaParser.parse()`` function. + * In contrast, ``JuliaParser`` provides no new type, but instead new + functionality in the ``JuliaParser.parse()`` function. 4. Err on the side of clarity, even if clarity seems long-winded to you. - * ``RandomMatrices`` is a less ambiguous name than ``RndMat`` or ``RMT``, - even though the latter are shorter. + * ``RandomMatrices`` is a less ambiguous name than ``RndMat`` or ``RMT``, + even though the latter are shorter. 5. A less systematic name may suit a package that implements one of several possible approaches to its domain. - * Julia does not have a single comprehensive plotting package. Instead, - ``Gadfly``, ``PyPlot``, ``Winston`` and other packages each implement a - unique approach based on a particular design philosophy. + * Julia does not have a single comprehensive plotting package. Instead, + ``Gadfly``, ``PyPlot``, ``Winston`` and other packages each implement a + unique approach based on a particular design philosophy. - * In contrast, ``SortingAlgorithms`` provides a consistent interface to use - many well-established sorting algorithms. + * In contrast, ``SortingAlgorithms`` provides a consistent interface to use + many well-established sorting algorithms. 6. Packages that wrap external libraries or programs should be named after those libraries or programs. - * ``CPLEX.jl`` wraps the ``CPLEX`` library, which can be identified easily in - a web search. + * ``CPLEX.jl`` wraps the ``CPLEX`` library, which can be identified easily in + a web search. - * ``MATLAB.jl`` provides an interface to call the MATLAB engine from within Julia. + * ``MATLAB.jl`` provides an interface to call the MATLAB engine from within Julia. Generating the package ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index 305ce9a6dbaa2..067dfc4480393 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -195,11 +195,11 @@ considered to be all processes other than process 1. The base Julia installation has in-built support for two types of clusters: - - A local cluster specified with the ``-p`` option as shown above. +- A local cluster specified with the ``-p`` option as shown above. - - A cluster spanning machines using the ``--machinefile`` option. This uses a passwordless - ``ssh`` login to start julia worker processes (from the same path as the current host) - on the specified machines. +- A cluster spanning machines using the ``--machinefile`` option. This uses a passwordless + ``ssh`` login to start julia worker processes (from the same path as the current host) + on the specified machines. Functions :func:`addprocs`, :func:`rmprocs`, :func:`workers`, and others are available as a programmatic means of adding, removing and querying the processes in a cluster. @@ -687,6 +687,7 @@ A julia cluster has the following characteristics: - All processes can directly communicate with each other. Connections between workers (using the in-built TCP/IP transport) is established in the following manner: + - :func:`addprocs` is called on the master process with a :obj:`ClusterManager` object - :func:`addprocs` calls the appropriate :func:`launch` method which spawns required number of worker processes on appropriate machines @@ -725,7 +726,6 @@ Thus, a minimal cluster manager would need to: ... end - As an example let us see how the :class:`LocalManager`, the manager responsible for starting workers on the same host, is implemented:: @@ -743,10 +743,10 @@ starting workers on the same host, is implemented:: The :func:`launch` method takes the following arguments: - - ``manager::ClusterManager`` - the cluster manager :func:`addprocs` is called with - - ``params::Dict`` - all the keyword arguments passed to :func:`addprocs` - - ``launched::Array`` - the array to append one or more ``WorkerConfig`` objects to - - ``c::Condition`` - the condition variable to be notified as and when workers are launched +- ``manager::ClusterManager`` - the cluster manager :func:`addprocs` is called with +- ``params::Dict`` - all the keyword arguments passed to :func:`addprocs` +- ``launched::Array`` - the array to append one or more ``WorkerConfig`` objects to +- ``c::Condition`` - the condition variable to be notified as and when workers are launched The :func:`launch` method is called asynchronously in a separate task. The termination of this task signals that all requested workers have been launched. Hence the :func:`launch` function MUST exit as soon @@ -765,31 +765,31 @@ before using any of the parallel constructs For every worker launched, the :func:`launch` method must add a :class:`WorkerConfig` object (with appropriate fields initialized) to ``launched`` :: - type WorkerConfig - # Common fields relevant to all cluster managers - io::Nullable{IO} - host::Nullable{AbstractString} - port::Nullable{Integer} + type WorkerConfig + # Common fields relevant to all cluster managers + io::Nullable{IO} + host::Nullable{AbstractString} + port::Nullable{Integer} - # Used when launching additional workers at a host - count::Nullable{Union{Int, Symbol}} - exename::Nullable{AbstractString} - exeflags::Nullable{Cmd} + # Used when launching additional workers at a host + count::Nullable{Union{Int, Symbol}} + exename::Nullable{AbstractString} + exeflags::Nullable{Cmd} - # External cluster managers can use this to store information at a per-worker level - # Can be a dict if multiple fields need to be stored. - userdata::Nullable{Any} + # External cluster managers can use this to store information at a per-worker level + # Can be a dict if multiple fields need to be stored. + userdata::Nullable{Any} - # SSHManager / SSH tunnel connections to workers - tunnel::Nullable{Bool} - bind_addr::Nullable{AbstractString} - sshflags::Nullable{Cmd} - max_parallel::Nullable{Integer} + # SSHManager / SSH tunnel connections to workers + tunnel::Nullable{Bool} + bind_addr::Nullable{AbstractString} + sshflags::Nullable{Cmd} + max_parallel::Nullable{Integer} - connect_at::Nullable{Any} + connect_at::Nullable{Any} - ..... - end + ..... + end Most of the fields in :class:`WorkerConfig` are used by the inbuilt managers. Custom cluster managers would typically specify only ``io`` or ``host`` / ``port``: @@ -817,12 +817,12 @@ required to connect to the workers from the master process. ``manage(manager::FooManager, id::Integer, config::WorkerConfig, op::Symbol)`` is called at different times during the worker's lifetime with appropriate ``op`` values: - - with ``:register``/``:deregister`` when a worker is added / removed - from the Julia worker pool. - - with ``:interrupt`` when ``interrupt(workers)`` is called. The - :class:`ClusterManager` should signal the appropriate worker with an - interrupt signal. - - with ``:finalize`` for cleanup purposes. +- with ``:register``/``:deregister`` when a worker is added / removed + from the Julia worker pool. +- with ``:interrupt`` when ``interrupt(workers)`` is called. The + :class:`ClusterManager` should signal the appropriate worker with an + interrupt signal. +- with ``:finalize`` for cleanup purposes. Cluster Managers with custom transports @@ -832,12 +832,12 @@ Replacing the default TCP/IP all-to-all socket connections with a custom transpo Each julia process has as many communication tasks as the workers it is connected to. For example, consider a julia cluster of 32 processes in a all-to-all mesh network: - - Each julia process thus has 31 communication tasks - - Each task handles all incoming messages from a single remote worker in a message processing loop - - The message processing loop waits on an ``AsyncStream`` object - for example, a TCP socket in the default implementation, reads an entire - message, processes it and waits for the next one - - Sending messages to a process is done directly from any julia task - not just communication tasks - again, via the appropriate - ``AsyncStream`` object +- Each julia process thus has 31 communication tasks +- Each task handles all incoming messages from a single remote worker in a message processing loop +- The message processing loop waits on an ``AsyncStream`` object - for example, a TCP socket in the default implementation, reads an entire + message, processes it and waits for the next one +- Sending messages to a process is done directly from any julia task - not just communication tasks - again, via the appropriate + ``AsyncStream`` object Replacing the default transport involves the new implementation to setup connections to remote workers, and to provide appropriate ``AsyncStream`` objects that the message processing loops can wait on. The manager specific callbacks to be implemented are:: @@ -858,15 +858,15 @@ Note: The julia processes are still all *logically* connected to each other - an awareness of 0MQ being used as the transport layer. When using custom transports: - - julia workers must NOT be started with ``--worker``. Starting with ``--worker`` will result in the newly launched - workers defaulting to the TCP/IP socket transport implementation - - For every incoming logical connection with a worker, ``Base.process_messages(rd::AsyncStream, wr::AsyncStream)`` must be called. - This launches a new task that handles reading and writing of messages from/to the worker represented by the ``AsyncStream`` objects - - ``init_worker(manager::FooManager)`` MUST be called as part of worker process initializaton - - Field ``connect_at::Any`` in :class:`WorkerConfig` can be set by the cluster manager when ``launch`` is called. The value of - this field is passed in in all ``connect`` callbacks. Typically, it carries information on *how to connect* to a worker. For example, - the TCP/IP socket transport uses this field to specify the ``(host, port)`` tuple at which to connect to a worker +- julia workers must NOT be started with ``--worker``. Starting with ``--worker`` will result in the newly launched + workers defaulting to the TCP/IP socket transport implementation +- For every incoming logical connection with a worker, ``Base.process_messages(rd::AsyncStream, wr::AsyncStream)`` must be called. + This launches a new task that handles reading and writing of messages from/to the worker represented by the ``AsyncStream`` objects +- ``init_worker(manager::FooManager)`` MUST be called as part of worker process initializaton +- Field ``connect_at::Any`` in :class:`WorkerConfig` can be set by the cluster manager when ``launch`` is called. The value of + this field is passed in in all ``connect`` callbacks. Typically, it carries information on *how to connect* to a worker. For example, + the TCP/IP socket transport uses this field to specify the ``(host, port)`` tuple at which to connect to a worker ``kill(manager, pid, config)`` is called to remove a worker from the cluster. On the master process, the corresponding ``AsyncStream`` objects must be closed by the implementation to ensure proper cleanup. The default @@ -880,14 +880,14 @@ Specifying network topology (Experimental) Keyword argument ``topology`` to ``addprocs`` is used to specify how the workers must be connected to each other: - - ``:all_to_all`` : is the default, where all workers are connected to each other. +- ``:all_to_all`` : is the default, where all workers are connected to each other. - - ``:master_slave`` : only the driver process, i.e. pid 1 has connections to the workers. +- ``:master_slave`` : only the driver process, i.e. pid 1 has connections to the workers. - - ``:custom`` : the ``launch`` method of the cluster manager specifes the connection topology. - Fields ``ident`` and ``connect_idents`` in ``WorkerConfig`` are used to specify the same. - ``connect_idents`` is a list of ``ClusterManager`` provided identifiers to workers that worker - with identified by ``ident`` must connect to. +- ``:custom`` : the ``launch`` method of the cluster manager specifes the connection topology. + Fields ``ident`` and ``connect_idents`` in ``WorkerConfig`` are used to specify the same. + ``connect_idents`` is a list of ``ClusterManager`` provided identifiers to workers that worker + with identified by ``ident`` must connect to. Currently sending a message between unconnected workers results in an error. This behaviour, as also the functionality and interface should be considered experimental in nature and may change in future releases. diff --git a/doc/manual/types.rst b/doc/manual/types.rst index 8429e96136fe1..5c0945c48d736 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -684,8 +684,8 @@ subtypes of each other: This last point is very important: - **Even though** ``Float64 <: Real`` **we DO NOT have** - ``Point{Float64} <: Point{Real}``\ **.** +- **Even though** ``Float64 <: Real`` **we DO NOT have** + ``Point{Float64} <: Point{Real}``\ **.** In other words, in the parlance of type theory, Julia's type parameters are *invariant*, rather than being covariant (or even contravariant). diff --git a/doc/manual/workflow-tips.rst b/doc/manual/workflow-tips.rst index 0068f836545b6..874d6a5daae79 100644 --- a/doc/manual/workflow-tips.rst +++ b/doc/manual/workflow-tips.rst @@ -21,48 +21,48 @@ The most basic Julia workflows involve using a text editor in conjunction with the ``julia`` command line. A common pattern includes the following elements: - - **Put code under development in a temporary module.** Create a file, - say ``Tmp.jl``, and include within it :: +- **Put code under development in a temporary module.** Create a file, + say ``Tmp.jl``, and include within it :: - module Tmp + module Tmp - + - end + end - - **Put your test code in another file.** Create another file, say - ``tst.jl``, which begins with :: +- **Put your test code in another file.** Create another file, say + ``tst.jl``, which begins with :: - import Tmp + import Tmp - and includes tests for the contents of ``Tmp``. The value of using - :obj:`import` versus :obj:`using` is that you can call :obj:`reload` - ``("Tmp")`` instead of having to restart the REPL when your - definitions change. Of course, the cost is the need to prepend - ``Tmp.`` to uses of names defined in your module. (You can lower that - cost by keeping your module name short.) + and includes tests for the contents of ``Tmp``. The value of using + :obj:`import` versus :obj:`using` is that you can call :obj:`reload` + ``("Tmp")`` instead of having to restart the REPL when your + definitions change. Of course, the cost is the need to prepend + ``Tmp.`` to uses of names defined in your module. (You can lower that + cost by keeping your module name short.) - Alternatively, you can wrap the contents of your test file in a - module, as :: + Alternatively, you can wrap the contents of your test file in a + module, as :: - module Tst - using Tmp + module Tst + using Tmp - + - end + end - The advantage is that you can now do :obj:`using` ``Tmp`` in your - test code and can therefore avoid prepending ``Tmp.`` everywhere. - The disadvantage is that code can no longer be selectively copied - to the REPL without some tweaking. + The advantage is that you can now do :obj:`using` ``Tmp`` in your + test code and can therefore avoid prepending ``Tmp.`` everywhere. + The disadvantage is that code can no longer be selectively copied + to the REPL without some tweaking. - - **Lather. Rinse. Repeat.** Explore ideas at the ``julia`` command - prompt. Save good ideas in ``tst.jl``. Occasionally - restart the REPL, issuing :: +- **Lather. Rinse. Repeat.** Explore ideas at the ``julia`` command + prompt. Save good ideas in ``tst.jl``. Occasionally + restart the REPL, issuing :: - reload("Tmp") - include("tst.jl") + reload("Tmp") + include("tst.jl") Simplify initialization ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 89e9e7c615b86..ae5bb7e30b031 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -872,9 +872,9 @@ System **Examples**: - * ``run(pipeline(`ls`, `grep xyz`))`` - * ``run(pipeline(`ls`, "out.txt"))`` - * ``run(pipeline("out.txt", `grep xyz`))`` + * ``run(pipeline(`ls`, `grep xyz`))`` + * ``run(pipeline(`ls`, "out.txt"))`` + * ``run(pipeline("out.txt", `grep xyz`))`` .. function:: pipeline(command; stdin, stdout, stderr, append=false) @@ -884,8 +884,8 @@ System **Examples**: - * ``run(pipeline(`dothings`, stdout="out.txt", stderr="errs.txt"))`` - * ``run(pipeline(`update`, stdout="log.txt", append=true))`` + * ``run(pipeline(`dothings`, stdout="out.txt", stderr="errs.txt"))`` + * ``run(pipeline(`update`, stdout="log.txt", append=true))`` .. function:: gethostname() -> AbstractString diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index ab19b3c0997ec..906374baa2c96 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -1335,7 +1335,7 @@ changed efficiently. :obj:`PriorityQueue` also behaves similarly to a :obj:`Dict` in that keys can be inserted and priorities accessed or changed using indexing notation. - .. doctest:: +.. doctest:: julia> # Julia code pq = Collections.PriorityQueue(); diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 9ab68658b67cf..241db44fbfbab 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -71,39 +71,39 @@ Linear algebra functions in Julia are largely implemented by calling functions f Compute the LU factorization of ``A``. The return type of ``F`` depends on the type of ``A``. In most cases, if ``A`` is a subtype ``S`` of AbstractMatrix with an element type ``T`` supporting ``+``, ``-``, ``*`` and ``/`` the return type is ``LU{T,S{T}}``. If pivoting is chosen (default) the element type should also support ``abs`` and ``<``. When ``A`` is sparse and have element of type ``Float32``, ``Float64``, ``Complex{Float32}``, or ``Complex{Float64}`` the return type is ``UmfpackLU``. Some examples are shown in the table below. - ======================= ========================= ======================================== - Type of input ``A`` Type of output ``F`` Relationship between ``F`` and ``A`` - ----------------------- ------------------------- ---------------------------------------- - :func:`Matrix` ``LU`` ``F[:L]*F[:U] == A[F[:p], :]`` - :func:`Tridiagonal` ``LU{T,Tridiagonal{T}}`` ``F[:L]*F[:U] == A[F[:p], :]`` - :func:`SparseMatrixCSC` ``UmfpackLU`` ``F[:L]*F[:U] == (F[:Rs] .* A)[F[:p], F[:q]]`` - ======================= ========================= ======================================== + ======================= ========================= ======================================== + Type of input ``A`` Type of output ``F`` Relationship between ``F`` and ``A`` + ----------------------- ------------------------- ---------------------------------------- + :func:`Matrix` ``LU`` ``F[:L]*F[:U] == A[F[:p], :]`` + :func:`Tridiagonal` ``LU{T,Tridiagonal{T}}`` ``F[:L]*F[:U] == A[F[:p], :]`` + :func:`SparseMatrixCSC` ``UmfpackLU`` ``F[:L]*F[:U] == (F[:Rs] .* A)[F[:p], F[:q]]`` + ======================= ========================= ======================================== The individual components of the factorization ``F`` can be accessed by indexing: - =========== ======================================= ====== ======================== ============= - Component Description ``LU`` ``LU{T,Tridiagonal{T}}`` ``UmfpackLU`` - ----------- --------------------------------------- ------ ------------------------ ------------- - ``F[:L]`` ``L`` (lower triangular) part of ``LU`` ✓ ✓ ✓ - ``F[:U]`` ``U`` (upper triangular) part of ``LU`` ✓ ✓ ✓ - ``F[:p]`` (right) permutation ``Vector`` ✓ ✓ ✓ - ``F[:P]`` (right) permutation ``Matrix`` ✓ ✓ - ``F[:q]`` left permutation ``Vector`` ✓ - ``F[:Rs]`` ``Vector`` of scaling factors ✓ - ``F[:(:)]`` ``(L,U,p,q,Rs)`` components ✓ - =========== ======================================= ====== ======================== ============= - - ================== ====== ======================== ============= - Supported function ``LU`` ``LU{T,Tridiagonal{T}}`` ``UmfpackLU`` - ------------------ ------ ------------------------ ------------- - ``/`` ✓ - ``\`` ✓ ✓ ✓ - ``cond`` ✓ ✓ - ``det`` ✓ ✓ ✓ - ``logdet`` ✓ ✓ - ``logabsdet`` ✓ ✓ - ``size`` ✓ ✓ - ================== ====== ======================== ============= + =========== ======================================= ====== ======================== ============= + Component Description ``LU`` ``LU{T,Tridiagonal{T}}`` ``UmfpackLU`` + ----------- --------------------------------------- ------ ------------------------ ------------- + ``F[:L]`` ``L`` (lower triangular) part of ``LU`` ✓ ✓ ✓ + ``F[:U]`` ``U`` (upper triangular) part of ``LU`` ✓ ✓ ✓ + ``F[:p]`` (right) permutation ``Vector`` ✓ ✓ ✓ + ``F[:P]`` (right) permutation ``Matrix`` ✓ ✓ + ``F[:q]`` left permutation ``Vector`` ✓ + ``F[:Rs]`` ``Vector`` of scaling factors ✓ + ``F[:(:)]`` ``(L,U,p,q,Rs)`` components ✓ + =========== ======================================= ====== ======================== ============= + + ================== ====== ======================== ============= + Supported function ``LU`` ``LU{T,Tridiagonal{T}}`` ``UmfpackLU`` + ------------------ ------ ------------------------ ------------- + ``/`` ✓ + ``\`` ✓ ✓ ✓ + ``cond`` ✓ ✓ + ``det`` ✓ ✓ ✓ + ``logdet`` ✓ ✓ + ``logabsdet`` ✓ ✓ + ``size`` ✓ ✓ + ================== ====== ======================== ============= .. function:: lufact!(A) -> LU @@ -173,26 +173,26 @@ Linear algebra functions in Julia are largely implemented by calling functions f Computes the QR factorization of ``A``. The return type of ``F`` depends on the element type of ``A`` and whether pivoting is specified (with ``pivot==Val{true}``). - ================ ================= ============== ===================================== - Return type ``eltype(A)`` ``pivot`` Relationship between ``F`` and ``A`` - ---------------- ----------------- -------------- ------------------------------------- - ``QR`` not ``BlasFloat`` either ``A==F[:Q]*F[:R]`` - ``QRCompactWY`` ``BlasFloat`` ``Val{false}`` ``A==F[:Q]*F[:R]`` - ``QRPivoted`` ``BlasFloat`` ``Val{true}`` ``A[:,F[:p]]==F[:Q]*F[:R]`` - ================ ================= ============== ===================================== + ================ ================= ============== ===================================== + Return type ``eltype(A)`` ``pivot`` Relationship between ``F`` and ``A`` + ---------------- ----------------- -------------- ------------------------------------- + ``QR`` not ``BlasFloat`` either ``A==F[:Q]*F[:R]`` + ``QRCompactWY`` ``BlasFloat`` ``Val{false}`` ``A==F[:Q]*F[:R]`` + ``QRPivoted`` ``BlasFloat`` ``Val{true}`` ``A[:,F[:p]]==F[:Q]*F[:R]`` + ================ ================= ============== ===================================== ``BlasFloat`` refers to any of: ``Float32``, ``Float64``, ``Complex64`` or ``Complex128``. The individual components of the factorization ``F`` can be accessed by indexing: - =========== ============================================= ================== ===================== ================== - Component Description ``QR`` ``QRCompactWY`` ``QRPivoted`` - ----------- --------------------------------------------- ------------------ --------------------- ------------------ - ``F[:Q]`` ``Q`` (orthogonal/unitary) part of ``QR`` ✓ (``QRPackedQ``) ✓ (``QRCompactWYQ``) ✓ (``QRPackedQ``) - ``F[:R]`` ``R`` (upper right triangular) part of ``QR`` ✓ ✓ ✓ - ``F[:p]`` pivot ``Vector`` ✓ - ``F[:P]`` (pivot) permutation ``Matrix`` ✓ - =========== ============================================= ================== ===================== ================== + =========== ============================================= ================== ===================== ================== + Component Description ``QR`` ``QRCompactWY`` ``QRPivoted`` + ----------- --------------------------------------------- ------------------ --------------------- ------------------ + ``F[:Q]`` ``Q`` (orthogonal/unitary) part of ``QR`` ✓ (``QRPackedQ``) ✓ (``QRCompactWYQ``) ✓ (``QRPackedQ``) + ``F[:R]`` ``R`` (upper right triangular) part of ``QR`` ✓ ✓ ✓ + ``F[:p]`` pivot ``Vector`` ✓ + ``F[:P]`` (pivot) permutation ``Matrix`` ✓ + =========== ============================================= ================== ===================== ================== The following functions are available for the ``QR`` objects: ``size``, ``\``. When ``A`` is rectangular, ``\`` will return a least squares solution and if the solution is not unique, the one with smallest norm is returned. @@ -204,17 +204,17 @@ Linear algebra functions in Julia are largely implemented by calling functions f The data contained in ``QR`` or ``QRPivoted`` can be used to construct the ``QRPackedQ`` type, which is a compact representation of the rotation matrix: - .. math:: + .. math:: - Q = \prod_{i=1}^{\min(m,n)} (I - \tau_i v_i v_i^T) + Q = \prod_{i=1}^{\min(m,n)} (I - \tau_i v_i v_i^T) where :math:`\tau_i` is the scale factor and :math:`v_i` is the projection vector associated with the :math:`i^{th}` Householder elementary reflector. The data contained in ``QRCompactWY`` can be used to construct the ``QRCompactWYQ`` type, which is a compact representation of the rotation matrix - .. math:: + .. math:: - Q = I + Y T Y^T + Q = I + Y T Y^T where ``Y`` is :math:`m \times r` lower trapezoidal and ``T`` is :math:`r \times r` upper triangular. The *compact WY* representation [Schreiber1989]_ is not to be confused with the older, *WY* representation [Bischof1987]_. (The LAPACK documentation uses ``V`` in lieu of ``Y``.) @@ -928,8 +928,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f The following keyword arguments are supported: * ``nev``: Number of eigenvalues * ``ncv``: Number of Krylov vectors used in the computation; should satisfy ``nev+1 <= ncv <= n`` for real symmetric problems and ``nev+2 <= ncv <= n`` for other problems, where ``n`` is the size of the input matrix ``A``. The default is ``ncv = max(20,2*nev+1)``. - - Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. + Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. * ``which``: type of eigenvalues to compute. See the note below. ========= ====================================================================================================================== @@ -971,8 +970,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f The following keyword arguments are supported: * ``nev``: Number of eigenvalues * ``ncv``: Number of Krylov vectors used in the computation; should satisfy ``nev+1 <= ncv <= n`` for real symmetric problems and ``nev+2 <= ncv <= n`` for other problems, where ``n`` is the size of the input matrices ``A`` and ``B``. The default is ``ncv = max(20,2*nev+1)``. - - Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. + Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. * ``which``: type of eigenvalues to compute. See the note below. ========= ====================================================================================================================== diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index b17c8b2d6d85a..592f1dfc9132b 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -1699,9 +1699,9 @@ Statistics This function accepts three keyword arguments: - * ``vardim``\ : the dimension of variables. When ``vardim = 1``\ , variables are considered in columns while observations in rows; when ``vardim = 2``\ , variables are in rows while observations in columns. By default, it is set to ``1``\ . - * ``corrected``\ : whether to apply Bessel's correction (divide by ``n-1`` instead of ``n``\ ). By default, it is set to ``true``\ . - * ``mean``\ : allow users to supply mean values that are known. By default, it is set to ``nothing``\ , which indicates that the mean(s) are unknown, and the function will compute the mean. Users can use ``mean=0`` to indicate that the input data are centered, and hence there's no need to subtract the mean. + * ``vardim``\ : the dimension of variables. When ``vardim = 1``\ , variables are considered in columns while observations in rows; when ``vardim = 2``\ , variables are in rows while observations in columns. By default, it is set to ``1``\ . + * ``corrected``\ : whether to apply Bessel's correction (divide by ``n-1`` instead of ``n``\ ). By default, it is set to ``true``\ . + * ``mean``\ : allow users to supply mean values that are known. By default, it is set to ``nothing``\ , which indicates that the mean(s) are unknown, and the function will compute the mean. Users can use ``mean=0`` to indicate that the input data are centered, and hence there's no need to subtract the mean. The size of the result depends on the size of ``v1`` and ``v2``\ . When both ``v1`` and ``v2`` are vectors, it returns the covariance between them as a scalar. When either one is a matrix, it returns a covariance matrix of size ``(n1, n2)``\ , where ``n1`` and ``n2`` are the numbers of slices in ``v1`` and ``v2``\ , which depend on the setting of ``vardim``\ . diff --git a/doc/stdlib/numbers.rst b/doc/stdlib/numbers.rst index ca5fe33b7bd7b..0f74f158c29fa 100644 --- a/doc/stdlib/numbers.rst +++ b/doc/stdlib/numbers.rst @@ -605,8 +605,8 @@ As ``BigInt`` represents unbounded integers, the interval must be specified (e.g Pick a random element or array of random elements from the set of values specified by ``S``\ ; ``S`` can be - * an indexable collection (for example ``1:n`` or ``['x','y','z']``\ ), or - * a type: the set of values to pick from is then equivalent to ``typemin(S):typemax(S)`` for integers (this is not applicable to ``BigInt``\ ), and to :math:`[0, 1)` for floating point numbers; + * an indexable collection (for example ``1:n`` or ``['x','y','z']``\ ), or + * a type: the set of values to pick from is then equivalent to ``typemin(S):typemax(S)`` for integers (this is not applicable to ``BigInt``\ ), and to :math:`[0, 1)` for floating point numbers; ``S`` defaults to ``Float64``\ . diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 6075736808aa8..7bde525093cb4 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -137,8 +137,8 @@ Tasks Other constructors: - * ``Channel()`` - equivalent to ``Channel{Any}(32)`` - * ``Channel(sz::Int)`` equivalent to ``Channel{Any}(sz)`` + * ``Channel()`` - equivalent to ``Channel{Any}(32)`` + * ``Channel(sz::Int)`` equivalent to ``Channel{Any}(sz)`` General Parallel Computing Support ---------------------------------- @@ -276,12 +276,12 @@ General Parallel Computing Support Block the current task until some event occurs, depending on the type of the argument: - * ``RemoteRef``\ : Wait for a value to become available for the specified remote reference. - * ``Channel``\ : Wait for a value to be appended to the channel. - * ``Condition``\ : Wait for ``notify`` on a condition. - * ``Process``\ : Wait for a process or process chain to exit. The ``exitcode`` field of a process can be used to determine success or failure. - * ``Task``\ : Wait for a ``Task`` to finish, returning its result value. If the task fails with an exception, the exception is propagated (re-thrown in the task that called ``wait``\ ). - * ``RawFD``\ : Wait for changes on a file descriptor (see ``poll_fd`` for keyword arguments and return code) + * ``RemoteRef``\ : Wait for a value to become available for the specified remote reference. + * ``Channel``\ : Wait for a value to be appended to the channel. + * ``Condition``\ : Wait for ``notify`` on a condition. + * ``Process``\ : Wait for a process or process chain to exit. The ``exitcode`` field of a process can be used to determine success or failure. + * ``Task``\ : Wait for a ``Task`` to finish, returning its result value. If the task fails with an exception, the exception is propagated (re-thrown in the task that called ``wait``\ ). + * ``RawFD``\ : Wait for changes on a file descriptor (see ``poll_fd`` for keyword arguments and return code) If no argument is passed, the task blocks for an undefined period. If the task's state is set to ``:waiting``\ , it can only be restarted by an explicit call to ``schedule`` or ``yieldto``\ . If the task's state is ``:runnable``\ , it might be restarted unpredictably. @@ -293,8 +293,8 @@ General Parallel Computing Support Waits and fetches a value from ``x`` depending on the type of ``x``\ . Does not remove the item fetched: - * ``RemoteRef``\ : Wait for and get the value of a remote reference. If the remote value is an exception, throws a ``RemoteException`` which captures the remote exception and backtrace. - * ``Channel`` : Wait for and get the first available item from the channel. + * ``RemoteRef``\ : Wait for and get the value of a remote reference. If the remote value is an exception, throws a ``RemoteException`` which captures the remote exception and backtrace. + * ``Channel`` : Wait for and get the first available item from the channel. .. function:: remotecall_wait(id, func, args...) @@ -352,8 +352,8 @@ General Parallel Computing Support Closes a channel. An exception is thrown by: - * ``put!`` on a closed channel. - * ``take!`` and ``fetch`` on an empty, closed channel. + * ``put!`` on a closed channel. + * ``take!`` and ``fetch`` on an empty, closed channel. .. function:: RemoteRef() @@ -470,10 +470,11 @@ Shared Arrays (Experimental, UNIX-only feature) Cluster Manager Interface ------------------------- - This interface provides a mechanism to launch and manage Julia workers on different cluster environments. - LocalManager, for launching additional workers on the same host and SSHManager, for launching on remote - hosts via ssh are present in Base. TCP/IP sockets are used to connect and transport messages - between processes. It is possible for Cluster Managers to provide a different transport. + +This interface provides a mechanism to launch and manage Julia workers on different cluster environments. +LocalManager, for launching additional workers on the same host and SSHManager, for launching on remote +hosts via ssh are present in Base. TCP/IP sockets are used to connect and transport messages +between processes. It is possible for Cluster Managers to provide a different transport. .. function:: launch(manager::FooManager, params::Dict, launched::Vector{WorkerConfig}, launch_ntfy::Condition) @@ -488,12 +489,12 @@ Cluster Manager Interface Implemented by cluster managers. It is called on the master process, during a worker's lifetime, with appropriate ``op`` values: - - with ``:register``/``:deregister`` when a worker is added / removed - from the Julia worker pool. - - with ``:interrupt`` when ``interrupt(workers)`` is called. The - :class:`ClusterManager` should signal the appropriate worker with an - interrupt signal. - - with ``:finalize`` for cleanup purposes. + - with ``:register``/``:deregister`` when a worker is added / removed + from the Julia worker pool. + - with ``:interrupt`` when ``interrupt(workers)`` is called. The + :class:`ClusterManager` should signal the appropriate worker with an + interrupt signal. + - with ``:finalize`` for cleanup purposes. .. function:: kill(manager::FooManager, pid::Int, config::WorkerConfig) diff --git a/doc/stdlib/punctuation.rst b/doc/stdlib/punctuation.rst index 42f987213ccbd..725ab284befdd 100644 --- a/doc/stdlib/punctuation.rst +++ b/doc/stdlib/punctuation.rst @@ -4,46 +4,45 @@ Extended documentation for mathematical symbols & functions is :ref:`here `. - ========= ================================================ - symbol meaning - ========= ================================================ - ``@m`` invoke macro ``m``; followed by space-separated expressions - ``!`` prefix "not" operator - ``a!( )`` at the end of a function name, `!` indicates that a function modifies its argument(s) - ``#`` begin single line comment - ``#=`` begin multi-line comment (these are nestable) - ``=#`` end multi-line comment - ``$`` bitwise xor operator, string and expression interpolation - ``%`` remainder operator - ``^`` exponent operator - ``&`` bitwise and - ``&&`` short-circuiting boolean and - ``|`` bitwise or - ``||`` short-circuiting boolean or - ``*`` multiply, or matrix multiply - ``()`` the empty tuple - ``~`` bitwise not operator - ``\`` backslash operator - ``'`` complex transpose operator A\ :sup:`H` - ``a[]`` array indexing - ``[,]`` vertical concatenation - ``[;]`` also vertical concatenation - ``[ ]`` with space-separated expressions, horizontal concatenation - ``T{ }`` parametric type instantiation - ``{ }`` construct a cell array (deprecated in 0.4 in favor of ``Any[]``) - ``;`` statement separator - ``,`` separate function arguments or tuple components - ``?`` 3-argument conditional operator (conditional ? if_true : if_false) - ``""`` delimit string literals - ``''`` delimit character literals - ``` ``` delimit external process (command) specifications - ``...`` splice arguments into a function call or declare a varargs function or type - ``.`` access named fields in objects or names inside modules, also prefixes elementwise operators - ``a:b`` range a, a+1, a+2, ..., b - ``a:s:b`` range a, a+s, a+2s, ..., b - ``:`` index an entire dimension (1:end) - ``::`` type annotation, depending on context - ``:( )`` quoted expression - ``:a`` symbol a - ========= ================================================ - +========= ================================================ +symbol meaning +========= ================================================ +``@m`` invoke macro ``m``; followed by space-separated expressions +``!`` prefix "not" operator +``a!( )`` at the end of a function name, `!` indicates that a function modifies its argument(s) +``#`` begin single line comment +``#=`` begin multi-line comment (these are nestable) +``=#`` end multi-line comment +``$`` bitwise xor operator, string and expression interpolation +``%`` remainder operator +``^`` exponent operator +``&`` bitwise and +``&&`` short-circuiting boolean and +``|`` bitwise or +``||`` short-circuiting boolean or +``*`` multiply, or matrix multiply +``()`` the empty tuple +``~`` bitwise not operator +``\`` backslash operator +``'`` complex transpose operator A\ :sup:`H` +``a[]`` array indexing +``[,]`` vertical concatenation +``[;]`` also vertical concatenation +``[ ]`` with space-separated expressions, horizontal concatenation +``T{ }`` parametric type instantiation +``{ }`` construct a cell array (deprecated in 0.4 in favor of ``Any[]``) +``;`` statement separator +``,`` separate function arguments or tuple components +``?`` 3-argument conditional operator (conditional ? if_true : if_false) +``""`` delimit string literals +``''`` delimit character literals +``` ``` delimit external process (command) specifications +``...`` splice arguments into a function call or declare a varargs function or type +``.`` access named fields in objects or names inside modules, also prefixes elementwise operators +``a:b`` range a, a+1, a+2, ..., b +``a:s:b`` range a, a+s, a+2s, ..., b +``:`` index an entire dimension (1:end) +``::`` type annotation, depending on context +``:( )`` quoted expression +``:a`` symbol a +========= ================================================ diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index cdefe2f971b6f..05f8d4adbf488 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -104,10 +104,10 @@ Construct a regex, such as ``r"^[a-z]*$"``\ . The regex also accepts one or more flags, listed after the ending quote, to change its behaviour: - * ``i`` enables case-insensitive matching - * ``m`` treats the ``^`` and ``$`` tokens as matching the start and and end of individual lines, as opposed to the whole string. - * ``s`` allows the ``.`` modifier to match newlines. - * ``x`` enables "comment mode": whitespace is enabled except when escaped with ``\``\ , and ``#`` is treated as starting a comment. + * ``i`` enables case-insensitive matching + * ``m`` treats the ``^`` and ``$`` tokens as matching the start and and end of individual lines, as opposed to the whole string. + * ``s`` allows the ``.`` modifier to match newlines. + * ``x`` enables "comment mode": whitespace is enabled except when escaped with ``\``\ , and ``#`` is treated as starting a comment. For example, this regex has all three flags enabled: @@ -136,16 +136,16 @@ Alternatively, finer control and additional transformations may be be obtained by calling ``normalize_string(s; keywords...)``\ , where any number of the following boolean keywords options (which all default to ``false`` except for ``compose``\ ) are specified: - * ``compose=false``\ : do not perform canonical composition - * ``decompose=true``\ : do canonical decomposition instead of canonical composition (``compose=true`` is ignored if present) - * ``compat=true``\ : compatibility equivalents are canonicalized - * ``casefold=true``\ : perform Unicode case folding, e.g. for case-insensitive string comparison - * ``newline2lf=true``\ , ``newline2ls=true``\ , or ``newline2ps=true``\ : convert various newline sequences (LF, CRLF, CR, NEL) into a linefeed (LF), line-separation (LS), or paragraph-separation (PS) character, respectively - * ``stripmark=true``\ : strip diacritical marks (e.g. accents) - * ``stripignore=true``\ : strip Unicode's "default ignorable" characters (e.g. the soft hyphen or the left-to-right marker) - * ``stripcc=true``\ : strip control characters; horizontal tabs and form feeds are converted to spaces; newlines are also converted to spaces unless a newline-conversion flag was specified - * ``rejectna=true``\ : throw an error if unassigned code points are found - * ``stable=true``\ : enforce Unicode Versioning Stability + * ``compose=false``\ : do not perform canonical composition + * ``decompose=true``\ : do canonical decomposition instead of canonical composition (``compose=true`` is ignored if present) + * ``compat=true``\ : compatibility equivalents are canonicalized + * ``casefold=true``\ : perform Unicode case folding, e.g. for case-insensitive string comparison + * ``newline2lf=true``\ , ``newline2ls=true``\ , or ``newline2ps=true``\ : convert various newline sequences (LF, CRLF, CR, NEL) into a linefeed (LF), line-separation (LS), or paragraph-separation (PS) character, respectively + * ``stripmark=true``\ : strip diacritical marks (e.g. accents) + * ``stripignore=true``\ : strip Unicode's "default ignorable" characters (e.g. the soft hyphen or the left-to-right marker) + * ``stripcc=true``\ : strip control characters; horizontal tabs and form feeds are converted to spaces; newlines are also converted to spaces unless a newline-conversion flag was specified + * ``rejectna=true``\ : throw an error if unassigned code points are found + * ``stable=true``\ : enforce Unicode Versioning Stability For example, NFKC corresponds to the options ``compose=true, compat=true, stable=true``\ . From b26d7993bbc0b7423d44526459179277e5f56182 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Mon, 28 Sep 2015 16:59:19 -0400 Subject: [PATCH 0265/1938] fix #13309: return correct real value from var, varm, etc of complex arrays --- base/docs/helpdb.jl | 2 +- base/statistics.jl | 46 ++++++++++++++++++++++----------------------- doc/stdlib/math.rst | 2 +- test/statistics.jl | 17 +++++++++++++++++ 4 files changed, 42 insertions(+), 25 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 23dbcac1673c6..3b0605766cc99 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -6354,7 +6354,7 @@ minimum(A,dims) doc""" var(v[, region]) -Compute the sample variance of a vector or array `v`, optionally along dimensions in `region`. The algorithm will return an estimator of the generative distribution's variance under the assumption that each entry of `v` is an IID drawn from that generative distribution. This computation is equivalent to calculating `sum((v - mean(v)).^2) / (length(v) - 1)`. Note: Julia does not ignore `NaN` values in the computation. For applications requiring the handling of missing data, the `DataArray` package is recommended. +Compute the sample variance of a vector or array `v`, optionally along dimensions in `region`. The algorithm will return an estimator of the generative distribution's variance under the assumption that each entry of `v` is an IID drawn from that generative distribution. This computation is equivalent to calculating `sumabs2(v - mean(v)) / (length(v) - 1)`. Note: Julia does not ignore `NaN` values in the computation. For applications requiring the handling of missing data, the `DataArray` package is recommended. """ var diff --git a/base/statistics.jl b/base/statistics.jl index dc77fca791d5e..5440db590d49a 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -34,6 +34,10 @@ mean{T}(A::AbstractArray{T}, region) = ##### variances ##### +# faster computation of real(conj(x)*y) +realXcY(x::Real, y::Real) = x*y +realXcY(x::Complex, y::Complex) = real(x)*real(y) + imag(x)*imag(y) + function var(iterable; corrected::Bool=true, mean=nothing) state = start(iterable) if done(iterable, state) @@ -45,12 +49,12 @@ function var(iterable; corrected::Bool=true, mean=nothing) # Use Welford algorithm as seen in (among other places) # Knuth's TAOCP, Vol 2, page 232, 3rd edition. M = value / 1 - S = zero(M) + S = real(zero(M)) while !done(iterable, state) value, state = next(iterable, state) count += 1 new_M = M + (value - M) / count - S = S + (value - M) * (value - new_M) + S = S + realXcY(value - M, value - new_M) M = new_M end return S / (count - Int(corrected)) @@ -60,11 +64,11 @@ function var(iterable; corrected::Bool=true, mean=nothing) # by Chan, Golub, and LeVeque, Technical Report STAN-CS-79-773, # Department of Computer Science, Stanford University, # because user can provide mean value that is different to mean(iterable) - sum2 = (value - mean::Number)^2 + sum2 = abs2(value - mean::Number) while !done(iterable, state) value, state = next(iterable, state) count += 1 - sum2 += (value - mean)^2 + sum2 += abs2(value - mean) end return sum2 / (count - Int(corrected)) else @@ -74,7 +78,7 @@ end function varzm{T}(A::AbstractArray{T}; corrected::Bool=true) n = length(A) - n == 0 && return convert(momenttype(T), NaN) + n == 0 && return convert(real(momenttype(T)), NaN) return sumabs2(A) / (n - Int(corrected)) end @@ -89,7 +93,7 @@ function varzm!{S}(R::AbstractArray{S}, A::AbstractArray; corrected::Bool=true) end varzm{T}(A::AbstractArray{T}, region; corrected::Bool=true) = - varzm!(reducedim_initarray(A, region, 0, momenttype(T)), A; corrected=corrected) + varzm!(reducedim_initarray(A, region, 0, real(momenttype(T))), A; corrected=corrected) immutable CentralizedAbs2Fun{T<:Number} <: Func{1} m::T @@ -139,8 +143,8 @@ end function varm{T}(A::AbstractArray{T}, m::Number; corrected::Bool=true) n = length(A) - n == 0 && return convert(momenttype(T), NaN) - n == 1 && return convert(momenttype(T), abs2(A[1] - m)/(1 - Int(corrected))) + n == 0 && return convert(real(momenttype(T)), NaN) + n == 1 && return convert(real(momenttype(T)), abs2(A[1] - m)/(1 - Int(corrected))) return centralize_sumabs2(A, m) / (n - Int(corrected)) end @@ -155,14 +159,15 @@ function varm!{S}(R::AbstractArray{S}, A::AbstractArray, m::AbstractArray; corre end varm{T}(A::AbstractArray{T}, m::AbstractArray, region; corrected::Bool=true) = - varm!(reducedim_initarray(A, region, 0, momenttype(T)), A, m; corrected=corrected) + varm!(reducedim_initarray(A, region, 0, real(momenttype(T))), A, m; corrected=corrected) function var{T}(A::AbstractArray{T}; corrected::Bool=true, mean=nothing) - convert(momenttype(T), mean == 0 ? varzm(A; corrected=corrected) : - mean === nothing ? varm(A, Base.mean(A); corrected=corrected) : - isa(mean, Number) ? varm(A, mean::Number; corrected=corrected) : - throw(ArgumentError("invalid value of mean, $(mean)::$(typeof(mean))")))::momenttype(T) + convert(real(momenttype(T)), + mean == 0 ? varzm(A; corrected=corrected) : + mean === nothing ? varm(A, Base.mean(A); corrected=corrected) : + isa(mean, Number) ? varm(A, mean::Number; corrected=corrected) : + throw(ArgumentError("invalid value of mean, $(mean)::$(typeof(mean))")))::real(momenttype(T)) end function var(A::AbstractArray, region; corrected::Bool=true, mean=nothing) @@ -243,7 +248,7 @@ _vmean(x::AbstractMatrix, vardim::Int) = mean(x, vardim) # core functions -unscaled_covzm(x::AbstractVector) = dot(x, x) +unscaled_covzm(x::AbstractVector) = sumabs2(x) unscaled_covzm(x::AbstractMatrix, vardim::Int) = (vardim == 1 ? _conj(x'x) : x * x') unscaled_covzm(x::AbstractVector, y::AbstractVector) = dot(x, y) @@ -256,7 +261,7 @@ unscaled_covzm(x::AbstractMatrix, y::AbstractMatrix, vardim::Int) = # covzm (with centered data) -covzm(x::AbstractVector; corrected::Bool=true) = unscaled_covzm(x, x) / (length(x) - Int(corrected)) +covzm(x::AbstractVector; corrected::Bool=true) = unscaled_covzm(x) / (length(x) - Int(corrected)) covzm(x::AbstractMatrix; vardim::Int=1, corrected::Bool=true) = scale!(unscaled_covzm(x, vardim), inv(size(x,vardim) - Int(corrected))) @@ -372,7 +377,7 @@ end # corzm (non-exported, with centered data) -corzm{T}(x::AbstractVector{T}) = float(one(T) * one(T)) +corzm{T}(x::AbstractVector{T}) = one(real(T)) corzm(x::AbstractMatrix; vardim::Int=1) = (c = unscaled_covzm(x, vardim); cov2cor!(c, sqrt!(diag(c)))) @@ -408,7 +413,7 @@ corzm(x::AbstractMatrix, y::AbstractMatrix; vardim::Int=1) = # corm -corm(x::AbstractVector, xmean) = corzm(x .- xmean) +corm{T}(x::AbstractVector{T}, xmean) = one(real(T)) corm(x::AbstractMatrix, xmean; vardim::Int=1) = corzm(x .- xmean; vardim=vardim) @@ -419,12 +424,7 @@ corm(x::AbstractVecOrMat, xmean, y::AbstractVecOrMat, ymean; vardim::Int=1) = # cor -function cor(x::AbstractVector; mean=nothing) - mean == 0 ? corzm(x) : - mean === nothing ? corm(x, Base.mean(x)) : - isa(mean, Number) ? corm(x, mean) : - throw(ArgumentError("invalid value of mean, $(mean)::$(typeof(mean))")) -end +cor{T}(x::AbstractVector{T}; mean=nothing) = one(real(T)) function cor(x::AbstractMatrix; vardim::Int=1, mean=nothing) mean == 0 ? corzm(x; vardim=vardim) : diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index b17c8b2d6d85a..2e22a6f2c9ed5 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -1587,7 +1587,7 @@ Statistics .. Docstring generated from Julia source - Compute the sample variance of a vector or array ``v``\ , optionally along dimensions in ``region``\ . The algorithm will return an estimator of the generative distribution's variance under the assumption that each entry of ``v`` is an IID drawn from that generative distribution. This computation is equivalent to calculating ``sum((v - mean(v)).^2) / (length(v) - 1)``\ . Note: Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArray`` package is recommended. + Compute the sample variance of a vector or array ``v``\ , optionally along dimensions in ``region``\ . The algorithm will return an estimator of the generative distribution's variance under the assumption that each entry of ``v`` is an IID drawn from that generative distribution. This computation is equivalent to calculating ``sumabs2(v - mean(v)) / (length(v) - 1)``\ . Note: Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArray`` package is recommended. .. function:: varm(v, m) diff --git a/test/statistics.jl b/test/statistics.jl index 7f313cf8051ec..af65a4cb15eb7 100644 --- a/test/statistics.jl +++ b/test/statistics.jl @@ -310,3 +310,20 @@ end @test_throws ArgumentError histrange([1, 10], 0) @test_throws ArgumentError histrange([1, 10], -1) @test_throws ArgumentError histrange(Float64[],-1) + +# variance of complex arrays (#13309) +let z = rand(Complex128, 10) + @test var(z) ≈ invoke(var, (Any,), z) ≈ cov(z) ≈ var(z,1)[1] ≈ sumabs2(z - mean(z))/9 + @test isa(var(z), Float64) + @test isa(invoke(var, (Any,), z), Float64) + @test isa(cov(z), Float64) + @test isa(var(z,1), Vector{Float64}) + @test varm(z, 0.0) ≈ invoke(varm, (Any,Float64), z, 0.0) ≈ sumabs2(z)/9 + @test isa(varm(z, 0.0), Float64) + @test isa(invoke(varm, (Any,Float64), z, 0.0), Float64) + @test cor(z) === 1.0 +end +let v = varm([1.0+2.0im], 0; corrected = false) + @test v ≈ 5 + @test isa(v, Float64) +end From 8580e9d2070b6a3deed9d0d709ce395e472e1c39 Mon Sep 17 00:00:00 2001 From: "Arch D. Robison" Date: Mon, 28 Sep 2015 16:03:22 -0500 Subject: [PATCH 0266/1938] Fix issue #13106 by adding TargetTransformInfoWrapperPass to pass list. --- src/codegen.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index e178a4597c861..64fdadb1d2f8a 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -35,6 +35,7 @@ #include #include #ifdef LLVM37 +#include #include #else #include @@ -5752,7 +5753,9 @@ static void init_julia_llvm_env(Module *m) FPM->add(llvm::createMemorySanitizerPass(true)); # endif #endif -#ifndef LLVM37 +#ifdef LLVM37 + FPM->add(createTargetTransformInfoWrapperPass(jl_TargetMachine->getTargetIRAnalysis())); +#else jl_TargetMachine->addAnalysisPasses(*FPM); #endif #ifdef LLVM38 From 046f099b5f56e3de5b71fe1609fd9b152715656f Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Mon, 28 Sep 2015 18:15:36 -0400 Subject: [PATCH 0267/1938] Make replace use print instead of write --- base/strings/util.jl | 4 ++-- test/strings/util.jl | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/base/strings/util.jl b/base/strings/util.jl index aefc53f4b2bad..f5106c1c989e3 100644 --- a/base/strings/util.jl +++ b/base/strings/util.jl @@ -173,9 +173,9 @@ function _rsplit{T<:AbstractString,U<:Array}(str::T, splitter, limit::Integer, k end #rsplit(str::AbstractString) = rsplit(str, _default_delims, 0, false) -_replace(io, repl, str, r, pattern) = write(io, repl) +_replace(io, repl, str, r, pattern) = print(io, repl) _replace(io, repl::Function, str, r, pattern) = - write(io, repl(SubString(str, first(r), last(r)))) + print(io, repl(SubString(str, first(r), last(r)))) function replace(str::ByteString, pattern, repl, limit::Integer) n = 1 diff --git a/test/strings/util.jl b/test/strings/util.jl index a1858b0c062e3..d3a4e92034bf6 100644 --- a/test/strings/util.jl +++ b/test/strings/util.jl @@ -205,6 +205,9 @@ end @test replace("foo", "oo", uppercase) == "fOO" +# Issue 13332 +@test replace("abc", 'b', 2.1) == "a2.1c" + # chomp/chop @test chomp("foo\n") == "foo" @test chop("foob") == "foo" From 41d2a217b413d3487bf85406446660537489c590 Mon Sep 17 00:00:00 2001 From: ScottPJones Date: Mon, 28 Sep 2015 20:14:56 -0400 Subject: [PATCH 0268/1938] Fix #13344 compiling issues in debuginfo.cpp --- src/debuginfo.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 86e1fbe1979c3..c2d10b24396d0 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -226,9 +226,9 @@ class JuliaJITEventListener: public JITEventListener uint64_t SectionAddr = 0; #endif uint64_t SectionSize = 0; - uint64_t SectionAddrCheck = 0; // assert that all of the Sections are at the same location #if defined(_OS_WINDOWS_) + uint64_t SectionAddrCheck = 0; // assert that all of the Sections are at the same location #if defined(_CPU_X86_64_) uint8_t *UnwindData = NULL; uint8_t *catchjmp = NULL; @@ -966,7 +966,7 @@ class RTDyldMemoryManagerOSX : public SectionMemoryManager RTDyldMemoryManagerOSX() {}; ~RTDyldMemoryManagerOSX() override {}; void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) override; - void deregisterEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size); + void deregisterEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) override; }; extern "C" void __register_frame(void*); From b936f75c58488217c15270b4f3783ffc64728b1f Mon Sep 17 00:00:00 2001 From: ScottPJones Date: Mon, 28 Sep 2015 20:36:27 -0400 Subject: [PATCH 0269/1938] Fix #13344 compiling issues in disasm.cpp --- src/disasm.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/disasm.cpp b/src/disasm.cpp index bfc3d41ff7bf4..aea9c565c4f90 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -313,7 +313,11 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, size_t slide, #else MCContext Ctx(*MAI, *MRI, MOFI.get(), &SrcMgr); #endif +#ifdef LLVM38 + MOFI->InitMCObjectFileInfo(TheTriple, Reloc::Default, CodeModel::Default, Ctx); +#else MOFI->InitMCObjectFileInfo(TripleName, Reloc::Default, CodeModel::Default, Ctx); +#endif // Set up Subtarget and Disassembler #ifdef LLVM35 From f8d05e0854ae29c60cb02ec1a439876f5733de7a Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Mon, 28 Sep 2015 18:52:45 -0700 Subject: [PATCH 0270/1938] Apply changes from 84204a8b3bd1e80217e600453760fa1f822257e3 in doc sources and adjust rst rendering so genstdlib is consistent --- base/docs/basedocs.jl | 12 ++-- base/docs/helpdb.jl | 116 ++++++++++++++++++------------------ base/linalg/arnoldi.jl | 4 +- base/markdown/render/rst.jl | 2 +- doc/stdlib/linalg.rst | 2 + doc/stdlib/strings.rst | 4 +- 6 files changed, 71 insertions(+), 69 deletions(-) diff --git a/base/docs/basedocs.jl b/base/docs/basedocs.jl index 61bb24616e753..bdf5972339a11 100644 --- a/base/docs/basedocs.jl +++ b/base/docs/basedocs.jl @@ -580,12 +580,12 @@ Construct a regex, such as `r"^[a-z]*$"`. The regex also accepts one or more flags, listed after the ending quote, to change its behaviour: - • `i` enables case-insensitive matching - • `m` treats the `^` and `$` tokens as matching the start and - and end of individual lines, as opposed to the whole string. - • `s` allows the `.` modifier to match newlines. - • `x` enables "comment mode": whitespace is enabled except when - escaped with `\`, and `#` is treated as starting a comment. +• `i` enables case-insensitive matching +• `m` treats the `^` and `$` tokens as matching the start and + end of individual lines, as opposed to the whole string. +• `s` allows the `.` modifier to match newlines. +• `x` enables "comment mode": whitespace is enabled except when + escaped with `\`, and `#` is treated as starting a comment. For example, this regex has all three flags enabled: diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 23dbcac1673c6..e3806b5693b9a 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -4079,39 +4079,39 @@ doc""" Compute the LU factorization of ``A``. The return type of ``F`` depends on the type of ``A``. In most cases, if ``A`` is a subtype ``S`` of AbstractMatrix with an element type ``T`` supporting ``+``, ``-``, ``*`` and ``/`` the return type is ``LU{T,S{T}}``. If pivoting is chosen (default) the element type should also support ``abs`` and ``<``. When ``A`` is sparse and have element of type ``Float32``, ``Float64``, ``Complex{Float32}``, or ``Complex{Float64}`` the return type is ``UmfpackLU``. Some examples are shown in the table below. - ======================= ========================= ======================================== - Type of input ``A`` Type of output ``F`` Relationship between ``F`` and ``A`` - ----------------------- ------------------------- ---------------------------------------- - :func:`Matrix` ``LU`` ``F[:L]*F[:U] == A[F[:p], :]`` - :func:`Tridiagonal` ``LU{T,Tridiagonal{T}}`` ``F[:L]*F[:U] == A[F[:p], :]`` - :func:`SparseMatrixCSC` ``UmfpackLU`` ``F[:L]*F[:U] == (F[:Rs] .* A)[F[:p], F[:q]]`` - ======================= ========================= ======================================== +======================= ========================= ======================================== +Type of input ``A`` Type of output ``F`` Relationship between ``F`` and ``A`` +----------------------- ------------------------- ---------------------------------------- +:func:`Matrix` ``LU`` ``F[:L]*F[:U] == A[F[:p], :]`` +:func:`Tridiagonal` ``LU{T,Tridiagonal{T}}`` ``F[:L]*F[:U] == A[F[:p], :]`` +:func:`SparseMatrixCSC` ``UmfpackLU`` ``F[:L]*F[:U] == (F[:Rs] .* A)[F[:p], F[:q]]`` +======================= ========================= ======================================== The individual components of the factorization ``F`` can be accessed by indexing: - =========== ======================================= ====== ======================== ============= - Component Description ``LU`` ``LU{T,Tridiagonal{T}}`` ``UmfpackLU`` - ----------- --------------------------------------- ------ ------------------------ ------------- - ``F[:L]`` ``L`` (lower triangular) part of ``LU`` ✓ ✓ ✓ - ``F[:U]`` ``U`` (upper triangular) part of ``LU`` ✓ ✓ ✓ - ``F[:p]`` (right) permutation ``Vector`` ✓ ✓ ✓ - ``F[:P]`` (right) permutation ``Matrix`` ✓ ✓ - ``F[:q]`` left permutation ``Vector`` ✓ - ``F[:Rs]`` ``Vector`` of scaling factors ✓ - ``F[:(:)]`` ``(L,U,p,q,Rs)`` components ✓ - =========== ======================================= ====== ======================== ============= - - ================== ====== ======================== ============= - Supported function ``LU`` ``LU{T,Tridiagonal{T}}`` ``UmfpackLU`` - ------------------ ------ ------------------------ ------------- - ``/`` ✓ - ``\`` ✓ ✓ ✓ - ``cond`` ✓ ✓ - ``det`` ✓ ✓ ✓ - ``logdet`` ✓ ✓ - ``logabsdet`` ✓ ✓ - ``size`` ✓ ✓ - ================== ====== ======================== ============= +=========== ======================================= ====== ======================== ============= +Component Description ``LU`` ``LU{T,Tridiagonal{T}}`` ``UmfpackLU`` +----------- --------------------------------------- ------ ------------------------ ------------- +``F[:L]`` ``L`` (lower triangular) part of ``LU`` ✓ ✓ ✓ +``F[:U]`` ``U`` (upper triangular) part of ``LU`` ✓ ✓ ✓ +``F[:p]`` (right) permutation ``Vector`` ✓ ✓ ✓ +``F[:P]`` (right) permutation ``Matrix`` ✓ ✓ +``F[:q]`` left permutation ``Vector`` ✓ +``F[:Rs]`` ``Vector`` of scaling factors ✓ +``F[:(:)]`` ``(L,U,p,q,Rs)`` components ✓ +=========== ======================================= ====== ======================== ============= + +================== ====== ======================== ============= +Supported function ``LU`` ``LU{T,Tridiagonal{T}}`` ``UmfpackLU`` +------------------ ------ ------------------------ ------------- + ``/`` ✓ + ``\`` ✓ ✓ ✓ + ``cond`` ✓ ✓ + ``det`` ✓ ✓ ✓ + ``logdet`` ✓ ✓ + ``logabsdet`` ✓ ✓ + ``size`` ✓ ✓ +================== ====== ======================== ============= ``` """ lufact @@ -5658,12 +5658,12 @@ doc""" Implemented by cluster managers. It is called on the master process, during a worker's lifetime, with appropriate ``op`` values: - - with ``:register``/``:deregister`` when a worker is added / removed - from the Julia worker pool. - - with ``:interrupt`` when ``interrupt(workers)`` is called. The - :class:`ClusterManager` should signal the appropriate worker with an - interrupt signal. - - with ``:finalize`` for cleanup purposes. +- with ``:register``/``:deregister`` when a worker is added / removed + from the Julia worker pool. +- with ``:interrupt`` when ``interrupt(workers)`` is called. The + :class:`ClusterManager` should signal the appropriate worker with an + interrupt signal. +- with ``:finalize`` for cleanup purposes. ``` """ manage @@ -9134,9 +9134,9 @@ Compute the Pearson covariance between the vector(s) in `v1` and `v2`. Here, `v1 This function accepts three keyword arguments: -- `vardim`: the dimension of variables. When `vardim = 1`, variables are considered in columns while observations in rows; when `vardim = 2`, variables are in rows while observations in columns. By default, it is set to `1`. -- `corrected`: whether to apply Bessel's correction (divide by `n-1` instead of `n`). By default, it is set to `true`. -- `mean`: allow users to supply mean values that are known. By default, it is set to `nothing`, which indicates that the mean(s) are unknown, and the function will compute the mean. Users can use `mean=0` to indicate that the input data are centered, and hence there's no need to subtract the mean. +- `vardim`: the dimension of variables. When `vardim = 1`, variables are considered in columns while observations in rows; when `vardim = 2`, variables are in rows while observations in columns. By default, it is set to `1`. +- `corrected`: whether to apply Bessel's correction (divide by `n-1` instead of `n`). By default, it is set to `true`. +- `mean`: allow users to supply mean values that are known. By default, it is set to `nothing`, which indicates that the mean(s) are unknown, and the function will compute the mean. Users can use `mean=0` to indicate that the input data are centered, and hence there's no need to subtract the mean. The size of the result depends on the size of `v1` and `v2`. When both `v1` and `v2` are vectors, it returns the covariance between them as a scalar. When either one is a matrix, it returns a covariance matrix of size `(n1, n2)`, where `n1` and `n2` are the numbers of slices in `v1` and `v2`, which depend on the setting of `vardim`. @@ -9978,26 +9978,26 @@ doc""" Computes the QR factorization of ``A``. The return type of ``F`` depends on the element type of ``A`` and whether pivoting is specified (with ``pivot==Val{true}``). - ================ ================= ============== ===================================== - Return type ``eltype(A)`` ``pivot`` Relationship between ``F`` and ``A`` - ---------------- ----------------- -------------- ------------------------------------- - ``QR`` not ``BlasFloat`` either ``A==F[:Q]*F[:R]`` - ``QRCompactWY`` ``BlasFloat`` ``Val{false}`` ``A==F[:Q]*F[:R]`` - ``QRPivoted`` ``BlasFloat`` ``Val{true}`` ``A[:,F[:p]]==F[:Q]*F[:R]`` - ================ ================= ============== ===================================== +================ ================= ============== ===================================== +Return type ``eltype(A)`` ``pivot`` Relationship between ``F`` and ``A`` +---------------- ----------------- -------------- ------------------------------------- +``QR`` not ``BlasFloat`` either ``A==F[:Q]*F[:R]`` +``QRCompactWY`` ``BlasFloat`` ``Val{false}`` ``A==F[:Q]*F[:R]`` +``QRPivoted`` ``BlasFloat`` ``Val{true}`` ``A[:,F[:p]]==F[:Q]*F[:R]`` +================ ================= ============== ===================================== ``BlasFloat`` refers to any of: ``Float32``, ``Float64``, ``Complex64`` or ``Complex128``. The individual components of the factorization ``F`` can be accessed by indexing: - =========== ============================================= ================== ===================== ================== - Component Description ``QR`` ``QRCompactWY`` ``QRPivoted`` - ----------- --------------------------------------------- ------------------ --------------------- ------------------ - ``F[:Q]`` ``Q`` (orthogonal/unitary) part of ``QR`` ✓ (``QRPackedQ``) ✓ (``QRCompactWYQ``) ✓ (``QRPackedQ``) - ``F[:R]`` ``R`` (upper right triangular) part of ``QR`` ✓ ✓ ✓ - ``F[:p]`` pivot ``Vector`` ✓ - ``F[:P]`` (pivot) permutation ``Matrix`` ✓ - =========== ============================================= ================== ===================== ================== +=========== ============================================= ================== ===================== ================== +Component Description ``QR`` ``QRCompactWY`` ``QRPivoted`` +----------- --------------------------------------------- ------------------ --------------------- ------------------ +``F[:Q]`` ``Q`` (orthogonal/unitary) part of ``QR`` ✓ (``QRPackedQ``) ✓ (``QRCompactWYQ``) ✓ (``QRPackedQ``) +``F[:R]`` ``R`` (upper right triangular) part of ``QR`` ✓ ✓ ✓ +``F[:p]`` pivot ``Vector`` ✓ +``F[:P]`` (pivot) permutation ``Matrix`` ✓ +=========== ============================================= ================== ===================== ================== The following functions are available for the ``QR`` objects: ``size``, ``\``. When ``A`` is rectangular, ``\`` will return a least squares solution and if the solution is not unique, the one with smallest norm is returned. @@ -10009,17 +10009,17 @@ Multiplication with respect to either thin or full ``Q`` is allowed, i.e. both ` The data contained in ``QR`` or ``QRPivoted`` can be used to construct the ``QRPackedQ`` type, which is a compact representation of the rotation matrix: - .. math:: + .. math:: - Q = \prod_{i=1}^{\min(m,n)} (I - \tau_i v_i v_i^T) + Q = \prod_{i=1}^{\min(m,n)} (I - \tau_i v_i v_i^T) where :math:`\tau_i` is the scale factor and :math:`v_i` is the projection vector associated with the :math:`i^{th}` Householder elementary reflector. The data contained in ``QRCompactWY`` can be used to construct the ``QRCompactWYQ`` type, which is a compact representation of the rotation matrix - .. math:: + .. math:: - Q = I + Y T Y^T + Q = I + Y T Y^T where ``Y`` is :math:`m \times r` lower trapezoidal and ``T`` is :math:`r \times r` upper triangular. The *compact WY* representation [Schreiber1989]_ is not to be confused with the older, *WY* representation [Bischof1987]_. (The LAPACK documentation uses ``V`` in lieu of ``Y``.) diff --git a/base/linalg/arnoldi.jl b/base/linalg/arnoldi.jl index 5cc69c167672c..95080b2878944 100644 --- a/base/linalg/arnoldi.jl +++ b/base/linalg/arnoldi.jl @@ -14,7 +14,7 @@ The following keyword arguments are supported: * ``nev``: Number of eigenvalues * ``ncv``: Number of Krylov vectors used in the computation; should satisfy ``nev+1 <= ncv <= n`` for real symmetric problems and ``nev+2 <= ncv <= n`` for other problems, where ``n`` is the size of the input matrix ``A``. The default is ``ncv = max(20,2*nev+1)``. - Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. + Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. * ``which``: type of eigenvalues to compute. See the note below. ========= ====================================================================================================================== @@ -61,7 +61,7 @@ The following keyword arguments are supported: * ``nev``: Number of eigenvalues * ``ncv``: Number of Krylov vectors used in the computation; should satisfy ``nev+1 <= ncv <= n`` for real symmetric problems and ``nev+2 <= ncv <= n`` for other problems, where ``n`` is the size of the input matrices ``A`` and ``B``. The default is ``ncv = max(20,2*nev+1)``. - Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. + Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. * ``which``: type of eigenvalues to compute. See the note below. ========= ====================================================================================================================== diff --git a/base/markdown/render/rst.jl b/base/markdown/render/rst.jl index 1651d2778a066..1ee39928db746 100644 --- a/base/markdown/render/rst.jl +++ b/base/markdown/render/rst.jl @@ -38,7 +38,7 @@ end function rst(io::IO, list::List) for (i, item) in enumerate(list.items) - print(io, list.ordered ? "$i. " : " * ") + print(io, list.ordered ? "$i. " : "* ") rstinline(io, item) println(io) end diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 241db44fbfbab..17f6d99b9e2c8 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -928,6 +928,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f The following keyword arguments are supported: * ``nev``: Number of eigenvalues * ``ncv``: Number of Krylov vectors used in the computation; should satisfy ``nev+1 <= ncv <= n`` for real symmetric problems and ``nev+2 <= ncv <= n`` for other problems, where ``n`` is the size of the input matrix ``A``. The default is ``ncv = max(20,2*nev+1)``. + Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. * ``which``: type of eigenvalues to compute. See the note below. @@ -970,6 +971,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f The following keyword arguments are supported: * ``nev``: Number of eigenvalues * ``ncv``: Number of Krylov vectors used in the computation; should satisfy ``nev+1 <= ncv <= n`` for real symmetric problems and ``nev+2 <= ncv <= n`` for other problems, where ``n`` is the size of the input matrices ``A`` and ``B``. The default is ``ncv = max(20,2*nev+1)``. + Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. * ``which``: type of eigenvalues to compute. See the note below. diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index 05f8d4adbf488..50a0a00a2f646 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -105,9 +105,9 @@ Construct a regex, such as ``r"^[a-z]*$"``\ . The regex also accepts one or more flags, listed after the ending quote, to change its behaviour: * ``i`` enables case-insensitive matching - * ``m`` treats the ``^`` and ``$`` tokens as matching the start and and end of individual lines, as opposed to the whole string. + * ``m`` treats the ``^`` and ``$`` tokens as matching the start and end of individual lines, as opposed to the whole string. * ``s`` allows the ``.`` modifier to match newlines. - * ``x`` enables "comment mode": whitespace is enabled except when escaped with ``\``\ , and ``#`` is treated as starting a comment. + * ``x`` enables "comment mode": whitespace is enabled except when escaped with ``\``\ , and ``#`` is treated as starting a comment. For example, this regex has all three flags enabled: From 17328bee64cec96b08e2c87070d6272c89d881dc Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Mon, 28 Sep 2015 19:18:08 -0700 Subject: [PATCH 0271/1938] Adjust test/markdown.jl for list spacing whoops, shame on me --- test/markdown.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/markdown.jl b/test/markdown.jl index 9129962c1b7c6..25c67524ca788 100644 --- a/test/markdown.jl +++ b/test/markdown.jl @@ -199,8 +199,8 @@ let out = Some **bolded** - * list1 - * list2 + * list1 + * list2 """ @test sprint(io -> writemime(io, "text/rst", book)) == out end From 449c6da9d38b43e7810cd09986f4628d902ca1fb Mon Sep 17 00:00:00 2001 From: kshyatt Date: Mon, 28 Sep 2015 20:51:30 -0700 Subject: [PATCH 0272/1938] Early-exit and pass-through tests --- test/linalg/lapack.jl | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/linalg/lapack.jl b/test/linalg/lapack.jl index bf0e537065579..6dfdb5b4be240 100644 --- a/test/linalg/lapack.jl +++ b/test/linalg/lapack.jl @@ -241,8 +241,7 @@ end #orglq and friends errors for elty in (Float32, Float64, Complex64, Complex128) A = rand(elty,10,10) - tau = zeros(elty,10) - A,tau = LAPACK.gelqf!(A,tau) + A,tau = LAPACK.gelqf!(A) @test_throws DimensionMismatch LAPACK.orglq!(A,tau,11) @test_throws DimensionMismatch LAPACK.ormlq!('R','N',A,tau,rand(elty,11,11)) @test_throws DimensionMismatch LAPACK.ormlq!('L','N',A,tau,rand(elty,11,11)) @@ -254,8 +253,7 @@ for elty in (Float32, Float64, Complex64, Complex128) @test LAPACK.ormlq!('R','N',A,tau,eye(elty,10)) ≈ C A = rand(elty,10,10) - tau = zeros(elty,10) - A,tau = LAPACK.geqrf!(A,tau) + A,tau = LAPACK.geqrf!(A) @test_throws DimensionMismatch LAPACK.orgqr!(A,tau,11) B = copy(A) @test LAPACK.orgqr!(B,tau) ≈ LAPACK.ormqr!('R','N',A,tau,eye(elty,10)) @@ -265,8 +263,7 @@ for elty in (Float32, Float64, Complex64, Complex128) @test_throws DimensionMismatch LAPACK.ormqr!('L','N',A,zeros(elty,11),rand(elty,10,10)) A = rand(elty,10,10) - tau = zeros(elty,10) - A,tau = LAPACK.geqlf!(A,tau) + A,tau = LAPACK.geqlf!(A) @test_throws DimensionMismatch LAPACK.orgql!(A,tau,11) B = copy(A) @test LAPACK.orgql!(B,tau) ≈ LAPACK.ormql!('R','N',A,tau,eye(elty,10)) @@ -276,8 +273,7 @@ for elty in (Float32, Float64, Complex64, Complex128) @test_throws DimensionMismatch LAPACK.ormql!('L','N',A,zeros(elty,11),rand(elty,10,10)) A = rand(elty,10,10) - tau = zeros(elty,10) - A,tau = LAPACK.gerqf!(A,tau) + A,tau = LAPACK.gerqf!(A) @test_throws DimensionMismatch LAPACK.orgrq!(A,tau,11) B = copy(A) @test LAPACK.orgrq!(B,tau) ≈ LAPACK.ormrq!('R','N',A,tau,eye(elty,10)) @@ -436,6 +432,8 @@ for elty in (Float32, Float64, Complex64, Complex128) @test_approx_eq A\B C @test_throws DimensionMismatch LAPACK.posv!('U',D,ones(elty,12,12)) @test_throws DimensionMismatch LAPACK.potrs!('U',D,ones(elty,12,12)) + + @test LAPACK.potrs!('U',zeros(elty,0,0),ones(elty,0)) == ones(elty,0) end #gesvx From f63f4ce80af51cb72f23929cf0c425d264f0e498 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Tue, 29 Sep 2015 04:55:59 -0700 Subject: [PATCH 0273/1938] fix #13360 by removing localtime_r from julia.h --- src/julia.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/julia.h b/src/julia.h index 0b8e000a40a9e..ec05783766d70 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1498,10 +1498,6 @@ DLLEXPORT int jl_tcp_bind(uv_tcp_t *handle, uint16_t port, uint32_t host, unsign DLLEXPORT int jl_sizeof_ios_t(void); -#ifdef _OS_WINDOWS_ -DLLEXPORT struct tm* localtime_r(const time_t *t, struct tm *tm); -#endif - DLLEXPORT jl_array_t *jl_takebuf_array(ios_t *s); DLLEXPORT jl_value_t *jl_takebuf_string(ios_t *s); DLLEXPORT void *jl_takebuf_raw(ios_t *s); From 1023b8552548c3884644cff7f47d810eae00474b Mon Sep 17 00:00:00 2001 From: Michael Hatherly Date: Tue, 29 Sep 2015 14:14:33 +0200 Subject: [PATCH 0274/1938] Update documenting macro-generated code docs [skip ci] --- doc/manual/documentation.rst | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/doc/manual/documentation.rst b/doc/manual/documentation.rst index 08d9b4ecea7be..864d2479825a4 100644 --- a/doc/manual/documentation.rst +++ b/doc/manual/documentation.rst @@ -351,10 +351,35 @@ Macro-generated code "..." @m expression -Adds docstring ``"..."`` to expression generated by expanding ``@m expression``. This syntax -is only valid if expanding ``@m`` results in a single expression that can be documented. -This allows for functions decorated with ``@inline``, ``@noinline``, and ``@generated`` to -be documented in the same way as other functions. +Adds docstring ``"..."`` to expression generated by expanding ``@m expression``. This allows +for expressions decorated with ``@inline``, ``@noinline``, ``@generated``, or any other +macro to be documented in the same way as undecorated expressions. + +Macro authors should take note that only macros that generate a `single expression` will +automatically support docstrings. If a macro returns a block containing multiple +subexpressions then the subexpression that should be documented must be marked using the +:func:`@__doc__` macro. + +The ``@enum`` macro makes use of ``@__doc__`` to allow for documenting ``Enum``\ s. +Examining it's definition should serve as an example of how to use ``@__doc__`` correctly. + +.. function:: @__doc__(ex) + + .. Docstring generated from Julia source + + Low-level macro used to mark expressions returned by a macro that should be documented. If more than one expression is marked then the same docstring is applied to each expression. + + .. code-block:: julia + + macro example(f) + quote + $(f)() = 0 + @__doc__ $(f)(x) = 1 + $(f)(x, y) = 2 + end |> esc + end + + ``@__doc__`` has no effect when a macro that uses it is not documented. Markdown Syntax Notes --------------------- From 4fb973b3a47252f475f1760fc290f5260f470d39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1ll=20Haraldsson?= Date: Tue, 29 Sep 2015 14:18:55 +0000 Subject: [PATCH 0275/1938] Raring link no longer works.. Older point to Julia 0.2. Thought about linking to current stable version, but I guess link might disappear in January 2016.. This one will be valid longer, but Wily isn't strictly released until October.. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c9f87103e7928..3d0cefa896c52 100644 --- a/README.md +++ b/README.md @@ -399,7 +399,7 @@ The following distributions include julia, but the versions may be out of date d * Git package for openSUSE: [OBS page](https://build.opensuse.org/package/show/science/julia-unstable), [1 Click Install](http://software.opensuse.org/download.html?project=science&package=julia-unstable) * [NixOS](https://github.com/NixOS/nixpkgs/blob/master/pkgs/development/compilers/julia) * Ubuntu - * [Ubuntu 13.04 (Raring Ringtail)](http://packages.ubuntu.com/raring/julia) + * [Ubuntu 15.10 (Wily Werewolf)](http://packages.ubuntu.com/wily/julia) * [Nightly builds PPA](https://launchpad.net/~staticfloat/+archive/julianightlies) (depends on the [julia-deps PPA](https://launchpad.net/~staticfloat/+archive/julia-deps/)) * [OS X Homebrew Tap](https://github.com/staticfloat/homebrew-julia/) From 865c87c9472d6e77054c8896ef8f958264f9d770 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Tue, 5 May 2015 16:30:39 -0400 Subject: [PATCH 0276/1938] changing `installed` based to support `libgit2` --- base/pkg.jl | 2 +- base/pkg/entry.jl | 6 +- base/pkg/installed.jl | 232 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 236 insertions(+), 4 deletions(-) create mode 100644 base/pkg/installed.jl diff --git a/base/pkg.jl b/base/pkg.jl index b7b48ad40e892..33ee3e97c5083 100644 --- a/base/pkg.jl +++ b/base/pkg.jl @@ -10,7 +10,7 @@ export dir, init, rm, add, available, installed, status, clone, checkout, const DEFAULT_META = "git://github.com/JuliaLang/METADATA.jl" const META_BRANCH = "metadata-v2" -for file in split("git dir github types reqs cache read query resolve write generate entry") +for file in split("git dir github types reqs cache read query resolve write generate installed entry") include("pkg/$file.jl") end const cd = Dir.cd diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 869fec4499e4f..cc3ef47832e40 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -3,7 +3,7 @@ module Entry import Base: thispatch, nextpatch, nextminor, nextmajor, check_new_version -import ..Git, ..Reqs, ..Read, ..Query, ..Resolve, ..Cache, ..Write, ..GitHub, ..Dir +import ..Git, ..Reqs, ..Read, ..Query, ..Resolve, ..Cache, ..Write, ..GitHub, ..Dir, ..Read2 using ..Types macro recover(ex) @@ -98,7 +98,7 @@ end function installed() pkgs = Dict{ASCIIString,VersionNumber}() - for (pkg,(ver,fix)) in Read.installed() + for (pkg,(ver,fix)) in Read2.installed() pkgs[pkg] = ver end return pkgs @@ -114,7 +114,7 @@ end function status(io::IO; pkgname::AbstractString = "") showpkg(pkg) = (pkgname == "") ? (true) : (pkg == pkgname) reqs = Reqs.parse("REQUIRE") - instd = Read.installed() + instd = Read2.installed() required = sort!(collect(keys(reqs))) if !isempty(required) showpkg("") && println(io, "$(length(required)) required packages:") diff --git a/base/pkg/installed.jl b/base/pkg/installed.jl new file mode 100644 index 0000000000000..9ef4a4cb7308f --- /dev/null +++ b/base/pkg/installed.jl @@ -0,0 +1,232 @@ +module Read2 + +import ..Git, ..Cache, ..Reqs +import ..Read: isinstalled, available, Available +using ..Types + +type Ref + ptr::Ptr{Void} + function Ref(ptr::Ptr{Void}) + r = new(ptr) + finalizer(r, r -> ccall((:git_reference_free, :libgit2), Void, (Ptr{Void},), r.ptr)) + return r + end +end + +type Obj + ptr::Ptr{Void} + function Obj(ptr::Ptr{Void}) + r = new(ptr) + finalizer(r, r -> ccall((:git_object_free, :libgit2), Void, (Ptr{Void},), r.ptr)) + return r + end +end + +function repo(path::String) + repo_ptr = Ptr{Void}[0] + err = ccall((:git_repository_open, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Uint8}), repo_ptr, path) + if err != 0 + if repo_ptr[1] != C_NULL + ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), repo_ptr[1]) + end + return C_NULL + end + return repo_ptr[1] +end + +function close(repo_ptr::Ptr{Void}) + ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), repo_ptr) +end + +function repo_head(r::Ptr{Void}) + head_ptr = Ptr{Void}[0] + err = ccall((:git_repository_head, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}), head_ptr, r) + (err != 0) && return nothing + return Ref(head_ptr[1]) +end + +function ref_id(ref::Ref) + ref == nothing && return "" + oid_ptr = ccall((:git_reference_target, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) + oid_ptr == C_NULL && return "" + return bytes2hex(pointer_to_array(oid_ptr, 20)) +end + +function need_update(repo::Ptr{Void}) + ccall((:git_repository_is_bare, :libgit2), Cint, (Ptr{Void},), repo) != 1 && "git update-index -q --really-refresh" +end + +function iscommit(id::String, repo::Ptr{Void}) + need_update(repo) + + oid = hex2bytes(id) + cmt_ptr = Ptr{Void}[0] + err = ccall((:git_commit_lookup, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Uint8}), cmt_ptr, repo, oid) + if err != 0 + return false + else + ccall((:git_commit_free, :libgit2), Void, (Ptr{Void},), cmt_ptr[1]) + return true + end +end + +function obj_id(ref::Obj) + ref == nothing && return "" + oid_ptr = ccall((:git_object_id, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) + oid_ptr == C_NULL && return "" + return bytes2hex(pointer_to_array(oid_ptr, 20)) +end + +function isdirty(repo::Ptr{Void}) + obj_ptr = Ptr{Void}[0] + obj = "HEAD^{tree}" + err = ccall((:git_revparse_single, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Uint8}), obj_ptr, repo, obj) + err != 0 && return true + + tree_oid = hex2bytes(obj_id(Obj(obj_ptr[1]))) + tree_ptr = Ptr{Void}[0] + err = ccall((:git_tree_lookup, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Uint8}), tree_ptr, repo, tree_oid) + err != 0 && return true + + diff_ptr = Ptr{Void}[0] + err = ccall((:git_diff_tree_to_workdir_with_index, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, Ptr{Void}), + diff_ptr, repo, tree_ptr[1], C_NULL, C_NULL) + err != 0 && return true + + c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff_ptr[1]) + c > 0 && return true + + return false +end + +function isattached(repo::Ptr{Void}) + ccall((:git_repository_head_detached, :libgit2), Cint, (Ptr{Void},), repo) != 1 +end + +function merge_base(one::String, two::String, repo::Ptr{Void}) + oid1 = hex2bytes(one) + oid2 = hex2bytes(two) + moid = zeros(UInt8 ,20) + err = ccall((:git_merge_base, :libgit2), Cint, + (Ptr{UInt8}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}), moid, repo, oid1, oid2) + if err != 0 + return "" + else + bytes2hex(moid) + end +end + +function is_ancestor_of(a::AbstractString, b::AbstractString, repo::Ptr{Void}) + #A = readchomp(`rev-parse $a`, dir=dir) + merge_base(a, b, prepo) == a +end + +function installed_version2(pkg::AbstractString, prepo::Ptr{Void}, avail::Dict=available(pkg)) + ispath(pkg,".git") || return typemin(VersionNumber) + + # get package repo head hash + head = ref_id(repo_head(prepo)) + + vers = collect(keys(filter((ver,info)->info.sha1==head, avail))) + !isempty(vers) && return maximum(vers) + cache = Cache.path(pkg) + crepo = repo(cache) # open Cache repo + cache_has_head = isdir(cache) && iscommit(head, crepo) + ancestors = VersionNumber[] + descendants = VersionNumber[] + for (ver,info) in avail + sha1 = info.sha1 + base = if cache_has_head && iscommit(sha1, crepo) + merge_base(head, sha1, crepo) + elseif iscommit(sha1, prepo) + merge_base(head, sha1, prepo) + else + Base.warn_once("unknown $pkg commit $(sha1[1:8]), metadata may be ahead of package cache") + continue + end + base == sha1 && push!(ancestors,ver) + base == head && push!(descendants,ver) + end + close(crepo) # close Cache repo + both = sort!(intersect(ancestors,descendants)) + isempty(both) || warn("$pkg: some versions are both ancestors and descendants of head: $both") + if !isempty(descendants) + v = minimum(descendants) + return VersionNumber(v.major, v.minor, v.patch, ("",), ()) + elseif !isempty(ancestors) + v = maximum(ancestors) + return VersionNumber(v.major, v.minor, v.patch, (), ("",)) + else + return typemin(VersionNumber) + end +end + +function isfixed2(pkg::AbstractString, prepo::Ptr{Void}, avail::Dict=available(pkg)) + isinstalled(pkg) || error("$pkg is not an installed package.") + isfile("METADATA", pkg, "url") || return true + ispath(pkg, ".git") || return true + + isdirty(prepo) && return true + isattached(prepo) && return true + #!Git.success(`cat-file -e HEAD:REQUIRE`, dir=pkg) && isfile(pkg,"REQUIRE") && return true + + head = ref_id(repo_head(prepo)) + for (ver,info) in avail + head == info.sha1 && return false + end + + cache = Cache.path(pkg) + crepo = repo(cache) # open Cache repo + cache_has_head = isdir(cache) && iscommit(head, crepo) + res = true + for (ver,info) in avail + if cache_has_head && iscommit(info.sha1, crepo) + if is_ancestor_of(head, info.sha1, crepo) + res = false + break + end + elseif iscommit(info.sha1, prepo) + if is_ancestor_of(head, info.sha1, prepo) + res = false + break + end + else + Base.warn_once("unknown $pkg commit $(info.sha1[1:8]), metadata may be ahead of package cache") + end + end + close(crepo) + return res +end + +# ;cd /home/art/.julia/v0.3 +# avails=available() +# pkg = "Blosc" +# avail = get(avails,pkg,Dict{VersionNumber,Available}()) +# prepo = repo(pkg) +# installed_version2(pkg, prepo, avail) +# close(prepo) + +# prepo = repo(pkg) +# phead_ref = repo_head(prepo) +# close(prepo) + +function installed(avail::Dict=available()) + pkgs = Dict{ByteString,Tuple{VersionNumber,Bool}}() + for pkg in readdir() + isinstalled(pkg) || continue + ap = get(avail,pkg,Dict{VersionNumber,Available}()) + prepo = repo(pkg) + ver = installed_version2(pkg, prepo, ap) + fixed = isfixed2(pkg, prepo, ap) + close(prepo) + pkgs[pkg] = (ver, fixed) + end + return pkgs +end +end \ No newline at end of file From 851c0537305db46f235df5ef042f6fc05cabc824 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Tue, 5 May 2015 16:53:08 -0400 Subject: [PATCH 0277/1938] merged installed & dependents into `Read`, moved `libgit2` calls into libgit.jl --- base/pkg.jl | 2 +- base/pkg/entry.jl | 13 +++- base/pkg/{installed.jl => libgit.jl} | 108 +-------------------------- base/pkg/read.jl | 61 ++++++++++----- 4 files changed, 52 insertions(+), 132 deletions(-) rename base/pkg/{installed.jl => libgit.jl} (51%) diff --git a/base/pkg.jl b/base/pkg.jl index 33ee3e97c5083..156a9899438cd 100644 --- a/base/pkg.jl +++ b/base/pkg.jl @@ -10,7 +10,7 @@ export dir, init, rm, add, available, installed, status, clone, checkout, const DEFAULT_META = "git://github.com/JuliaLang/METADATA.jl" const META_BRANCH = "metadata-v2" -for file in split("git dir github types reqs cache read query resolve write generate installed entry") +for file in split("git libgit dir github types reqs cache read query resolve write generate entry") include("pkg/$file.jl") end const cd = Dir.cd diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index cc3ef47832e40..77eebbba8060a 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -3,7 +3,7 @@ module Entry import Base: thispatch, nextpatch, nextminor, nextmajor, check_new_version -import ..Git, ..Reqs, ..Read, ..Query, ..Resolve, ..Cache, ..Write, ..GitHub, ..Dir, ..Read2 +import ..LibGit, ..Git, ..Reqs, ..Read, ..Query, ..Resolve, ..Cache, ..Write, ..GitHub, ..Dir using ..Types macro recover(ex) @@ -98,7 +98,7 @@ end function installed() pkgs = Dict{ASCIIString,VersionNumber}() - for (pkg,(ver,fix)) in Read2.installed() + for (pkg,(ver,fix)) in Read.installed() pkgs[pkg] = ver end return pkgs @@ -106,7 +106,12 @@ end function installed(pkg::AbstractString) avail = Read.available(pkg) - Read.isinstalled(pkg) && return Read.installed_version(pkg,avail) + if Read.isinstalled(pkg) + prepo = LibGit.repo(pkg) + res = Read.installed_version(pkg, prepo, avail) + LibGit.close(prepo) + return res + end isempty(avail) && error("$pkg is not a package (not registered or installed)") return nothing # registered but not installed end @@ -114,7 +119,7 @@ end function status(io::IO; pkgname::AbstractString = "") showpkg(pkg) = (pkgname == "") ? (true) : (pkg == pkgname) reqs = Reqs.parse("REQUIRE") - instd = Read2.installed() + instd = Read.installed() required = sort!(collect(keys(reqs))) if !isempty(required) showpkg("") && println(io, "$(length(required)) required packages:") diff --git a/base/pkg/installed.jl b/base/pkg/libgit.jl similarity index 51% rename from base/pkg/installed.jl rename to base/pkg/libgit.jl index 9ef4a4cb7308f..cabd558f96f70 100644 --- a/base/pkg/installed.jl +++ b/base/pkg/libgit.jl @@ -1,8 +1,4 @@ -module Read2 - -import ..Git, ..Cache, ..Reqs -import ..Read: isinstalled, available, Available -using ..Types +module LibGit type Ref ptr::Ptr{Void} @@ -127,106 +123,4 @@ function is_ancestor_of(a::AbstractString, b::AbstractString, repo::Ptr{Void}) merge_base(a, b, prepo) == a end -function installed_version2(pkg::AbstractString, prepo::Ptr{Void}, avail::Dict=available(pkg)) - ispath(pkg,".git") || return typemin(VersionNumber) - - # get package repo head hash - head = ref_id(repo_head(prepo)) - - vers = collect(keys(filter((ver,info)->info.sha1==head, avail))) - !isempty(vers) && return maximum(vers) - cache = Cache.path(pkg) - crepo = repo(cache) # open Cache repo - cache_has_head = isdir(cache) && iscommit(head, crepo) - ancestors = VersionNumber[] - descendants = VersionNumber[] - for (ver,info) in avail - sha1 = info.sha1 - base = if cache_has_head && iscommit(sha1, crepo) - merge_base(head, sha1, crepo) - elseif iscommit(sha1, prepo) - merge_base(head, sha1, prepo) - else - Base.warn_once("unknown $pkg commit $(sha1[1:8]), metadata may be ahead of package cache") - continue - end - base == sha1 && push!(ancestors,ver) - base == head && push!(descendants,ver) - end - close(crepo) # close Cache repo - both = sort!(intersect(ancestors,descendants)) - isempty(both) || warn("$pkg: some versions are both ancestors and descendants of head: $both") - if !isempty(descendants) - v = minimum(descendants) - return VersionNumber(v.major, v.minor, v.patch, ("",), ()) - elseif !isempty(ancestors) - v = maximum(ancestors) - return VersionNumber(v.major, v.minor, v.patch, (), ("",)) - else - return typemin(VersionNumber) - end -end - -function isfixed2(pkg::AbstractString, prepo::Ptr{Void}, avail::Dict=available(pkg)) - isinstalled(pkg) || error("$pkg is not an installed package.") - isfile("METADATA", pkg, "url") || return true - ispath(pkg, ".git") || return true - - isdirty(prepo) && return true - isattached(prepo) && return true - #!Git.success(`cat-file -e HEAD:REQUIRE`, dir=pkg) && isfile(pkg,"REQUIRE") && return true - - head = ref_id(repo_head(prepo)) - for (ver,info) in avail - head == info.sha1 && return false - end - - cache = Cache.path(pkg) - crepo = repo(cache) # open Cache repo - cache_has_head = isdir(cache) && iscommit(head, crepo) - res = true - for (ver,info) in avail - if cache_has_head && iscommit(info.sha1, crepo) - if is_ancestor_of(head, info.sha1, crepo) - res = false - break - end - elseif iscommit(info.sha1, prepo) - if is_ancestor_of(head, info.sha1, prepo) - res = false - break - end - else - Base.warn_once("unknown $pkg commit $(info.sha1[1:8]), metadata may be ahead of package cache") - end - end - close(crepo) - return res -end - -# ;cd /home/art/.julia/v0.3 -# avails=available() -# pkg = "Blosc" -# avail = get(avails,pkg,Dict{VersionNumber,Available}()) -# prepo = repo(pkg) -# installed_version2(pkg, prepo, avail) -# close(prepo) - -# prepo = repo(pkg) -# phead_ref = repo_head(prepo) -# close(prepo) - -function installed(avail::Dict=available()) - pkgs = Dict{ByteString,Tuple{VersionNumber,Bool}}() - for pkg in readdir() - isinstalled(pkg) || continue - ap = get(avail,pkg,Dict{VersionNumber,Available}()) - prepo = repo(pkg) - ver = installed_version2(pkg, prepo, ap) - fixed = isfixed2(pkg, prepo, ap) - close(prepo) - pkgs[pkg] = (ver, fixed) - end - return pkgs -end end \ No newline at end of file diff --git a/base/pkg/read.jl b/base/pkg/read.jl index 059c1fd1eaa3e..9889d8606e3ae 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -2,7 +2,7 @@ module Read -import ..Git, ..Cache, ..Reqs +import ..Git, ..Cache, ..Reqs, ..LibGit using ..Types readstrip(path...) = strip(readall(joinpath(path...))) @@ -55,46 +55,62 @@ end isinstalled(pkg::AbstractString) = pkg != "METADATA" && pkg != "REQUIRE" && pkg[1] != '.' && isdir(pkg) -function isfixed(pkg::AbstractString, avail::Dict=available(pkg)) +function isfixed(pkg::AbstractString, prepo::Ptr{Void}, avail::Dict=available(pkg)) isinstalled(pkg) || error("$pkg is not an installed package.") isfile("METADATA", pkg, "url") || return true ispath(pkg, ".git") || return true - Git.dirty(dir=pkg) && return true - Git.attached(dir=pkg) && return true - !Git.success(`cat-file -e HEAD:REQUIRE`, dir=pkg) && isfile(pkg,"REQUIRE") && return true - head = Git.head(dir=pkg) + + LibGit.isdirty(prepo) && return true + LibGit.isattached(prepo) && return true + #!Git.success(`cat-file -e HEAD:REQUIRE`, dir=pkg) && isfile(pkg,"REQUIRE") && return true + + head = LibGit.ref_id(LibGit.repo_head(prepo)) for (ver,info) in avail head == info.sha1 && return false end + cache = Cache.path(pkg) - cache_has_head = isdir(cache) && Git.iscommit(head, dir=cache) + crepo = LibGit.repo(cache) # open Cache repo + cache_has_head = isdir(cache) && LibGit.iscommit(head, crepo) + res = true for (ver,info) in avail - if cache_has_head && Git.iscommit(info.sha1, dir=cache) - Git.is_ancestor_of(head, info.sha1, dir=cache) && return false - elseif Git.iscommit(info.sha1, dir=pkg) - Git.is_ancestor_of(head, info.sha1, dir=pkg) && return false + if cache_has_head && LibGit.iscommit(info.sha1, crepo) + if LibGit.is_ancestor_of(head, info.sha1, crepo) + res = false + break + end + elseif LibGit.iscommit(info.sha1, prepo) + if LibGit.is_ancestor_of(head, info.sha1, prepo) + res = false + break + end else Base.warn_once("unknown $pkg commit $(info.sha1[1:8]), metadata may be ahead of package cache") end end - return true + close(crepo) + return res end -function installed_version(pkg::AbstractString, avail::Dict=available(pkg)) +function installed_version(pkg::AbstractString, prepo::Ptr{Void}, avail::Dict=available(pkg)) ispath(pkg,".git") || return typemin(VersionNumber) - head = Git.head(dir=pkg) + + # get package repo head hash + head = LibGit.ref_id(LibGit.repo_head(prepo)) + vers = collect(keys(filter((ver,info)->info.sha1==head, avail))) !isempty(vers) && return maximum(vers) cache = Cache.path(pkg) - cache_has_head = isdir(cache) && Git.iscommit(head, dir=cache) + crepo = LibGit.repo(cache) # open Cache repo + cache_has_head = isdir(cache) && LibGit.iscommit(head, crepo) ancestors = VersionNumber[] descendants = VersionNumber[] for (ver,info) in avail sha1 = info.sha1 - base = if cache_has_head && Git.iscommit(sha1, dir=cache) - Git.readchomp(`merge-base $head $sha1`, dir=cache) - elseif Git.iscommit(sha1, dir=pkg) - Git.readchomp(`merge-base $head $sha1`, dir=pkg) + base = if cache_has_head && LibGit.iscommit(sha1, crepo) + LibGit.merge_base(head, sha1, crepo) + elseif LibGit.iscommit(sha1, prepo) + LibGit.merge_base(head, sha1, prepo) else Base.warn_once("unknown $pkg commit $(sha1[1:8]), metadata may be ahead of package cache") continue @@ -102,6 +118,7 @@ function installed_version(pkg::AbstractString, avail::Dict=available(pkg)) base == sha1 && push!(ancestors,ver) base == head && push!(descendants,ver) end + LibGit.close(crepo) # close Cache repo both = sort!(intersect(ancestors,descendants)) isempty(both) || warn("$pkg: some versions are both ancestors and descendants of head: $both") if !isempty(descendants) @@ -140,7 +157,11 @@ function installed(avail::Dict=available()) for pkg in readdir() isinstalled(pkg) || continue ap = get(avail,pkg,Dict{VersionNumber,Available}()) - pkgs[pkg] = (installed_version(pkg,ap),isfixed(pkg,ap)) + prepo = LibGit.repo(pkg) + ver = installed_version(pkg, prepo, ap) + fixed = isfixed(pkg, prepo, ap) + LibGit.close(prepo) + pkgs[pkg] = (ver, fixed) end return pkgs end From 12a525c4d1c22ab320790cada4b2b173c4f2341c Mon Sep 17 00:00:00 2001 From: Art Wild Date: Tue, 5 May 2015 17:38:47 -0400 Subject: [PATCH 0278/1938] removed `Git` module calls from `Read` `Repo` object auto-frees itself --- base/pkg/libgit.jl | 65 +++++++++++++++++++++++++++++----------------- base/pkg/read.jl | 34 ++++++++++++------------ 2 files changed, 58 insertions(+), 41 deletions(-) diff --git a/base/pkg/libgit.jl b/base/pkg/libgit.jl index cabd558f96f70..4f196fedc038e 100644 --- a/base/pkg/libgit.jl +++ b/base/pkg/libgit.jl @@ -1,5 +1,17 @@ module LibGit +const GITHUB_REGEX = + r"^(?:git@|git://|https://(?:[\w\.\+\-]+@)?)github.com[:/](([^/].+)/(.+?))(?:\.git)?$"i + +type Repo + ptr::Ptr{Void} + function Repo(ptr::Ptr{Void}) + r = new(ptr) + finalizer(r, r -> ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), r.ptr)) + return r + end +end + type Ref ptr::Ptr{Void} function Ref(ptr::Ptr{Void}) @@ -18,7 +30,7 @@ type Obj end end -function repo(path::String) +function Repo(path::String) repo_ptr = Ptr{Void}[0] err = ccall((:git_repository_open, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Uint8}), repo_ptr, path) @@ -26,19 +38,19 @@ function repo(path::String) if repo_ptr[1] != C_NULL ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), repo_ptr[1]) end - return C_NULL + return nothing end - return repo_ptr[1] + return Repo(repo_ptr[1]) end -function close(repo_ptr::Ptr{Void}) - ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), repo_ptr) +function close(repo::Repo) + ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), repo.ptr) end -function repo_head(r::Ptr{Void}) +function head(repo::Repo) head_ptr = Ptr{Void}[0] err = ccall((:git_repository_head, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}), head_ptr, r) + (Ptr{Ptr{Void}}, Ptr{Void}), head_ptr, repo.ptr) (err != 0) && return nothing return Ref(head_ptr[1]) end @@ -50,17 +62,17 @@ function ref_id(ref::Ref) return bytes2hex(pointer_to_array(oid_ptr, 20)) end -function need_update(repo::Ptr{Void}) - ccall((:git_repository_is_bare, :libgit2), Cint, (Ptr{Void},), repo) != 1 && "git update-index -q --really-refresh" +function need_update(repo::Repo) + ccall((:git_repository_is_bare, :libgit2), Cint, (Ptr{Void},), repo.ptr) != 1 && "git update-index -q --really-refresh" end -function iscommit(id::String, repo::Ptr{Void}) +function iscommit(id::String, repo::Repo) need_update(repo) oid = hex2bytes(id) cmt_ptr = Ptr{Void}[0] err = ccall((:git_commit_lookup, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Uint8}), cmt_ptr, repo, oid) + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Uint8}), cmt_ptr, repo.ptr, oid) if err != 0 return false else @@ -76,41 +88,46 @@ function obj_id(ref::Obj) return bytes2hex(pointer_to_array(oid_ptr, 20)) end -function isdirty(repo::Ptr{Void}) +function isdirty(repo::Repo, paths::AbstractString="") obj_ptr = Ptr{Void}[0] obj = "HEAD^{tree}" err = ccall((:git_revparse_single, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Uint8}), obj_ptr, repo, obj) + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Uint8}), obj_ptr, repo.ptr, obj) err != 0 && return true tree_oid = hex2bytes(obj_id(Obj(obj_ptr[1]))) tree_ptr = Ptr{Void}[0] err = ccall((:git_tree_lookup, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Uint8}), tree_ptr, repo, tree_oid) + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Uint8}), tree_ptr, repo.ptr, tree_oid) err != 0 && return true diff_ptr = Ptr{Void}[0] err = ccall((:git_diff_tree_to_workdir_with_index, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, Ptr{Void}), - diff_ptr, repo, tree_ptr[1], C_NULL, C_NULL) + diff_ptr, repo.ptr, tree_ptr[1], C_NULL, C_NULL) err != 0 && return true - c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff_ptr[1]) - c > 0 && return true - + if isempty(paths) + c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff_ptr[1]) + c > 0 && return true + else + # TODO look for specified path + c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff_ptr[1]) + c > 0 && return true + end return false end -function isattached(repo::Ptr{Void}) - ccall((:git_repository_head_detached, :libgit2), Cint, (Ptr{Void},), repo) != 1 +function isattached(repo::Repo) + ccall((:git_repository_head_detached, :libgit2), Cint, (Ptr{Void},), repo.ptr) != 1 end -function merge_base(one::String, two::String, repo::Ptr{Void}) +function merge_base(one::String, two::String, repo::Repo) oid1 = hex2bytes(one) oid2 = hex2bytes(two) moid = zeros(UInt8 ,20) err = ccall((:git_merge_base, :libgit2), Cint, - (Ptr{UInt8}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}), moid, repo, oid1, oid2) + (Ptr{UInt8}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}), moid, repo.ptr, oid1, oid2) if err != 0 return "" else @@ -118,9 +135,9 @@ function merge_base(one::String, two::String, repo::Ptr{Void}) end end -function is_ancestor_of(a::AbstractString, b::AbstractString, repo::Ptr{Void}) +function is_ancestor_of(a::AbstractString, b::AbstractString, repo::Repo) #A = readchomp(`rev-parse $a`, dir=dir) - merge_base(a, b, prepo) == a + merge_base(a, b, repo) == a end end \ No newline at end of file diff --git a/base/pkg/read.jl b/base/pkg/read.jl index 9889d8606e3ae..edd0ef12ec3ad 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -2,7 +2,7 @@ module Read -import ..Git, ..Cache, ..Reqs, ..LibGit +import ..LibGit, ..Cache, ..Reqs using ..Types readstrip(path...) = strip(readall(joinpath(path...))) @@ -55,22 +55,22 @@ end isinstalled(pkg::AbstractString) = pkg != "METADATA" && pkg != "REQUIRE" && pkg[1] != '.' && isdir(pkg) -function isfixed(pkg::AbstractString, prepo::Ptr{Void}, avail::Dict=available(pkg)) +function isfixed(pkg::AbstractString, prepo::LibGit.Repo, avail::Dict=available(pkg)) isinstalled(pkg) || error("$pkg is not an installed package.") isfile("METADATA", pkg, "url") || return true ispath(pkg, ".git") || return true LibGit.isdirty(prepo) && return true LibGit.isattached(prepo) && return true - #!Git.success(`cat-file -e HEAD:REQUIRE`, dir=pkg) && isfile(pkg,"REQUIRE") && return true + #TODO: !Git.success(`cat-file -e HEAD:REQUIRE`, dir=pkg) && isfile(pkg,"REQUIRE") && return true - head = LibGit.ref_id(LibGit.repo_head(prepo)) + head = LibGit.ref_id(LibGit.head(prepo)) for (ver,info) in avail head == info.sha1 && return false end cache = Cache.path(pkg) - crepo = LibGit.repo(cache) # open Cache repo + crepo = LibGit.Repo(cache) # open Cache repo cache_has_head = isdir(cache) && LibGit.iscommit(head, crepo) res = true for (ver,info) in avail @@ -88,20 +88,20 @@ function isfixed(pkg::AbstractString, prepo::Ptr{Void}, avail::Dict=available(pk Base.warn_once("unknown $pkg commit $(info.sha1[1:8]), metadata may be ahead of package cache") end end - close(crepo) return res end -function installed_version(pkg::AbstractString, prepo::Ptr{Void}, avail::Dict=available(pkg)) +function installed_version(pkg::AbstractString, prepo::LibGit.Repo, avail::Dict=available(pkg)) ispath(pkg,".git") || return typemin(VersionNumber) # get package repo head hash - head = LibGit.ref_id(LibGit.repo_head(prepo)) + head = LibGit.ref_id(LibGit.head(prepo)) vers = collect(keys(filter((ver,info)->info.sha1==head, avail))) !isempty(vers) && return maximum(vers) + cache = Cache.path(pkg) - crepo = LibGit.repo(cache) # open Cache repo + crepo = LibGit.Repo(cache) # open Cache repo cache_has_head = isdir(cache) && LibGit.iscommit(head, crepo) ancestors = VersionNumber[] descendants = VersionNumber[] @@ -118,7 +118,7 @@ function installed_version(pkg::AbstractString, prepo::Ptr{Void}, avail::Dict=av base == sha1 && push!(ancestors,ver) base == head && push!(descendants,ver) end - LibGit.close(crepo) # close Cache repo + both = sort!(intersect(ancestors,descendants)) isempty(both) || warn("$pkg: some versions are both ancestors and descendants of head: $both") if !isempty(descendants) @@ -135,9 +135,10 @@ end function requires_path(pkg::AbstractString, avail::Dict=available(pkg)) pkgreq = joinpath(pkg,"REQUIRE") ispath(pkg,".git") || return pkgreq - Git.dirty("REQUIRE", dir=pkg) && return pkgreq - !Git.success(`cat-file -e HEAD:REQUIRE`, dir=pkg) && isfile(pkgreq) && return pkgreq - head = Git.head(dir=pkg) + repo = LibGit.Repo(pkg) + LibGit.dirty(repo, "REQUIRE") && return pkgreq + #TODO: !Git.success(`cat-file -e HEAD:REQUIRE`, dir=pkg) && isfile(pkgreq) && return pkgreq + head = LibGit.ref_id(LibGit.head(repo)) for (ver,info) in avail if head == info.sha1 return joinpath("METADATA", pkg, "versions", string(ver), "requires") @@ -157,10 +158,9 @@ function installed(avail::Dict=available()) for pkg in readdir() isinstalled(pkg) || continue ap = get(avail,pkg,Dict{VersionNumber,Available}()) - prepo = LibGit.repo(pkg) + prepo = LibGit.Repo(pkg) ver = installed_version(pkg, prepo, ap) fixed = isfixed(pkg, prepo, ap) - LibGit.close(prepo) pkgs[pkg] = (ver, fixed) end return pkgs @@ -189,8 +189,8 @@ end function issue_url(pkg::AbstractString) ispath(pkg,".git") || return "" - m = match(Git.GITHUB_REGEX, url(pkg)) - m === nothing && return "" + m = match(LibGit.GITHUB_REGEX, url(pkg)) + m == nothing && return "" return "https://github.com/" * m.captures[1] * "/issues" end From b2be2d79f10ff03c2679836848a34aae84ce62de Mon Sep 17 00:00:00 2001 From: Art Wild Date: Tue, 5 May 2015 17:54:35 -0400 Subject: [PATCH 0279/1938] changed string type --- base/pkg/libgit.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/pkg/libgit.jl b/base/pkg/libgit.jl index 4f196fedc038e..52c5d120c3093 100644 --- a/base/pkg/libgit.jl +++ b/base/pkg/libgit.jl @@ -30,7 +30,7 @@ type Obj end end -function Repo(path::String) +function Repo(path::AbstractString) repo_ptr = Ptr{Void}[0] err = ccall((:git_repository_open, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Uint8}), repo_ptr, path) @@ -66,7 +66,7 @@ function need_update(repo::Repo) ccall((:git_repository_is_bare, :libgit2), Cint, (Ptr{Void},), repo.ptr) != 1 && "git update-index -q --really-refresh" end -function iscommit(id::String, repo::Repo) +function iscommit(id::AbstractString, repo::Repo) need_update(repo) oid = hex2bytes(id) @@ -122,7 +122,7 @@ function isattached(repo::Repo) ccall((:git_repository_head_detached, :libgit2), Cint, (Ptr{Void},), repo.ptr) != 1 end -function merge_base(one::String, two::String, repo::Repo) +function merge_base(one::AbstractString, two::AbstractString, repo::Repo) oid1 = hex2bytes(one) oid2 = hex2bytes(two) moid = zeros(UInt8 ,20) From 9c88dbc0971eb97be18616844c4bb6fee0d72d4d Mon Sep 17 00:00:00 2001 From: Art Wild Date: Wed, 6 May 2015 12:46:14 -0400 Subject: [PATCH 0280/1938] renamed to 'libgit2' moved parts to `libgit2` dir converted github.jl refactored repository and configuration types (as in LibGit2 pkg) --- base/pkg.jl | 2 +- base/pkg/dir.jl | 11 +- base/pkg/entry.jl | 17 ++- base/pkg/github.jl | 7 +- base/pkg/{libgit.jl => libgit2.jl} | 119 +++++++++------ base/pkg/libgit2/config.jl | 56 +++++++ base/pkg/libgit2/const.jl | 229 +++++++++++++++++++++++++++++ base/pkg/libgit2/repository.jl | 36 +++++ base/pkg/read.jl | 64 ++++---- 9 files changed, 457 insertions(+), 84 deletions(-) rename base/pkg/{libgit.jl => libgit2.jl} (56%) create mode 100644 base/pkg/libgit2/config.jl create mode 100644 base/pkg/libgit2/const.jl create mode 100644 base/pkg/libgit2/repository.jl diff --git a/base/pkg.jl b/base/pkg.jl index 156a9899438cd..c2c8a529d4a11 100644 --- a/base/pkg.jl +++ b/base/pkg.jl @@ -10,7 +10,7 @@ export dir, init, rm, add, available, installed, status, clone, checkout, const DEFAULT_META = "git://github.com/JuliaLang/METADATA.jl" const META_BRANCH = "metadata-v2" -for file in split("git libgit dir github types reqs cache read query resolve write generate entry") +for file in split("git libgit2 dir github types reqs cache read query resolve write generate entry") include("pkg/$file.jl") end const cd = Dir.cd diff --git a/base/pkg/dir.jl b/base/pkg/dir.jl index 0e253001fe96f..b6a186297499c 100644 --- a/base/pkg/dir.jl +++ b/base/pkg/dir.jl @@ -3,7 +3,7 @@ module Dir import ..Pkg: DEFAULT_META, META_BRANCH -import ..Git +import ..LibGit2 const DIR_NAME = ".julia" @@ -32,15 +32,12 @@ function cd(f::Function, args...; kws...) end function init(meta::AbstractString=DEFAULT_META, branch::AbstractString=META_BRANCH) - if Git.version() < v"1.7.3" - warn("Pkg only works with git versions greater than v1.7.3") - end dir = path() info("Initializing package repository $dir") metadata_dir = joinpath(dir, "METADATA") if isdir(metadata_dir) info("Package directory $dir is already initialized.") - Git.set_remote_url(meta, dir=metadata_dir) + LibGit2.set_remote_url(metadata_dir, meta) return end local temp_dir @@ -49,8 +46,8 @@ function init(meta::AbstractString=DEFAULT_META, branch::AbstractString=META_BRA temp_dir = mktempdir(dir) Base.cd(temp_dir) do info("Cloning METADATA from $meta") - run(`git clone -q -b $branch $meta METADATA`) - Git.set_remote_url(meta, dir="METADATA") + run(`git clone -q -b $branch $meta METADATA`) #TODO: LibGit2.clone + LibGit2.set_remote_url("METADATA", meta) touch("REQUIRE") touch("META_BRANCH") open("META_BRANCH", "w") do io diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 77eebbba8060a..224388f4a608b 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -3,7 +3,7 @@ module Entry import Base: thispatch, nextpatch, nextminor, nextmajor, check_new_version -import ..LibGit, ..Git, ..Reqs, ..Read, ..Query, ..Resolve, ..Cache, ..Write, ..GitHub, ..Dir +import ..LibGit2, ..Git, ..Reqs, ..Read, ..Query, ..Resolve, ..Cache, ..Write, ..GitHub, ..Dir using ..Types macro recover(ex) @@ -107,9 +107,9 @@ end function installed(pkg::AbstractString) avail = Read.available(pkg) if Read.isinstalled(pkg) - prepo = LibGit.repo(pkg) + prepo = LibGit2.GitRepo(pkg) res = Read.installed_version(pkg, prepo, avail) - LibGit.close(prepo) + LibGit2.free!(prepo) return res end isempty(avail) && error("$pkg is not a package (not registered or installed)") @@ -148,11 +148,18 @@ function status(io::IO, pkg::AbstractString, ver::VersionNumber, fix::Bool) fix || return println(io,ver) @printf io "%-19s" ver if ispath(pkg,".git") - print(io, Git.attached(dir=pkg) ? Git.branch(dir=pkg) : Git.head(dir=pkg)[1:8]) + prepo = LibGit2.GitRepo(pkg) + phead = LibGit2.head(prepo) + if LibGit2.isattached(prepo) + print(io, LibGit2.ref_name(phead)) + else + print(io, LibGit2.ref_id(phead)[1:8]) + end attrs = AbstractString[] isfile("METADATA",pkg,"url") || push!(attrs,"unregistered") - Git.dirty(dir=pkg) && push!(attrs,"dirty") + LibGit2.isdirty(prepo) && push!(attrs,"dirty") isempty(attrs) || print(io, " (",join(attrs,", "),")") + LibGit2.free!(prepo) else print(io, "non-repo (unregistered)") end diff --git a/base/pkg/github.jl b/base/pkg/github.jl index 24310b8276f50..161324c8dd6c0 100644 --- a/base/pkg/github.jl +++ b/base/pkg/github.jl @@ -2,7 +2,7 @@ module GitHub -import Main, ..Git, ..Dir +import Main, ..LibGit2, ..Dir const AUTH_NOTE = "Julia Package Manager" const AUTH_DATA = Dict{Any,Any}( @@ -12,7 +12,8 @@ const AUTH_DATA = Dict{Any,Any}( ) function user() - if !success(`git config --global github.user`) + usr = LibGit2.lookup(String, LibGit2.GitConfig(), "github.user") + if usr == nothing error(""" no GitHub user name configured; please configure it with: @@ -21,7 +22,7 @@ function user() where USERNAME is replaced with your GitHub user name. """) end - readchomp(`git config --global github.user`) + return usr end function json() diff --git a/base/pkg/libgit.jl b/base/pkg/libgit2.jl similarity index 56% rename from base/pkg/libgit.jl rename to base/pkg/libgit2.jl index 52c5d120c3093..34db174596e58 100644 --- a/base/pkg/libgit.jl +++ b/base/pkg/libgit2.jl @@ -1,17 +1,21 @@ -module LibGit +module LibGit2 const GITHUB_REGEX = r"^(?:git@|git://|https://(?:[\w\.\+\-]+@)?)github.com[:/](([^/].+)/(.+?))(?:\.git)?$"i -type Repo - ptr::Ptr{Void} - function Repo(ptr::Ptr{Void}) - r = new(ptr) - finalizer(r, r -> ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), r.ptr)) - return r +function __init__() + err = ccall((:git_libgit2_init, :libgit2), Cint, ()) + err > 0 || error("error initializing LibGit2 module") + atexit() do + gc() + ccall((:git_libgit2_shutdown, :libgit2), Cint, ()) end end +include("libgit2/const.jl") +include("libgit2/repository.jl") +include("libgit2/config.jl") + type Ref ptr::Ptr{Void} function Ref(ptr::Ptr{Void}) @@ -30,43 +34,45 @@ type Obj end end -function Repo(path::AbstractString) - repo_ptr = Ptr{Void}[0] - err = ccall((:git_repository_open, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Uint8}), repo_ptr, path) - if err != 0 - if repo_ptr[1] != C_NULL - ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), repo_ptr[1]) - end - return nothing - end - return Repo(repo_ptr[1]) -end - -function close(repo::Repo) - ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), repo.ptr) -end - -function head(repo::Repo) +function head(repo::GitRepo) head_ptr = Ptr{Void}[0] err = ccall((:git_repository_head, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}), head_ptr, repo.ptr) + (err != 0) && return nothing return Ref(head_ptr[1]) end function ref_id(ref::Ref) ref == nothing && return "" - oid_ptr = ccall((:git_reference_target, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) - oid_ptr == C_NULL && return "" - return bytes2hex(pointer_to_array(oid_ptr, 20)) + + typ = ccall((:git_reference_type, :libgit2), Cint, (Ptr{Void},), ref.ptr) + if typ == 1 + oid_ptr = ccall((:git_reference_target, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) + oid_ptr == C_NULL && return "" + return bytes2hex(pointer_to_array(oid_ptr, 20)) + else + oid_ptr = ccall((:git_reference_symbolic_target, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) + oid_ptr == C_NULL && return "" + return bytestring(oid_ptr) + end end -function need_update(repo::Repo) +head_oid(repo::GitRepo) = head(repo) |> ref_id + +function ref_name(ref::Ref) + ref == nothing && return "" + + name_ptr = ccall((:git_reference_shorthand, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) + name_ptr == C_NULL && return "" + return bytestring(name_ptr) +end + +function need_update(repo::GitRepo) ccall((:git_repository_is_bare, :libgit2), Cint, (Ptr{Void},), repo.ptr) != 1 && "git update-index -q --really-refresh" end -function iscommit(id::AbstractString, repo::Repo) +function iscommit(id::AbstractString, repo::GitRepo) need_update(repo) oid = hex2bytes(id) @@ -88,14 +94,10 @@ function obj_id(ref::Obj) return bytes2hex(pointer_to_array(oid_ptr, 20)) end -function isdirty(repo::Repo, paths::AbstractString="") - obj_ptr = Ptr{Void}[0] - obj = "HEAD^{tree}" - err = ccall((:git_revparse_single, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Uint8}), obj_ptr, repo.ptr, obj) - err != 0 && return true +function isdirty(repo::GitRepo, paths::AbstractString="") + tree_oid =revparse(repo, "HEAD^{tree}") + tree_oid == nothing && return true - tree_oid = hex2bytes(obj_id(Obj(obj_ptr[1]))) tree_ptr = Ptr{Void}[0] err = ccall((:git_tree_lookup, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Uint8}), tree_ptr, repo.ptr, tree_oid) @@ -118,11 +120,11 @@ function isdirty(repo::Repo, paths::AbstractString="") return false end -function isattached(repo::Repo) +function isattached(repo::GitRepo) ccall((:git_repository_head_detached, :libgit2), Cint, (Ptr{Void},), repo.ptr) != 1 end -function merge_base(one::AbstractString, two::AbstractString, repo::Repo) +function merge_base(one::AbstractString, two::AbstractString, repo::GitRepo) oid1 = hex2bytes(one) oid2 = hex2bytes(two) moid = zeros(UInt8 ,20) @@ -135,9 +137,42 @@ function merge_base(one::AbstractString, two::AbstractString, repo::Repo) end end -function is_ancestor_of(a::AbstractString, b::AbstractString, repo::Repo) - #A = readchomp(`rev-parse $a`, dir=dir) - merge_base(a, b, repo) == a +function revparse(repo::GitRepo, obj::AbstractString) + obj_ptr = Ptr{Void}[0] + err = ccall((:git_revparse_single, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Uint8}), obj_ptr, repo.ptr, obj) + err != 0 && return nothing + return hex2bytes(obj_id(Obj(obj_ptr[1]))) +end + +function is_ancestor_of(a::AbstractString, b::AbstractString, repo::GitRepo) + A = revparse(repo, a) + merge_base(a, b, repo) == A +end + +function set_remote_url(repo::GitRepo, url::AbstractString; remote::AbstractString="origin") + cfg = GitConfig(repo) + + err = set_config_string(cfg_ptr[1], "remote.$remote.url", url) + err !=0 && return + + m = match(GITHUB_REGEX,url) + m == nothing && return + push = "git@github.com:$(m.captures[1]).git" + if push != url + err = set_config_string(cfg_ptr[1], "remote.$remote.pushurl", push) + end +end + +function set_remote_url(path::AbstractString, url::AbstractString; remote::AbstractString="origin") + repo = GitRepo(path) + set_remote_url(repo, url, remote=remote) + LibGit2.free!(prepo) +end + +function normalize_url(url::AbstractString) + m = match(GITHUB_REGEX,url) + m == nothing ? url : "git://github.com/$(m.captures[1]).git" end end \ No newline at end of file diff --git a/base/pkg/libgit2/config.jl b/base/pkg/libgit2/config.jl new file mode 100644 index 0000000000000..46a501b9da464 --- /dev/null +++ b/base/pkg/libgit2/config.jl @@ -0,0 +1,56 @@ +type GitConfig + ptr::Ptr{Void} + + function GitConfig(ptr::Ptr{Void}) + @assert ptr != C_NULL + cfg = new(ptr) + finalizer(cfg, free!) + return cfg + end +end + +function free!(cfg::GitConfig) + if cfg.ptr != C_NULL + ccall((:git_config_free, :libgit2), Void, (Ptr{Void},), cfg.ptr) + cfg.ptr = C_NULL + end +end + +GitConfig(path::AbstractString) = begin + cfg_ptr = Ptr{Void}[C_NULL] + err = ccall((:git_config_open_ondisk, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Uint8}), cfg_ptr, path) + err !=0 && return nothing + return GitConfig(cfg_ptr[1]) +end + +GitConfig(r::GitRepo) = begin + cfg_ptr = Ptr{Void}[C_NULL] + err = ccall((:git_repository_config, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}), cfg_ptr, r) + err !=0 && return nothing + return GitConfig(cfg_ptr[1]) +end + +GitConfig() = begin + cfg_ptr = Ptr{Void}[C_NULL] + ccall((:git_config_open_default, :libgit2), Cint, (Ptr{Ptr{Void}}, ), cfg_ptr) + return GitConfig(cfg_ptr[1]) +end + +function lookup{T<:AbstractString}(::Type{T}, c::GitConfig, name::AbstractString) + out = Ptr{Uint8}[0] + err = ccall((:git_config_get_string, :libgit2), Cint, + (Ptr{Ptr{Uint8}}, Ptr{Void}, Ptr{Uint8}), out, c.ptr, name) + if err == GitErrorConst.GIT_OK + return bytestring(out[1]) + else + return nothing + end +end + +function set!{T<:AbstractString}(::Type{T}, c::GitConfig, name::AbstractString, value::AbstractString) + err = ccall((:git_config_set_string, :libgit2), Cint, + (Ptr{Void}, Ptr{Uint8}, Ptr{Uint8}), c.ptr, name, value) + return err +end diff --git a/base/pkg/libgit2/const.jl b/base/pkg/libgit2/const.jl new file mode 100644 index 0000000000000..876cd98149dbc --- /dev/null +++ b/base/pkg/libgit2/const.jl @@ -0,0 +1,229 @@ +module GitConst + + const OBJ_ANY = Cint(-2) + const OBJ_BAD = Cint(-1) + const OBJ_COMMIT = Cint(1) + const OBJ_TREE = Cint(2) + const OBJ_BLOB = Cint(3) + const OBJ_TAG = Cint(4) + + const SORT_NONE = Cint(0) + const SORT_TOPOLOGICAL = Cint(1) << Cint(0) + const SORT_TIME = Cint(1) << Cint(1) + const SORT_REVERSE = Cint(1) << Cint(2) + + const REF_INVALID = Cint(0) + const REF_OID = Cint(1) + const REF_SYMBOLIC = Cint(2) + const REF_LISTALL = REF_OID | REF_SYMBOLIC + + const BRANCH_LOCAL = Cint(1) + const BRANCH_REMOTE = Cint(2) + + const FILEMODE_NEW = Cint(00000) + const FILEMODE_TREE = Cint(16384) + const FILEMODE_BLOB = Cint(33188) + const FILEMODE_BLOB_EXECUTABLE = Cint(33261) + const FILEMODE_LINK = Cint(40960) + const FILEMODE_COMMIT = Cint(57344) + + const CHECKOUT_NONE = Cint(0) + const CHECKOUT_SAFE = Cuint(1) << Cint(0) + const CHECKOUT_SAFE_CREATE = Cuint(1) << Cint(1) + const CHECKOUT_FORCE = Cuint(1) << Cint(2) + const CHECKOUT_ALLOW_CONFLICTS = Cuint(1) << Cint(4) + const CHECKOUT_REMOVE_UNTRACKED = Cuint(1) << Cint(5) + const CHECKOUT_REMOVE_IGNORED = Cuint(1) << Cint(6) + const CHECKOUT_UPDATE_ONLY = Cuint(1) << Cint(7) + const CHECKOUT_DONT_UPDATE_INDEX = Cuint(1) << Cint(8) + const CHECKOUT_NO_REFRESH = Cuint(1) << Cint(9) + const CHECKOUT_SKIP_UNMERGED = Cuint(1) << Cint(10) + const CHECKOUT_USE_OURS = Cuint(1) << Cint(11) + const CHECKOUT_USE_THEIRS = Cuint(1) << Cint(12) + const CHECKOUT_DISABLE_PATHSPEC_MATCH = Cuint(1) << Cint(13) + const CHECKOUT_SKIP_LOCKED_DIRECTORIES = Cuint(1) << Cint(18) + const CHECKOUT_DONT_OVERWRITE_IGNORED = Cuint(1) << Cint(19) + + const CHECKOUT_UPDATE_SUBMODULES = Cuint(1) << Cint(16) + const CHECKOUT_UPDATE_SUBMODULES_IF_CHANGED = Cuint(1) << Cint(17) + + const CHECKOUT_NOTIFY_NONE = Cint(0) + const CHECKOUT_NOTIFY_CONFLICT = Cuint(1) << Cint(0) + const CHECKOUT_NOTIFY_DIRTY = Cuint(1) << Cint(1) + const CHECKOUT_NOTIFY_UPDATED = Cuint(1) << Cint(2) + const CHECKOUT_NOTIFY_UNTRACKED = Cuint(1) << Cint(3) + const CHECKOUT_NOTIFY_IGNORED = Cuint(1) << Cint(4) + const CHECKOUT_NOTIFY_ALL = 0x0FFFF + + const SUBMODULE_UPDATE_RESET = Cint(-1) + const SUBMODULE_UPDATE_CHECKOUT = Cint(1) + const SUBMODULE_UPDATE_REBASE = Cint(2) + const SUBMODULE_UPDATE_MERGE = Cint(3) + const SUBMODULE_UPDATE_NONE = Cint(4) + const SUBMODULE_UPDATE_DEFAULT = Cint(0) + + # git_submodule_ignore_t + const SUBMODULE_IGNORE_RESET = Cint(-1) + const SUBMODULE_IGNORE_NONE = Cint(1) + const SUBMODULE_IGNORE_UNTRACKED = Cint(2) + const SUBMODULE_IGNORE_DIRTY = Cint(3) + const SUBMODULE_IGNORE_ALL = Cint(4) + const SUBMODULE_IGNORE_DEFAULT = Cint(0) + + const TREEWALK_PRE = Cint(0) + const TREEWALK_POST = Cint(1) + + const GIT_PATH_MAX = Cint(4096) + + const DIFF_OPTIONS_VERSION = Cint(1) + + const DIFF_NORMAL = Cint(0) + const DIFF_REVERSE = Cuint(1) << Cint(0) + const DIFF_INCLUDE_IGNORED = Cuint(1) << Cint(1) + const DIFF_RECURSE_IGNORED_DIRS = Cuint(1) << Cint(2) + const DIFF_INCLUDE_UNTRACKED = Cuint(1) << Cint(3) + const DIFF_RECURSE_UNTRACKED_DIRS = Cuint(1) << Cint(4) + const DIFF_INCLUDE_UNMODIFIED = Cuint(1) << Cint(5) + const DIFF_INCLUDE_TYPECHANGE = Cuint(1) << Cint(6) + const DIFF_INCLUDE_TYPECHANGE_TREES = Cuint(1) << Cint(7) + const DIFF_IGNORE_FILEMODE = Cuint(1) << Cint(8) + const DIFF_IGNORE_SUBMODULES = Cuint(1) << Cint(9) + const DIFF_IGNORE_CASE = Cuint(1) << Cint(10) + const DIFF_DISABLE_PATHSPEC_MATCH = Cuint(1) << Cint(12) + const DIFF_SKIP_BINARY_CHECK = Cuint(1) << Cint(13) + const DIFF_ENABLE_FAST_UNTRACKED_DIRS = Cuint(1) << Cint(14) + + const DIFF_FORCE_TEXT = Cuint(1) << Cint(20) + const DIFF_FORCE_BINARY = Cuint(1) << Cint(21) + const DIFF_IGNORE_WHITESPACE = Cuint(1) << Cint(22) + const DIFF_IGNORE_WHITESPACE_CHANGE = Cuint(1) << Cint(23) + const DIFF_IGNORE_WHITESPACE_EOL = Cuint(1) << Cint(24) + const DIFF_SHOW_UNTRACKED_CONTENT = Cuint(1) << Cint(25) + const DIFF_SHOW_UNMODIFIED = Cuint(1) << Cint(26) + const DIFF_PATIENCE = Cuint(1) << Cint(28) + const DIFF_MINIMAL = Cuint(1) << Cint(29) + + const DIFF_FLAG_BINARY = Cuint(1) << Cint(0) + const DIFF_FLAG_NOT_BINARY = Cuint(1) << Cint(1) + const DIFF_FLAG_VALID_OID = Cuint(1) << Cint(2) + + const DIFF_FORMAT_PATCH = Cuint(1) + const DIFF_FORMAT_PATCH_HEADER = Cuint(2) + const DIFF_FORMAT_RAW = Cuint(3) + const DIFF_FORMAT_NAME_ONLY = Cuint(4) + const DIFF_FORMAT_NAME_STATUS = Cuint(5) + + const DELTA_UNMODIFIED = Cint(0) + const DELTA_ADDED = Cint(1) + const DELTA_DELETED = Cint(2) + const DELTA_MODIFIED = Cint(3) + const DELTA_RENAMED = Cint(4) + const DELTA_COPIED = Cint(5) + const DELTA_IGNORED = Cint(6) + const DELTA_UNTRACKED = Cint(7) + const DELTA_TYPECHANGE = Cint(8) + + const DIFF_LINE_CONTEXT = Cchar(' ') + const DIFF_LINE_ADDITION = Cchar('+') + const DIFF_LINE_DELETION = Cchar('-') + + const DIFF_LINE_CONTEXT_EOFNL = Cchar('=') + const DIFF_LINE_ADD_EOFNL = Cchar('>') + const DIFF_LINE_DEL_EOFNL = Cchar('<') + + const DIFF_LINE_FILE_HDR = Cchar('F') + const DIFF_LINE_HUNK_HDR = Cchar('H') + const DIFF_LINE_BINARY = Cchar('B') + + # index + const IDXENTRY_NAMEMASK = (0x0fff) + const IDXENTRY_STAGEMASK = (0x3000) + const IDXENTRY_EXTENDED = (0x4000) + const IDXENTRY_VALID = (0x8000) + const IDXENTRY_STAGESHIFT = Cint(12) + + const IDXENTRY_UPDATE = Cint(1) << Cint(0) + const IDXENTRY_REMOVE = Cint(1) << Cint(1) + const IDXENTRY_UPTODATE = Cint(1) << Cint(2) + const IDXENTRY_ADDED = Cint(1) << Cint(3) + + const IDXENTRY_HASHED = Cint(1) << Cint(4) + const IDXENTRY_UNHASHED = Cint(1) << Cint(5) + const IDXENTRY_WT_REMOVE = Cint(1) << Cint(6) + const IDXENTRY_CONFLICTED = Cint(1) << Cint(7) + + const IDXENTRY_UNPACKED = Cint(1) << Cint(8) + const IDXENTRY_NEW_SKIP_WORKTREE = Cint(1) << Cint(9) + + const INDEXCAP_IGNORE_CASE = Cuint(1) + const INDEXCAP_NO_FILEMODE = Cuint(2) + const INDEXCAP_NO_SYMLINKS = Cuint(4) + const INDEXCAP_FROM_OWNER = ~(Cuint(0)) + + const INDEX_ADD_DEFAULT = Cint(0) + const INDEX_ADD_FORCE = Cuint(1) << Cint(0) + const INDEX_ADD_DISABLE_PATHSPEC_MATCH = Cuint(1) << Cint(1) + const INDEX_ADD_CHECK_PATHSPEC = Cuint(1) << Cint(2) + + const INDEX_STAGE_ANY = Cint(-1) + + const MERGE_TREE_FIND_RENAMES = Cint(1) << Cint(0) + + const MERGE_AUTOMERGE_NORMAL = Cint(0) + const MERGE_AUTOMERGE_FAVOR_OURS = Cint(1) + const MERGE_AUTOMERGE_FAVOR_THEIRS = Cint(2) + const MERGE_AUTOMERGE_FAVOR_UNION = Cint(3) + + const MERGE_NO_FASTFORWARD = Cint(1) + const MERGE_FASTFORWARD_ONLY = Cint(2) + + const GIT_MERGE_ANALYSIS_NONE = 0, + const GIT_MERGE_ANALYSIS_NORMAL = (1 << 0) + const GIT_MERGE_ANALYSIS_UP_TO_DATE = (1 << 1) + const GIT_MERGE_ANALYSIS_FASTFORWARD = (1 << 2) + const GIT_MERGE_ANALYSIS_UNBORN = (1 << 3) + + const GIT_MERGE_PREFERENCE_NONE = 0 + const GIT_MERGE_PREFERENCE_NO_FASTFORWARD = (1 << 0) + const GIT_MERGE_PREFERENCE_FASTFORWARD_ONLY = (1 << 1) + + const DIRECTION_FETCH = Cint(0) + const DIRECTION_PUSH = Cint(1) + + const BLAME_NORMAL = Cint(0) + + const CREDTYPE_USERPASS_PLAINTEXT = Cuint(1) << Cint(0) + const CREDTYPE_SSH_KEY = Cuint(1) << Cint(1) + const CREDTYPE_SSH_CUSTOM = Cuint(1) << Cint(2) + const CREDTYPE_DEFAULT = Cuint(1) << Cint(3) + + const GIT_REPOSITORY_STATE_NONE = Cint(0) + const GIT_REPOSITORY_STATE_MERGE = Cint(1) + const GIT_REPOSITORY_STATE_REVERT = Cint(2) + const GIT_REPOSITORY_STATE_CHERRY_PICK = Cint(3) + const GIT_REPOSITORY_STATE_BISECT = Cint(4) + const GIT_REPOSITORY_STATE_REBASE = Cint(5) + const GIT_REPOSITORY_STATE_REBASE_INTERACTIVE = Cint(6) + const GIT_REPOSITORY_STATE_REBASE_MERGE = Cint(7) + const GIT_REPOSITORY_STATE_APPLY_MAILBOX = Cint(8) + const GIT_REPOSITORY_STATE_APPLY_MAILBOX_OR_REBASE = Cint(9) +end + +module GitErrorConst + const GIT_OK = Cint(0) + const ERROR = Cint(-01) + const ENOTFOUND = Cint(-03) + const EEXISTS = Cint(-04) + const EAMBIGUOUS = Cint(-05) + const EBUFS = Cint(-06) + const EUSER = Cint(-07) + const EBAREREPO = Cint(-08) + const EUNBORNBRANCH = Cint(-09) + const EUNMERGED = Cint(-10) + const ENONFASTFORWARD = Cint(-11) + const EINVALIDSPEC = Cint(-12) + const EMERGECONFLICT = Cint(-13) + const ELOCKED = Cint(-14) + const PASSTHROUGH = Cint(-30) + const ITEROVER = Cint(-31) +end \ No newline at end of file diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl new file mode 100644 index 0000000000000..d7ff8c0809555 --- /dev/null +++ b/base/pkg/libgit2/repository.jl @@ -0,0 +1,36 @@ +type GitRepo + ptr::Ptr{Void} + + function GitRepo(ptr::Ptr{Void}, own::Bool=true) + @assert ptr != C_NULL + r = new(ptr) + own && finalizer(r, free!) + return r + end +end + +GitRepo(path::String) = begin + repo_ptr = Ptr{Void}[0] + err = ccall((:git_repository_open, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Uint8}), repo_ptr, path) + if err != GitErrorConst.GIT_OK + if repo_ptr[1] != C_NULL + ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), repo_ptr[1]) + end + return nothing + end + return GitRepo(repo_ptr[1]) +end + +close(r::GitRepo) = begin + if r.ptr != C_NULL + ccall((:git_repository__cleanup, :libgit2), Void, (Ptr{Void},), r.ptr) + end +end + +free!(r::GitRepo) = begin + if r.ptr != C_NULL + ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), r.ptr) + r.ptr = C_NULL + end +end \ No newline at end of file diff --git a/base/pkg/read.jl b/base/pkg/read.jl index edd0ef12ec3ad..7061d54defc8a 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -2,7 +2,7 @@ module Read -import ..LibGit, ..Cache, ..Reqs +import ..LibGit2, ..Cache, ..Reqs using ..Types readstrip(path...) = strip(readall(joinpath(path...))) @@ -55,32 +55,36 @@ end isinstalled(pkg::AbstractString) = pkg != "METADATA" && pkg != "REQUIRE" && pkg[1] != '.' && isdir(pkg) -function isfixed(pkg::AbstractString, prepo::LibGit.Repo, avail::Dict=available(pkg)) +function isfixed(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::Dict=available(pkg)) isinstalled(pkg) || error("$pkg is not an installed package.") isfile("METADATA", pkg, "url") || return true ispath(pkg, ".git") || return true - LibGit.isdirty(prepo) && return true - LibGit.isattached(prepo) && return true - #TODO: !Git.success(`cat-file -e HEAD:REQUIRE`, dir=pkg) && isfile(pkg,"REQUIRE") && return true + LibGit2.isdirty(prepo) && return true + LibGit2.isattached(prepo) && return true + LibGit2.revparse(prepo, "HEAD:REQUIRE") == nothing && isfile(pkg,"REQUIRE") && return true - head = LibGit.ref_id(LibGit.head(prepo)) + head = LibGit2.ref_id(LibGit2.head(prepo)) for (ver,info) in avail head == info.sha1 && return false end cache = Cache.path(pkg) - crepo = LibGit.Repo(cache) # open Cache repo - cache_has_head = isdir(cache) && LibGit.iscommit(head, crepo) + cache_has_head = if isdir(cache) + crepo = LibGit2.GitRepo(cache) + LibGit2.iscommit(head, crepo) + else + false + end res = true for (ver,info) in avail - if cache_has_head && LibGit.iscommit(info.sha1, crepo) - if LibGit.is_ancestor_of(head, info.sha1, crepo) + if cache_has_head && LibGit2.iscommit(info.sha1, crepo) + if LibGit2.is_ancestor_of(head, info.sha1, crepo) res = false break end - elseif LibGit.iscommit(info.sha1, prepo) - if LibGit.is_ancestor_of(head, info.sha1, prepo) + elseif LibGit2.iscommit(info.sha1, prepo) + if LibGit2.is_ancestor_of(head, info.sha1, prepo) res = false break end @@ -88,29 +92,34 @@ function isfixed(pkg::AbstractString, prepo::LibGit.Repo, avail::Dict=available( Base.warn_once("unknown $pkg commit $(info.sha1[1:8]), metadata may be ahead of package cache") end end + cache_has_head && LibGit2.free!(crepo) return res end -function installed_version(pkg::AbstractString, prepo::LibGit.Repo, avail::Dict=available(pkg)) +function installed_version(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::Dict=available(pkg)) ispath(pkg,".git") || return typemin(VersionNumber) # get package repo head hash - head = LibGit.ref_id(LibGit.head(prepo)) + head = LibGit2.ref_id(LibGit2.head(prepo)) vers = collect(keys(filter((ver,info)->info.sha1==head, avail))) !isempty(vers) && return maximum(vers) cache = Cache.path(pkg) - crepo = LibGit.Repo(cache) # open Cache repo - cache_has_head = isdir(cache) && LibGit.iscommit(head, crepo) + cache_has_head = if isdir(cache) + crepo = LibGit2.GitRepo(cache) + LibGit2.iscommit(head, crepo) + else + false + end ancestors = VersionNumber[] descendants = VersionNumber[] for (ver,info) in avail sha1 = info.sha1 - base = if cache_has_head && LibGit.iscommit(sha1, crepo) - LibGit.merge_base(head, sha1, crepo) - elseif LibGit.iscommit(sha1, prepo) - LibGit.merge_base(head, sha1, prepo) + base = if cache_has_head && LibGit2.iscommit(sha1, crepo) + LibGit2.merge_base(head, sha1, crepo) + elseif LibGit2.iscommit(sha1, prepo) + LibGit2.merge_base(head, sha1, prepo) else Base.warn_once("unknown $pkg commit $(sha1[1:8]), metadata may be ahead of package cache") continue @@ -118,6 +127,7 @@ function installed_version(pkg::AbstractString, prepo::LibGit.Repo, avail::Dict= base == sha1 && push!(ancestors,ver) base == head && push!(descendants,ver) end + cache_has_head && LibGit2.free!(crepo) both = sort!(intersect(ancestors,descendants)) isempty(both) || warn("$pkg: some versions are both ancestors and descendants of head: $both") @@ -135,10 +145,11 @@ end function requires_path(pkg::AbstractString, avail::Dict=available(pkg)) pkgreq = joinpath(pkg,"REQUIRE") ispath(pkg,".git") || return pkgreq - repo = LibGit.Repo(pkg) - LibGit.dirty(repo, "REQUIRE") && return pkgreq - #TODO: !Git.success(`cat-file -e HEAD:REQUIRE`, dir=pkg) && isfile(pkgreq) && return pkgreq - head = LibGit.ref_id(LibGit.head(repo)) + repo = LibGit2.GitRepo(pkg) + LibGit2.isdirty(repo, "REQUIRE") && return pkgreq + LibGit2.revparse(prepo, "HEAD:REQUIRE") == nothing && isfile(pkgreq) && return pkgreq + head = LibGit2.ref_id(LibGit2.head(repo)) + LibGit2.free!(repo) for (ver,info) in avail if head == info.sha1 return joinpath("METADATA", pkg, "versions", string(ver), "requires") @@ -158,10 +169,11 @@ function installed(avail::Dict=available()) for pkg in readdir() isinstalled(pkg) || continue ap = get(avail,pkg,Dict{VersionNumber,Available}()) - prepo = LibGit.Repo(pkg) + prepo = LibGit2.GitRepo(pkg) ver = installed_version(pkg, prepo, ap) fixed = isfixed(pkg, prepo, ap) pkgs[pkg] = (ver, fixed) + LibGit2.free!(prepo) end return pkgs end @@ -189,7 +201,7 @@ end function issue_url(pkg::AbstractString) ispath(pkg,".git") || return "" - m = match(LibGit.GITHUB_REGEX, url(pkg)) + m = match(LibGit2.GITHUB_REGEX, url(pkg)) m == nothing && return "" return "https://github.com/" * m.captures[1] * "/issues" end From cdf28d4775cd0c11e75f3304d55dafb7c634b106 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Thu, 7 May 2015 14:59:17 -0400 Subject: [PATCH 0281/1938] fixed to AbstractString --- base/pkg/github.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/pkg/github.jl b/base/pkg/github.jl index 161324c8dd6c0..71580ec076514 100644 --- a/base/pkg/github.jl +++ b/base/pkg/github.jl @@ -12,7 +12,7 @@ const AUTH_DATA = Dict{Any,Any}( ) function user() - usr = LibGit2.lookup(String, LibGit2.GitConfig(), "github.user") + usr = LibGit2.lookup(AbstractString, LibGit2.GitConfig(), "github.user") if usr == nothing error(""" no GitHub user name configured; please configure it with: From c57ec723c04d0e8f7b3d74b6e78b43e213a44227 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Thu, 7 May 2015 21:49:06 -0400 Subject: [PATCH 0282/1938] added libgit2 error handling --- base/pkg/libgit2/const.jl | 21 +-------- base/pkg/libgit2/error.jl | 96 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 20 deletions(-) create mode 100644 base/pkg/libgit2/error.jl diff --git a/base/pkg/libgit2/const.jl b/base/pkg/libgit2/const.jl index 876cd98149dbc..43f442410d469 100644 --- a/base/pkg/libgit2/const.jl +++ b/base/pkg/libgit2/const.jl @@ -47,7 +47,7 @@ module GitConst const CHECKOUT_UPDATE_SUBMODULES = Cuint(1) << Cint(16) const CHECKOUT_UPDATE_SUBMODULES_IF_CHANGED = Cuint(1) << Cint(17) - const CHECKOUT_NOTIFY_NONE = Cint(0) + const CHECKOUT_NOTIFY_NONE = Cuint(0) const CHECKOUT_NOTIFY_CONFLICT = Cuint(1) << Cint(0) const CHECKOUT_NOTIFY_DIRTY = Cuint(1) << Cint(1) const CHECKOUT_NOTIFY_UPDATED = Cuint(1) << Cint(2) @@ -207,23 +207,4 @@ module GitConst const GIT_REPOSITORY_STATE_REBASE_MERGE = Cint(7) const GIT_REPOSITORY_STATE_APPLY_MAILBOX = Cint(8) const GIT_REPOSITORY_STATE_APPLY_MAILBOX_OR_REBASE = Cint(9) -end - -module GitErrorConst - const GIT_OK = Cint(0) - const ERROR = Cint(-01) - const ENOTFOUND = Cint(-03) - const EEXISTS = Cint(-04) - const EAMBIGUOUS = Cint(-05) - const EBUFS = Cint(-06) - const EUSER = Cint(-07) - const EBAREREPO = Cint(-08) - const EUNBORNBRANCH = Cint(-09) - const EUNMERGED = Cint(-10) - const ENONFASTFORWARD = Cint(-11) - const EINVALIDSPEC = Cint(-12) - const EMERGECONFLICT = Cint(-13) - const ELOCKED = Cint(-14) - const PASSTHROUGH = Cint(-30) - const ITEROVER = Cint(-31) end \ No newline at end of file diff --git a/base/pkg/libgit2/error.jl b/base/pkg/libgit2/error.jl new file mode 100644 index 0000000000000..f6ecb3f05449c --- /dev/null +++ b/base/pkg/libgit2/error.jl @@ -0,0 +1,96 @@ +module GitErrorConst + const GIT_OK = Cint(0) + const ERROR = Cint(-01) + const ENOTFOUND = Cint(-03) + const EEXISTS = Cint(-04) + const EAMBIGUOUS = Cint(-05) + const EBUFS = Cint(-06) + const EUSER = Cint(-07) + const EBAREREPO = Cint(-08) + const EUNBORNBRANCH = Cint(-09) + const EUNMERGED = Cint(-10) + const ENONFASTFORWARD = Cint(-11) + const EINVALIDSPEC = Cint(-12) + const EMERGECONFLICT = Cint(-13) + const ELOCKED = Cint(-14) + const PASSTHROUGH = Cint(-30) + const ITEROVER = Cint(-31) +end + +const git_error_code = Dict{Int,Symbol}( + 00 => :OK, # no error + -01 => :Error, # generic error + -03 => :NotFound, # requested object could not be found + -04 => :Exists, # object exits preventing op + -05 => :Ambiguous, # more than one object matches + -06 => :Bufs, # output buffer too small to hold data + -07 => :User, # user callback generated error + -08 => :BareRepo, # operation not allowed on bare repo + -09 => :UnbornBranch, # HEAD refers to branch with 0 commits + -10 => :Unmerged, # merge in progress prevented op + -11 => :NonFastForward, # ref not fast-forwardable + -12 => :InvalidSpec, # name / ref not in valid format + -13 => :MergeConflict, # merge conflict prevented op + -14 => :Locked, # lock file prevented op + -15 => :Modified, # ref value does not match expected + -31 => :Iterover # signals end of iteration +) + +const git_error_class = Dict{Int,Symbol}( + 0 => :None, + 1 => :NoMemory, + 2 => :OS, + 3 => :Invalid, + 4 => :Ref, + 5 => :Zlib, + 6 => :Repo, + 7 => :Config, + 8 => :Regex, + 9 => :Odb, + 10 => :Index, + 11 => :Object, + 12 => :Net, + 13 => :Tag, + 14 => :Tree, + 15 => :Indexer, + 16 => :SSL, + 17 => :Submodule, + 18 => :Thread, + 19 => :Stash, + 20 => :Checkout, + 21 => :FetchHead, + 22 => :Merge, + 23 => :SSH, + 24 => :Filter, + 25 => :Revert, + 26 => :Callback, + 27 => :CherryPick +) + +immutable ErrorStruct + message::Ptr{Uint8} + class::Cint +end + +immutable GitError{Class, Code} + msg::UTF8String +end + +function last_error() + err = ccall((:giterr_last, :libgit2), Ptr{ErrorStruct}, ()) + if err != C_NULL + err_obj = unsafe_load(err) + err_class = git_error_class[Int(err_obj.class)] + err_msg = bytestring(err_obj.message) + else + err_class = git_error_class[0] + err_msg = "No errors" + end + return (err_class, err_msg) +end + +GitError(code::Integer) = begin + err_code = git_error_code[Int(code)] + err_class, err_msg = last_error() + return GitError{err_class, err_code}(err_msg) +end \ No newline at end of file From 71401413f9a802d1f7f3705e65be8e7eba7cfaf5 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Thu, 7 May 2015 21:50:37 -0400 Subject: [PATCH 0283/1938] write to repo configuration value with any accepted type --- base/pkg/libgit2/config.jl | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/base/pkg/libgit2/config.jl b/base/pkg/libgit2/config.jl index 46a501b9da464..13ab1cc4df93f 100644 --- a/base/pkg/libgit2/config.jl +++ b/base/pkg/libgit2/config.jl @@ -27,7 +27,7 @@ end GitConfig(r::GitRepo) = begin cfg_ptr = Ptr{Void}[C_NULL] err = ccall((:git_repository_config, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}), cfg_ptr, r) + (Ptr{Ptr{Void}}, Ptr{Void}), cfg_ptr, r.ptr) err !=0 && return nothing return GitConfig(cfg_ptr[1]) end @@ -49,8 +49,23 @@ function lookup{T<:AbstractString}(::Type{T}, c::GitConfig, name::AbstractString end end -function set!{T<:AbstractString}(::Type{T}, c::GitConfig, name::AbstractString, value::AbstractString) - err = ccall((:git_config_set_string, :libgit2), Cint, +function set!{T}(c::GitConfig, name::AbstractString, value::T) + err = if T<:AbstractString + ccall((:git_config_set_string, :libgit2), Cint, (Ptr{Void}, Ptr{Uint8}, Ptr{Uint8}), c.ptr, name, value) + elseif is(T, Bool) + bval = Int32(value) + ccall((:git_config_set_bool, :libgit2), Cint, + (Ptr{Void}, Ptr{Uint8}, Cint), c.ptr, name, bval) + elseif is(T, Int32) + ccall((:git_config_set_int32, :libgit2), Cint, + (Ptr{Void}, Ptr{Uint8}, Cint), c.ptr, name, value) + elseif is(T, Int64) + ccall((:git_config_set_int64, :libgit2), Cint, + (Ptr{Void}, Ptr{Uint8}, Cintmax_t), c.ptr, name, value) + else + warn("Type $T is not supported") + end return err end + From f836a71a6854c70a55cd6f342212cbc81e3fb29e Mon Sep 17 00:00:00 2001 From: Art Wild Date: Thu, 7 May 2015 21:53:37 -0400 Subject: [PATCH 0284/1938] added `clone` & `fetch`, updated 'Dir' and `Cache` --- base/pkg/cache.jl | 23 ++++---- base/pkg/dir.jl | 5 +- base/pkg/libgit2.jl | 48 +++++++++++++--- base/pkg/libgit2/clone.jl | 114 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 169 insertions(+), 21 deletions(-) create mode 100644 base/pkg/libgit2/clone.jl diff --git a/base/pkg/cache.jl b/base/pkg/cache.jl index 22107f430e70c..83d2fd761289b 100644 --- a/base/pkg/cache.jl +++ b/base/pkg/cache.jl @@ -2,7 +2,7 @@ module Cache -import ..Git, ..Dir +import ..LibGit2, ..Dir using ..Types path(pkg::AbstractString) = abspath(".cache", pkg) @@ -28,27 +28,26 @@ function mkcachedir() mkdir(cache) end - function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) isdir(".cache") || mkcachedir() cache = path(pkg) if !isdir(cache) info("Cloning cache of $pkg from $url") - try Git.run(`clone -q --mirror $url $cache`) - catch + repo = LibGit2.clone(url, path, bare = true, remote_cb = Pkg.LibGit2.mirror_cb) + if repo == noting rm(cache, recursive=true) - rethrow() + error("Cannot clone $pkg from $url") end + else + repo = LibGit2.GitRepo(cache) end - Git.set_remote_url(url, dir=cache) - in_cache = Git.iscommit(sha1s, dir=cache) - if !all(in_cache) + LibGit2.set_remote_url(repo, url) + if !all(sha1->LibGit2.iscommit(sha1, repo), sha1s) info("Updating cache of $pkg...") - Git.success(`remote update`, dir=cache) || - error("couldn't update $cache using `git remote update`") - in_cache = Git.iscommit(sha1s, dir=cache) + isa(LibGit2.fetch(repo), LibGit2.GitError) && + error("couldn't update $cache using `git remote update`") end - sha1s[!in_cache] + filter(sha1->!LibGit2.iscommit(sha1, repo), sha1s) end prefetch(pkg::AbstractString, url::AbstractString, sha1::AbstractString...) = prefetch(pkg, url, AbstractString[sha1...]) diff --git a/base/pkg/dir.jl b/base/pkg/dir.jl index b6a186297499c..c8f573d2adb8f 100644 --- a/base/pkg/dir.jl +++ b/base/pkg/dir.jl @@ -46,8 +46,9 @@ function init(meta::AbstractString=DEFAULT_META, branch::AbstractString=META_BRA temp_dir = mktempdir(dir) Base.cd(temp_dir) do info("Cloning METADATA from $meta") - run(`git clone -q -b $branch $meta METADATA`) #TODO: LibGit2.clone - LibGit2.set_remote_url("METADATA", meta) + metadata_repo = LibGit2.clone(meta, "METADATA", branch = branch) + LibGit2.set_remote_url(metadata_repo, meta) + LibGit2.free!(metadata_repo) touch("REQUIRE") touch("META_BRANCH") open("META_BRANCH", "w") do io diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 34db174596e58..c3cf61691e678 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -13,12 +13,14 @@ function __init__() end include("libgit2/const.jl") +include("libgit2/error.jl") include("libgit2/repository.jl") include("libgit2/config.jl") +include("libgit2/clone.jl") -type Ref +type GitRef ptr::Ptr{Void} - function Ref(ptr::Ptr{Void}) + function GitRef(ptr::Ptr{Void}) r = new(ptr) finalizer(r, r -> ccall((:git_reference_free, :libgit2), Void, (Ptr{Void},), r.ptr)) return r @@ -40,10 +42,10 @@ function head(repo::GitRepo) (Ptr{Ptr{Void}}, Ptr{Void}), head_ptr, repo.ptr) (err != 0) && return nothing - return Ref(head_ptr[1]) + return GitRef(head_ptr[1]) end -function ref_id(ref::Ref) +function ref_id(ref::GitRef) ref == nothing && return "" typ = ccall((:git_reference_type, :libgit2), Cint, (Ptr{Void},), ref.ptr) @@ -60,7 +62,7 @@ end head_oid(repo::GitRepo) = head(repo) |> ref_id -function ref_name(ref::Ref) +function ref_name(ref::GitRef) ref == nothing && return "" name_ptr = ccall((:git_reference_shorthand, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) @@ -153,14 +155,14 @@ end function set_remote_url(repo::GitRepo, url::AbstractString; remote::AbstractString="origin") cfg = GitConfig(repo) - err = set_config_string(cfg_ptr[1], "remote.$remote.url", url) + err = set!(AbstractString, cfg, "remote.$remote.url", url) err !=0 && return m = match(GITHUB_REGEX,url) m == nothing && return push = "git@github.com:$(m.captures[1]).git" if push != url - err = set_config_string(cfg_ptr[1], "remote.$remote.pushurl", push) + err = set!(AbstractString, cfg, "remote.$remote.pushurl", push) end end @@ -170,6 +172,38 @@ function set_remote_url(path::AbstractString, url::AbstractString; remote::Abstr LibGit2.free!(prepo) end +function mirror_callback(remote::Ptr{Ptr{Void}}, repo::Ptr{Void}, name::Ptr{UInt8}, url::Ptr{UInt8}, payload::Ptr{Void}) + # Create the remote with a mirroring url + fetch_spec = "+refs/*:refs/*" + err = ccall((:git_remote_create_with_fetchspec, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}, Ptr{UInt8}), + remote, repo, name, url, fetch_spec) + err != 0 && return Cint(err) + + # And set the configuration option to true for the push command + config = GitConfig(GitRepo(repo, false)) + name_str = bytestring(name) + err = set!(config, "remote.$name_str.mirror", true) + free!(config) + err != 0 && return Cint(err) + + return Cint(0) +end +const mirror_cb = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}, Ptr{Void})) + +function fetch(repo::GitRepo, remote::AbstractString="origin") + remote_ptr = [C_NULL] + err = ccall((:git_remote_lookup, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), + remote_ptr, repo.ptr, remote) + err != 0 && return GitError(err) + + err = ccall((:git_remote_fetch, :libgit2), Cint, + (Ptr{Void}, Ptr{Void}, Ptr{UInt8}), + remote_ptr[1], C_NULL, C_NULL) + err != 0 && return GitError(err) +end + function normalize_url(url::AbstractString) m = match(GITHUB_REGEX,url) m == nothing ? url : "git://github.com/$(m.captures[1]).git" diff --git a/base/pkg/libgit2/clone.jl b/base/pkg/libgit2/clone.jl new file mode 100644 index 0000000000000..0a02d2dd73f18 --- /dev/null +++ b/base/pkg/libgit2/clone.jl @@ -0,0 +1,114 @@ +immutable StrArrayStruct + strings::Ptr{Ptr{Uint8}} + count::Csize_t +end +StrArrayStruct() = StrArrayStruct(Ptr{Uint8}(0), zero(Csize_t)) + +immutable CheckoutOptionsStruct + version::Cuint + checkout_strategy::Cuint + disable_filters::Cint + dir_mode::Cuint + file_mode::Cuint + file_open_flags::Cint + notify_flags::Cuint + notify_cb::Ptr{Void} + notify_payload::Ptr{Void} + progress_cb::Ptr{Void} + progress_payload::Ptr{Void} + paths::StrArrayStruct + baseline::Ptr{Void} + target_directory::Ptr{Uint8} + ancestor_label::Ptr{Uint8} + our_label::Ptr{Uint8} + their_label::Ptr{Uint8} +end +CheckoutOptionsStruct() = CheckoutOptionsStruct(one(Cuint), + GitConst.CHECKOUT_SAFE_CREATE, + zero(Cint), + zero(Cuint), # Cuint(0o755), # + zero(Cuint), # Cuint(0o755), # + zero(Cint), + GitConst.CHECKOUT_NOTIFY_NONE, + Ptr{Void}(0), Ptr{Void}(0), + Ptr{Void}(0), Ptr{Void}(0), + StrArrayStruct(), + Ptr{Void}(0), + Ptr{Uint8}(0), + Ptr{Uint8}(0), + Ptr{Uint8}(0), + Ptr{Uint8}(0)) + +immutable RemoteCallbacksStruct + version::Cuint + sideband_progress::Ptr{Void} + completion::Ptr{Void} + credentials::Ptr{Void} + certificate_check::Ptr{Void} + transfer_progress::Ptr{Void} + update_tips::Ptr{Void} + pack_progress::Ptr{Void} + push_transfer_progress::Ptr{Void} + push_update_reference::Ptr{Void} + payload::Ptr{Void} +end +RemoteCallbacksStruct() = RemoteCallbacksStruct(one(Cuint), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0)) + +type CloneOptionsStruct + version::Cuint + checkout_opts::CheckoutOptionsStruct + remote_callbacks::RemoteCallbacksStruct + bare::Cint + localclone::Cint + checkout_branch::Ptr{Uint8} + signature::Ptr{Void} + repository_cb::Ptr{Void} + repository_cb_payload::Ptr{Void} + remote_cb::Ptr{Void} + remote_cb_payload::Ptr{Void} +end + +CloneOptionsStruct() = CloneOptionsStruct(one(Cuint), + CheckoutOptionsStruct(), + RemoteCallbacksStruct(), + zero(Cint), + zero(Cint), + Ptr{UInt8}(0), + Ptr{Void}(0), + Ptr{Void}(0), Ptr{Void}(0), + Ptr{Void}(0), Ptr{Void}(0) + ) + +function clone(url::AbstractString, path::AbstractString; + branch::AbstractString="", + bare::Bool = false, + remote_cb::Ptr{Void} = C_NULL) + # start cloning + clone_opts = CloneOptionsStruct() + clone_opts.bare = Int32(bare) + if !isempty(branch) + clone_opts.checkout_branch = pointer(branch) + end + if remote_cb != C_NULL + clone_opts.remote_cb = remote_cb + end + + clone_opts_ref = Ref(clone_opts) + repo_ptr = Ptr{Void}[C_NULL] + err = ccall((:git_clone, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{UInt8}, Ptr{UInt8}, Ref{CloneOptionsStruct}), + repo_ptr, url, path, clone_opts_ref) + err != 0 && return nothing + + return GitRepo(repo_ptr[1]) +end From 61ba8f070aa1be58211d245d491989c16c1c7401 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Fri, 8 May 2015 01:54:24 -0400 Subject: [PATCH 0285/1938] switching to `Ref` type --- base/pkg/libgit2/config.jl | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/base/pkg/libgit2/config.jl b/base/pkg/libgit2/config.jl index 13ab1cc4df93f..fe31440dccca9 100644 --- a/base/pkg/libgit2/config.jl +++ b/base/pkg/libgit2/config.jl @@ -16,34 +16,35 @@ function free!(cfg::GitConfig) end end -GitConfig(path::AbstractString) = begin - cfg_ptr = Ptr{Void}[C_NULL] +function GitConfig(path::AbstractString) + cfg_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_config_open_ondisk, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Uint8}), cfg_ptr, path) + (Ptr{Ptr{Void}}, Ptr{Uint8}), cfg_ptr_ptr, path) err !=0 && return nothing - return GitConfig(cfg_ptr[1]) + return GitConfig(cfg_ptr_ptr[]) end -GitConfig(r::GitRepo) = begin - cfg_ptr = Ptr{Void}[C_NULL] +function GitConfig(r::GitRepo) + cfg_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_repository_config, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}), cfg_ptr, r.ptr) + (Ptr{Ptr{Void}}, Ptr{Void}), cfg_ptr_ptr, r.ptr) err !=0 && return nothing - return GitConfig(cfg_ptr[1]) + return GitConfig(cfg_ptr_ptr[]) end -GitConfig() = begin - cfg_ptr = Ptr{Void}[C_NULL] - ccall((:git_config_open_default, :libgit2), Cint, (Ptr{Ptr{Void}}, ), cfg_ptr) - return GitConfig(cfg_ptr[1]) +function GitConfig() + cfg_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + ccall((:git_config_open_default, :libgit2), Cint, + (Ptr{Ptr{Void}}, ), cfg_ptr_ptr) + return GitConfig(cfg_ptr_ptr[]) end function lookup{T<:AbstractString}(::Type{T}, c::GitConfig, name::AbstractString) - out = Ptr{Uint8}[0] + str_ptr = Ref{Ptr{Uint8}}(C_NULL) err = ccall((:git_config_get_string, :libgit2), Cint, - (Ptr{Ptr{Uint8}}, Ptr{Void}, Ptr{Uint8}), out, c.ptr, name) + (Ptr{Ptr{Uint8}}, Ptr{Void}, Ptr{Uint8}), str_ptr, c.ptr, name) if err == GitErrorConst.GIT_OK - return bytestring(out[1]) + return bytestring(str_ptr[]) else return nothing end From ba591fec3d353d97c39e79ffe2df1a3705c38a85 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Fri, 8 May 2015 02:33:27 -0400 Subject: [PATCH 0286/1938] changed incorrect types & syntax --- base/pkg/libgit2/clone.jl | 22 +++++++++++----------- base/pkg/libgit2/error.jl | 6 +++--- base/pkg/libgit2/repository.jl | 6 +++--- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/base/pkg/libgit2/clone.jl b/base/pkg/libgit2/clone.jl index 0a02d2dd73f18..44ed42a84da6d 100644 --- a/base/pkg/libgit2/clone.jl +++ b/base/pkg/libgit2/clone.jl @@ -1,8 +1,8 @@ immutable StrArrayStruct - strings::Ptr{Ptr{Uint8}} + strings::Ptr{Ptr{UInt8}} count::Csize_t end -StrArrayStruct() = StrArrayStruct(Ptr{Uint8}(0), zero(Csize_t)) +StrArrayStruct() = StrArrayStruct(Ptr{UInt8}(0), zero(Csize_t)) immutable CheckoutOptionsStruct version::Cuint @@ -18,10 +18,10 @@ immutable CheckoutOptionsStruct progress_payload::Ptr{Void} paths::StrArrayStruct baseline::Ptr{Void} - target_directory::Ptr{Uint8} - ancestor_label::Ptr{Uint8} - our_label::Ptr{Uint8} - their_label::Ptr{Uint8} + target_directory::Ptr{UInt8} + ancestor_label::Ptr{UInt8} + our_label::Ptr{UInt8} + their_label::Ptr{UInt8} end CheckoutOptionsStruct() = CheckoutOptionsStruct(one(Cuint), GitConst.CHECKOUT_SAFE_CREATE, @@ -34,10 +34,10 @@ CheckoutOptionsStruct() = CheckoutOptionsStruct(one(Cuint), Ptr{Void}(0), Ptr{Void}(0), StrArrayStruct(), Ptr{Void}(0), - Ptr{Uint8}(0), - Ptr{Uint8}(0), - Ptr{Uint8}(0), - Ptr{Uint8}(0)) + Ptr{UInt8}(0), + Ptr{UInt8}(0), + Ptr{UInt8}(0), + Ptr{UInt8}(0)) immutable RemoteCallbacksStruct version::Cuint @@ -70,7 +70,7 @@ type CloneOptionsStruct remote_callbacks::RemoteCallbacksStruct bare::Cint localclone::Cint - checkout_branch::Ptr{Uint8} + checkout_branch::Ptr{UInt8} signature::Ptr{Void} repository_cb::Ptr{Void} repository_cb_payload::Ptr{Void} diff --git a/base/pkg/libgit2/error.jl b/base/pkg/libgit2/error.jl index f6ecb3f05449c..bec7ad2d8645f 100644 --- a/base/pkg/libgit2/error.jl +++ b/base/pkg/libgit2/error.jl @@ -68,12 +68,12 @@ const git_error_class = Dict{Int,Symbol}( ) immutable ErrorStruct - message::Ptr{Uint8} + message::Ptr{UInt8} class::Cint end immutable GitError{Class, Code} - msg::UTF8String + msg::AbstractString end function last_error() @@ -89,7 +89,7 @@ function last_error() return (err_class, err_msg) end -GitError(code::Integer) = begin +function GitError(code::Integer) err_code = git_error_code[Int(code)] err_class, err_msg = last_error() return GitError{err_class, err_code}(err_msg) diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index d7ff8c0809555..8bf66caf50f95 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -9,7 +9,7 @@ type GitRepo end end -GitRepo(path::String) = begin +function GitRepo(path::AbstractString) repo_ptr = Ptr{Void}[0] err = ccall((:git_repository_open, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Uint8}), repo_ptr, path) @@ -22,13 +22,13 @@ GitRepo(path::String) = begin return GitRepo(repo_ptr[1]) end -close(r::GitRepo) = begin +function close(r::GitRepo) if r.ptr != C_NULL ccall((:git_repository__cleanup, :libgit2), Void, (Ptr{Void},), r.ptr) end end -free!(r::GitRepo) = begin +function free!(r::GitRepo) if r.ptr != C_NULL ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), r.ptr) r.ptr = C_NULL From efb5bbab87ac8601fc9f9b4ad13ef533872cac71 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Fri, 8 May 2015 22:05:35 -0400 Subject: [PATCH 0287/1938] working on `Generate` added multiples types, refactored files in `libgit2` it builds!!! --- base/pkg/generate.jl | 23 ++-- base/pkg/github.jl | 2 +- base/pkg/libgit2.jl | 141 +++++++++--------------- base/pkg/libgit2/commit.jl | 21 ++++ base/pkg/libgit2/config.jl | 55 +++++---- base/pkg/libgit2/oid.jl | 107 ++++++++++++++++++ base/pkg/libgit2/reference.jl | 17 +++ base/pkg/libgit2/repository.jl | 64 ++++++++--- base/pkg/libgit2/signature.jl | 19 ++++ base/pkg/libgit2/{clone.jl => types.jl} | 136 ++++++++++++++++++++--- base/pkg/libgit2/walker.jl | 64 +++++++++++ base/pkg/read.jl | 6 +- 12 files changed, 485 insertions(+), 170 deletions(-) create mode 100644 base/pkg/libgit2/commit.jl create mode 100644 base/pkg/libgit2/oid.jl create mode 100644 base/pkg/libgit2/reference.jl create mode 100644 base/pkg/libgit2/signature.jl rename base/pkg/libgit2/{clone.jl => types.jl} (58%) create mode 100644 base/pkg/libgit2/walker.jl diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index 3076878bdf642..1323fad7fedb8 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -2,25 +2,24 @@ module Generate -import ..Git, ..Read +import ..Git, ..LibGit2, ..Read -copyright_year() = readchomp(`date +%Y`) -copyright_name(dir::AbstractString) = readchomp(Git.cmd(`config --get user.name`, dir=dir)) -github_user() = readchomp(ignorestatus(`git config --global --get github.user`)) +copyright_year() = Dates.year(Dates.today()) +copyright_name(dir::AbstractString) = LibGit2.get(AbstractString, LibGit2.GitConfig(LibGit2.GitRepo(dir)), "user.name") +github_user() = LibGit2.get(AbstractString, LibGit2.GitConfig(), "github.user") function git_contributors(dir::AbstractString, n::Int=typemax(Int)) contrib = Dict() - tty = @windows? "CON:" : "/dev/tty" - for line in eachline(pipeline(tty, Git.cmd(`shortlog -nes`, dir=dir))) - m = match(r"\s*(\d+)\s+(.+?)\s+\<(.+?)\>\s*$", line) - m === nothing && continue - commits, name, email = m.captures - if haskey(contrib,email) - contrib[email][1] += parse(Int,commits) + repo = LibGit2.GitRepo(dir) + for sig in LibGit2.authors(repo) + if haskey(contrib, sig.email) + contrib[sig.email][1] += 1 else - contrib[email] = [parse(Int,commits), name] + contrib[sig.email] = [1, sig.name] end end + LibGit2.free!(repo) + names = Dict() for (commits,name) in values(contrib) names[name] = get(names,name,0) + commits diff --git a/base/pkg/github.jl b/base/pkg/github.jl index 71580ec076514..3ca072eba5129 100644 --- a/base/pkg/github.jl +++ b/base/pkg/github.jl @@ -12,7 +12,7 @@ const AUTH_DATA = Dict{Any,Any}( ) function user() - usr = LibGit2.lookup(AbstractString, LibGit2.GitConfig(), "github.user") + usr = LibGit2.get(AbstractString, LibGit2.GitConfig(), "github.user") if usr == nothing error(""" no GitHub user name configured; please configure it with: diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index c3cf61691e678..041bb25c1f623 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -13,87 +13,29 @@ function __init__() end include("libgit2/const.jl") +include("libgit2/types.jl") include("libgit2/error.jl") +include("libgit2/signature.jl") +include("libgit2/oid.jl") +include("libgit2/reference.jl") +include("libgit2/commit.jl") include("libgit2/repository.jl") include("libgit2/config.jl") -include("libgit2/clone.jl") - -type GitRef - ptr::Ptr{Void} - function GitRef(ptr::Ptr{Void}) - r = new(ptr) - finalizer(r, r -> ccall((:git_reference_free, :libgit2), Void, (Ptr{Void},), r.ptr)) - return r - end -end - -type Obj - ptr::Ptr{Void} - function Obj(ptr::Ptr{Void}) - r = new(ptr) - finalizer(r, r -> ccall((:git_object_free, :libgit2), Void, (Ptr{Void},), r.ptr)) - return r - end -end +include("libgit2/walker.jl") -function head(repo::GitRepo) - head_ptr = Ptr{Void}[0] - err = ccall((:git_repository_head, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}), head_ptr, repo.ptr) - - (err != 0) && return nothing - return GitRef(head_ptr[1]) -end - -function ref_id(ref::GitRef) - ref == nothing && return "" - - typ = ccall((:git_reference_type, :libgit2), Cint, (Ptr{Void},), ref.ptr) - if typ == 1 - oid_ptr = ccall((:git_reference_target, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) - oid_ptr == C_NULL && return "" - return bytes2hex(pointer_to_array(oid_ptr, 20)) - else - oid_ptr = ccall((:git_reference_symbolic_target, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) - oid_ptr == C_NULL && return "" - return bytestring(oid_ptr) +function need_update(repo::GitRepo) + if !isbare(repo) + "git update-index -q --really-refresh" #TODO: update-index end end -head_oid(repo::GitRepo) = head(repo) |> ref_id - -function ref_name(ref::GitRef) - ref == nothing && return "" - - name_ptr = ccall((:git_reference_shorthand, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) - name_ptr == C_NULL && return "" - return bytestring(name_ptr) -end - -function need_update(repo::GitRepo) - ccall((:git_repository_is_bare, :libgit2), Cint, (Ptr{Void},), repo.ptr) != 1 && "git update-index -q --really-refresh" +function isattached(repo::GitRepo) + ccall((:git_repository_head_detached, :libgit2), Cint, (Ptr{Void},), repo.ptr) != 1 end function iscommit(id::AbstractString, repo::GitRepo) need_update(repo) - - oid = hex2bytes(id) - cmt_ptr = Ptr{Void}[0] - err = ccall((:git_commit_lookup, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Uint8}), cmt_ptr, repo.ptr, oid) - if err != 0 - return false - else - ccall((:git_commit_free, :libgit2), Void, (Ptr{Void},), cmt_ptr[1]) - return true - end -end - -function obj_id(ref::Obj) - ref == nothing && return "" - oid_ptr = ccall((:git_object_id, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) - oid_ptr == C_NULL && return "" - return bytes2hex(pointer_to_array(oid_ptr, 20)) + return !isa(get(GitCommit, repo, id), GitError) end function isdirty(repo::GitRepo, paths::AbstractString="") @@ -102,7 +44,7 @@ function isdirty(repo::GitRepo, paths::AbstractString="") tree_ptr = Ptr{Void}[0] err = ccall((:git_tree_lookup, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Uint8}), tree_ptr, repo.ptr, tree_oid) + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), tree_ptr, repo.ptr, tree_oid) err != 0 && return true diff_ptr = Ptr{Void}[0] @@ -122,29 +64,15 @@ function isdirty(repo::GitRepo, paths::AbstractString="") return false end -function isattached(repo::GitRepo) - ccall((:git_repository_head_detached, :libgit2), Cint, (Ptr{Void},), repo.ptr) != 1 -end - function merge_base(one::AbstractString, two::AbstractString, repo::GitRepo) - oid1 = hex2bytes(one) - oid2 = hex2bytes(two) - moid = zeros(UInt8 ,20) + oid1_ptr = Ref(Oid(one)) + oid2_ptr = Ref(Oid(two)) + moid_ptr = Ref(Oid()) err = ccall((:git_merge_base, :libgit2), Cint, - (Ptr{UInt8}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}), moid, repo.ptr, oid1, oid2) - if err != 0 - return "" - else - bytes2hex(moid) - end -end - -function revparse(repo::GitRepo, obj::AbstractString) - obj_ptr = Ptr{Void}[0] - err = ccall((:git_revparse_single, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Uint8}), obj_ptr, repo.ptr, obj) + (Ptr{Oid}, Ptr{Void}, Ptr{Oid}, Ptr{Oid}), + moid_ptr, repo.ptr, oid1_ptr, oid2_ptr) err != 0 && return nothing - return hex2bytes(obj_id(Obj(obj_ptr[1]))) + return moid[] end function is_ancestor_of(a::AbstractString, b::AbstractString, repo::GitRepo) @@ -204,6 +132,37 @@ function fetch(repo::GitRepo, remote::AbstractString="origin") err != 0 && return GitError(err) end +function clone(url::AbstractString, path::AbstractString; + branch::AbstractString="", + bare::Bool = false, + remote_cb::Ptr{Void} = C_NULL) + # start cloning + clone_opts = CloneOptionsStruct() + clone_opts.bare = Int32(bare) + if !isempty(branch) + clone_opts.checkout_branch = pointer(branch) + end + if remote_cb != C_NULL + clone_opts.remote_cb = remote_cb + end + + clone_opts_ref = Ref(clone_opts) + repo_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + err = ccall((:git_clone, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{UInt8}, Ptr{UInt8}, Ref{CloneOptionsStruct}), + repo_ptr_ptr, url, path, clone_opts_ref) + err != 0 && return nothing + + return GitRepo(repo_ptr_ptr[]) +end + +function authors(repo::GitRepo) + athrs = Pkg.LibGit2.map( + (oid,repo)->author(get(GitCommit, repo, oid)), + repo) #, by = Pkg.LibGit2.GitConst.SORT_TIME) + return athrs +end + function normalize_url(url::AbstractString) m = match(GITHUB_REGEX,url) m == nothing ? url : "git://github.com/$(m.captures[1]).git" diff --git a/base/pkg/libgit2/commit.jl b/base/pkg/libgit2/commit.jl new file mode 100644 index 0000000000000..8b175d6a75563 --- /dev/null +++ b/base/pkg/libgit2/commit.jl @@ -0,0 +1,21 @@ +function message(c::GitCommit, raw::Bool=false) + local msg_ptr::Ptr{UInt8} + msg_ptr = raw? ccall((:git_commit_message_raw, :libgit2), Ptr{UInt8}, (Ptr{Void},), c.ptr) : + ccall((:git_commit_message, :libgit2), Ptr{UInt8}, (Ptr{Void},), c.ptr) + if msg_ptr == C_NULL + return nothing + end + return bytestring(msg_ptr) +end + +function author(c::GitCommit) + ptr = ccall((:git_commit_author, :libgit2), Ptr{SignatureStruct}, (Ptr{Void},), c.ptr) + @assert ptr != C_NULL + return Signature(ptr) +end + +function committer(c::GitCommit) + ptr = ccall((:git_commit_committer, :libgit2), Ptr{SignatureStruct}, (Ptr{Void},), c.ptr) + @assert ptr != C_NULL + return Signature(ptr) +end \ No newline at end of file diff --git a/base/pkg/libgit2/config.jl b/base/pkg/libgit2/config.jl index fe31440dccca9..b125d08c1c944 100644 --- a/base/pkg/libgit2/config.jl +++ b/base/pkg/libgit2/config.jl @@ -1,25 +1,7 @@ -type GitConfig - ptr::Ptr{Void} - - function GitConfig(ptr::Ptr{Void}) - @assert ptr != C_NULL - cfg = new(ptr) - finalizer(cfg, free!) - return cfg - end -end - -function free!(cfg::GitConfig) - if cfg.ptr != C_NULL - ccall((:git_config_free, :libgit2), Void, (Ptr{Void},), cfg.ptr) - cfg.ptr = C_NULL - end -end - function GitConfig(path::AbstractString) cfg_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_config_open_ondisk, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Uint8}), cfg_ptr_ptr, path) + (Ptr{Ptr{Void}}, Ptr{UInt8}), cfg_ptr_ptr, path) err !=0 && return nothing return GitConfig(cfg_ptr_ptr[]) end @@ -39,12 +21,27 @@ function GitConfig() return GitConfig(cfg_ptr_ptr[]) end -function lookup{T<:AbstractString}(::Type{T}, c::GitConfig, name::AbstractString) - str_ptr = Ref{Ptr{Uint8}}(C_NULL) - err = ccall((:git_config_get_string, :libgit2), Cint, - (Ptr{Ptr{Uint8}}, Ptr{Void}, Ptr{Uint8}), str_ptr, c.ptr, name) - if err == GitErrorConst.GIT_OK - return bytestring(str_ptr[]) +function get{T}(::Type{T}, c::GitConfig, name::AbstractString) + if T<:AbstractString + str_ptr = Ref{Ptr{UInt8}}(C_NULL) + err = ccall((:git_config_get_string, :libgit2), Cint, + (Ptr{Ptr{UInt8}}, Ptr{Void}, Ptr{UInt8}), str_ptr, c.ptr, name) + err == GitErrorConst.GIT_OK && return bytestring(str_ptr[]) + elseif is(T, Bool) + val_ptr = Ref(Cint(0)) + ccall((:git_config_get_bool, :libgit2), Cint, + (Ptr{Cint}, Ptr{Void}, Ptr{UInt8}), val_ptr, c.ptr, name) + err == GitErrorConst.GIT_OK && return Bool(val_ptr[]) + elseif is(T, Int32) + val_ptr = Ref(Cint(0)) + ccall((:git_config_get_bool, :libgit2), Cint, + (Ptr{Cint}, Ptr{Void}, Ptr{UInt8}), val_ptr, c.ptr, name) + err == GitErrorConst.GIT_OK && return val_ptr[] + elseif is(T, Int64) + val_ptr = Ref(Cintmax_t(0)) + ccall((:git_config_get_bool, :libgit2), Cint, + (Ptr{Cintmax_t}, Ptr{Void}, Ptr{UInt8}), val_ptr, c.ptr, name) + err == GitErrorConst.GIT_OK && return Bool(val_ptr[]) else return nothing end @@ -53,17 +50,17 @@ end function set!{T}(c::GitConfig, name::AbstractString, value::T) err = if T<:AbstractString ccall((:git_config_set_string, :libgit2), Cint, - (Ptr{Void}, Ptr{Uint8}, Ptr{Uint8}), c.ptr, name, value) + (Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}), c.ptr, name, value) elseif is(T, Bool) bval = Int32(value) ccall((:git_config_set_bool, :libgit2), Cint, - (Ptr{Void}, Ptr{Uint8}, Cint), c.ptr, name, bval) + (Ptr{Void}, Ptr{UInt8}, Cint), c.ptr, name, bval) elseif is(T, Int32) ccall((:git_config_set_int32, :libgit2), Cint, - (Ptr{Void}, Ptr{Uint8}, Cint), c.ptr, name, value) + (Ptr{Void}, Ptr{UInt8}, Cint), c.ptr, name, value) elseif is(T, Int64) ccall((:git_config_set_int64, :libgit2), Cint, - (Ptr{Void}, Ptr{Uint8}, Cintmax_t), c.ptr, name, value) + (Ptr{Void}, Ptr{UInt8}, Cintmax_t), c.ptr, name, value) else warn("Type $T is not supported") end diff --git a/base/pkg/libgit2/oid.jl b/base/pkg/libgit2/oid.jl new file mode 100644 index 0000000000000..6f45eb16b2452 --- /dev/null +++ b/base/pkg/libgit2/oid.jl @@ -0,0 +1,107 @@ +const OID_RAWSZ = 20 +const OID_HEXSZ = OID_RAWSZ * 2 +const OID_MINPREFIXLEN = 4 + +# immutable Oid +# id1::UInt8 +# id2::UInt8 +# ... +# id20::UInt8 +# end +@eval begin + $(Expr(:type, false, :Oid, + Expr(:block, + [Expr(:(::), symbol("id$i"), :UInt8) for i=1:OID_RAWSZ]...))) +end + +# default Oid constructor (all zeros) +@generated function Oid() + return Expr(:call, :Oid, [:(zero(UInt8)) for _=1:OID_RAWSZ]...) +end +Oid(id::Oid) = id +Oid(ptr::Ptr{Oid}) = unsafe_load(ptr)::Oid + +function Oid(ptr::Ptr{UInt8}) + if ptr == C_NULL + throw(ArgumentError("NULL pointer passed to Oid() constructor")) + end + oid_ptr = Ref(Oid()) + ccall((:git_oid_fromraw, :libgit2), Void, (Ptr{Oid}, Ptr{UInt8}), oid_ptr, ptr) + return oid_ptr[] +end + +function Oid(id::Array{UInt8,1}) + if length(id) != OID_RAWSZ + throw(ArgumentError("invalid raw buffer size")) + end + return Oid(pointer(id)) +end + +function Oid(id::AbstractString) + bstr = bytestring(id) + len = sizeof(bstr) + oid_ptr = Ref(Oid()) + err = if len < OID_HEXSZ + ccall((:git_oid_fromstrn, :libgit2), Cint, + (Ptr{Oid}, Ptr{UInt8}, Csize_t), oid_ptr, bstr, len) + else + ccall((:git_oid_fromstrp, :libgit2), Cint, + (Ptr{Oid}, Ptr{Cchar}), oid_ptr, bstr) + end + err != 0 && return nothing + return oid_ptr[] +end + +function Oid(ref::GitReference) + ref == nothing && return Oid() + + typ = ccall((:git_reference_type, :libgit2), Cint, (Ptr{Void},), ref.ptr) + typ != 1 && return Oid() + oid_ptr = ccall((:git_reference_target, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) + oid_ptr == C_NULL && return "" + return Oid(oid_ptr) +end + +function Oid(obj::Ptr{Void}) + oid_ptr = ccall((:git_object_id, :libgit2), Ptr{UInt8}, (Ptr{Void},), obj) + oid_ptr == C_NULL && return "" + return Oid(oid_ptr) +end + +function Oid(obj::GitObject) + obj == nothing && return Oid() + return Oid(obj.ptr) +end + +function hex(id::Oid) + _hexstr = zeros(UInt8, OID_HEXSZ) + ccall((:git_oid_nfmt, :libgit2), Void, + (Ptr{UInt8}, Csize_t, Ptr{Oid}), Ref(_hexstr), OID_HEXSZ, Ref(id)) + return bytestring(_hexstr) +end + +raw(id::Oid) = hex2bytes(hex(id)) + +Base.string(id::Oid) = hex(id) + +Base.show(io::IO, id::Oid) = print(io, "Oid($(string(id)))") + +hash(id::Oid) = hash(hex(id)) + +cmp(id1::Oid, id2::Oid) = int(ccall((:git_oid_cmp, :libgit2), Cint, + (Ptr{Oid}, Ptr{Oid}), Ref(id1), Ref(id2))) + +Base.isequal(id1::Oid, id2::Oid) = cmp(id1, id2) == 0 +Base.isless(id1::Oid, id2::Oid) = cmp(id1, id2) < 0 + +iszero(id::Oid) = begin + bytes = raw(id) + for i=1:OID_RAWSZ + if bytes[i] != zero(UInt8) + return false + end + end + return true +end + +#randoid() = bytestring(rand("0123456789abcdef".data,OID_HEXSZ)) diff --git a/base/pkg/libgit2/reference.jl b/base/pkg/libgit2/reference.jl new file mode 100644 index 0000000000000..0560b9d4f0c4f --- /dev/null +++ b/base/pkg/libgit2/reference.jl @@ -0,0 +1,17 @@ +function shortname(ref::GitReference) + ref == nothing && return "" + + name_ptr = ccall((:git_reference_shorthand, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) + name_ptr == C_NULL && return "" + return bytestring(name_ptr) +end + +function fullname(ref::GitReference) + ref == nothing && return "" + + typ = ccall((:git_reference_type, :libgit2), Cint, (Ptr{Void},), ref.ptr) + typ == 1 && return "" + rname = ccall((:git_reference_symbolic_target, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) + rname == C_NULL && return "" + return bytestring(rname) +end \ No newline at end of file diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index 8bf66caf50f95..b795fdd4a454a 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -1,18 +1,7 @@ -type GitRepo - ptr::Ptr{Void} - - function GitRepo(ptr::Ptr{Void}, own::Bool=true) - @assert ptr != C_NULL - r = new(ptr) - own && finalizer(r, free!) - return r - end -end - function GitRepo(path::AbstractString) repo_ptr = Ptr{Void}[0] err = ccall((:git_repository_open, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Uint8}), repo_ptr, path) + (Ptr{Ptr{Void}}, Ptr{UInt8}), repo_ptr, path) if err != GitErrorConst.GIT_OK if repo_ptr[1] != C_NULL ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), repo_ptr[1]) @@ -28,9 +17,52 @@ function close(r::GitRepo) end end -function free!(r::GitRepo) - if r.ptr != C_NULL - ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), r.ptr) - r.ptr = C_NULL +function head(repo::GitRepo) + head_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + err = ccall((:git_repository_head, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}), head_ptr_ptr, repo.ptr) + (err != 0) && return nothing + return GitReference(head_ptr_ptr[]) +end +head_oid(repo::GitRepo) = Oid(head(repo)) + +function isbare(repo::GitRepo) + return ccall((:git_repository_is_bare, :libgit2), Cint, (Ptr{Void},), repo.ptr) == 1 +end + +function revparse(repo::GitRepo, obj::AbstractString) + obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + err = ccall((:git_revparse_single, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), obj_ptr_ptr, repo.ptr, obj) + err != 0 && return nothing + return Oid(obj_ptr_ptr[]) +end + +function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::Oid, prefix::Bool=false) + id_ptr = Ref(oid) + obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + + git_otype = if T == GitCommit + GitConst.OBJ_COMMIT + elseif T == GitTree + GitConst.OBJ_TREE + else + error("Type $T is not supported") end + + err = if prefix + ccall((:git_object_lookup_prefix, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}, Csize_t, Cint), + obj_ptr_ptr, r.ptr, id_ptr, len, git_otype) + else + ccall((:git_object_lookup, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}, Cint), + obj_ptr_ptr, r.ptr, id_ptr, git_otype) + end + err != 0 && return GitError(err) + return T(obj_ptr_ptr[]) +end + +function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::AbstractString) + return get(T, r, Oid(oid), length(oid) != OID_HEXSZ) end \ No newline at end of file diff --git a/base/pkg/libgit2/signature.jl b/base/pkg/libgit2/signature.jl new file mode 100644 index 0000000000000..6cb1f0c74dd60 --- /dev/null +++ b/base/pkg/libgit2/signature.jl @@ -0,0 +1,19 @@ +function Signature(ptr::Ptr{SignatureStruct}) + sig = unsafe_load(ptr)::SignatureStruct + name = bytestring(sig.name) + email = bytestring(sig.email) + time = sig.when.time + offset = sig.when.offset + return Signature(name, email, time, offset) +end + +function Signature(name::AbstractString, email::AbstractString) + sig_ptr_ptr = Ref{Ptr{SignatureStruct}}(C_NULL) + err = ccall((:git_signature_now, :libgit2), Cint, + (Ptr{Ptr{SignatureStruct}}, Ptr{UInt8}, Ptr{UInt8}), sig_ptr_ptr, name, email) + err != 0 && return GitError(err) + sig_ptr = sig_ptr_ptr[] + s = Signature(sig_ptr) + ccall((:git_signature_free, :libgit2), Void, (Ptr{SignatureStruct},), sig_ptr) + return s +end diff --git a/base/pkg/libgit2/clone.jl b/base/pkg/libgit2/types.jl similarity index 58% rename from base/pkg/libgit2/clone.jl rename to base/pkg/libgit2/types.jl index 44ed42a84da6d..d08726df1df7e 100644 --- a/base/pkg/libgit2/clone.jl +++ b/base/pkg/libgit2/types.jl @@ -1,3 +1,18 @@ +immutable TimeStruct + time::Int64 # time in seconds from epoch + offset::Cint # timezone offset in minutes +end +TimeStruct() = TimeStruct(zero(Int64), zero(Cint)) + +immutable SignatureStruct + name::Ptr{UInt8} # full name of the author + email::Ptr{UInt8} # email of the author + when::TimeStruct # time when the action happened +end +SignatureStruct() = SignatureStruct(Ptr{UInt8}(0), + Ptr{UInt8}(0), + TimeStruct()) + immutable StrArrayStruct strings::Ptr{Ptr{UInt8}} count::Csize_t @@ -89,26 +104,111 @@ CloneOptionsStruct() = CloneOptionsStruct(one(Cuint), Ptr{Void}(0), Ptr{Void}(0) ) -function clone(url::AbstractString, path::AbstractString; - branch::AbstractString="", - bare::Bool = false, - remote_cb::Ptr{Void} = C_NULL) - # start cloning - clone_opts = CloneOptionsStruct() - clone_opts.bare = Int32(bare) - if !isempty(branch) - clone_opts.checkout_branch = pointer(branch) +type GitRepo + ptr::Ptr{Void} + + function GitRepo(ptr::Ptr{Void}, own::Bool=true) + @assert ptr != C_NULL + r = new(ptr) + own && finalizer(r, free!) + return r + end +end + +function free!(r::GitRepo) + if r.ptr != C_NULL + ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), r.ptr) + r.ptr = C_NULL + end +end + +abstract GitObject + +function free!(o::GitObject) + if o.ptr != C_NULL + ccall((:git_object_free, :libgit2), Void, (Ptr{Void},), o.ptr) + o.ptr = C_NULL + end +end + +type GitCommit <: GitObject + ptr::Ptr{Void} + + function GitCommit(ptr::Ptr{Void}) + @assert ptr != C_NULL + this = new(ptr) + finalizer(this, free!) + return this + end +end + +type GitTree <: GitObject + ptr::Ptr{Void} + + function GitTree(ptr::Ptr{Void}) + @assert ptr != C_NULL + this = new(ptr) + finalizer(this, free!) + return this end - if remote_cb != C_NULL - clone_opts.remote_cb = remote_cb +end + +type GitReference + ptr::Ptr{Void} + + function GitReference(ptr::Ptr{Void}) + r = new(ptr) + finalizer(r, free!) + return r + end +end + +function free!(r::GitReference) + if r.ptr != C_NULL + ccall((:git_reference_free, :libgit2), Void, (Ptr{Void},), r.ptr) + r.ptr = C_NULL + end +end + +type GitConfig + ptr::Ptr{Void} + + function GitConfig(ptr::Ptr{Void}) + @assert ptr != C_NULL + cfg = new(ptr) + finalizer(cfg, free!) + return cfg end +end + +function free!(cfg::GitConfig) + if cfg.ptr != C_NULL + ccall((:git_config_free, :libgit2), Void, (Ptr{Void},), cfg.ptr) + cfg.ptr = C_NULL + end +end + +type GitRevWalker + ptr::Ptr{Void} - clone_opts_ref = Ref(clone_opts) - repo_ptr = Ptr{Void}[C_NULL] - err = ccall((:git_clone, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{UInt8}, Ptr{UInt8}, Ref{CloneOptionsStruct}), - repo_ptr, url, path, clone_opts_ref) - err != 0 && return nothing + function GitRevWalker(ptr::Ptr{Void}) + @assert ptr != C_NULL + w = new(ptr) + finalizer(w, free!) + return w + end +end + +function free!(w::GitRevWalker) + if w.ptr != C_NULL + ccall((:git_revwalk_free, :libgit2), Void, (Ptr{Void},), w.ptr) + w.ptr = C_NULL + end +end - return GitRepo(repo_ptr[1]) +type Signature + name::UTF8String + email::UTF8String + time::Int32 + time_offset::Int32 end diff --git a/base/pkg/libgit2/walker.jl b/base/pkg/libgit2/walker.jl new file mode 100644 index 0000000000000..dd7a26c5a3489 --- /dev/null +++ b/base/pkg/libgit2/walker.jl @@ -0,0 +1,64 @@ +function GitRevWalker(r::GitRepo) + w_ptr = Ref{Ptr{Void}}(C_NULL) + err = ccall((:git_revwalk_new, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}), w_ptr, r.ptr) + err != GitErrorConst.GIT_OK && return nothing + return GitRevWalker(w_ptr[]) +end + +function Base.start(w::GitRevWalker) + id_ptr = Ref(Oid()) + err = ccall((:git_revwalk_next, :libgit2), Cint, + (Ptr{Oid}, Ptr{Void}), id_ptr, w.ptr) + err != GitErrorConst.GIT_OK && return (nothing, true) + return (id_ptr[], false) +end + +Base.done(w::GitRevWalker, state) = Bool(state[2]) + +function Base.next(w::GitRevWalker, state) + id_ptr = Ref(Oid()) + err = ccall((:git_revwalk_next, :libgit2), Cint, + (Ptr{Oid}, Ptr{Void}), id_ptr, w.ptr) + err != GitErrorConst.GIT_OK && return (state[1], (nothing, true)) + return (state[1], (id_ptr[], false)) +end + +function push_head!(w::GitRevWalker) + err = ccall((:git_revwalk_push_head, :libgit2), Cint, (Ptr{Void},), w.ptr) + err != GitErrorConst.GIT_OK && return GitError(err) + return w +end + +function Base.push!(w::GitRevWalker, cid::Oid) + err = ccall((:git_revwalk_push, :libgit2), Cint, (Ptr{Void}, Ptr{Oid}), w.ptr, Ref(cid)) + err != GitErrorConst.GIT_OK && return GitError(err) + return w +end + +function Base.sort!(w::GitRevWalker; by::Cint = GitConst.SORT_NONE, rev::Bool=false) + rev && (by |= GitConst.SORT_REVERSE) + ccall((:git_revwalk_sorting, :libgit2), Void, (Ptr{Void}, Cint), w.ptr, by) + return w +end + +function Base.map(f::Function, repo::GitRepo; oid::Oid=Oid(), by::Cint = GitConst.SORT_NONE, rev::Bool=false) + walker = GitRevWalker(repo) + sort!(walker, by=by, rev=rev) + if iszero(oid) + push_head!(walker) + else + push!(walker, oid) + end + s = start(walker) + res = nothing + while !done(walker, s) + val = f(s[1], repo) + if res == nothing + res = Array(typeof(val),0) + end + push!(res, val) + val, s = next(walker, s) + end + return res +end \ No newline at end of file diff --git a/base/pkg/read.jl b/base/pkg/read.jl index 7061d54defc8a..b6d85d24eb2cc 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -64,7 +64,7 @@ function isfixed(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::Dict=availa LibGit2.isattached(prepo) && return true LibGit2.revparse(prepo, "HEAD:REQUIRE") == nothing && isfile(pkg,"REQUIRE") && return true - head = LibGit2.ref_id(LibGit2.head(prepo)) + head = string(LibGit2.head_oid(prepo)) for (ver,info) in avail head == info.sha1 && return false end @@ -100,7 +100,7 @@ function installed_version(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::D ispath(pkg,".git") || return typemin(VersionNumber) # get package repo head hash - head = LibGit2.ref_id(LibGit2.head(prepo)) + head = string(LibGit2.head_oid(prepo)) vers = collect(keys(filter((ver,info)->info.sha1==head, avail))) !isempty(vers) && return maximum(vers) @@ -148,7 +148,7 @@ function requires_path(pkg::AbstractString, avail::Dict=available(pkg)) repo = LibGit2.GitRepo(pkg) LibGit2.isdirty(repo, "REQUIRE") && return pkgreq LibGit2.revparse(prepo, "HEAD:REQUIRE") == nothing && isfile(pkgreq) && return pkgreq - head = LibGit2.ref_id(LibGit2.head(repo)) + head = string(LibGit2.head_oid(prepo)) LibGit2.free!(repo) for (ver,info) in avail if head == info.sha1 From bb40811d890c66a38c0cbb0853bf42eea0264e65 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Fri, 8 May 2015 22:49:37 -0400 Subject: [PATCH 0288/1938] fixed changed api --- base/pkg/entry.jl | 4 ++-- base/pkg/libgit2.jl | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 224388f4a608b..283bd54e3541a 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -151,9 +151,9 @@ function status(io::IO, pkg::AbstractString, ver::VersionNumber, fix::Bool) prepo = LibGit2.GitRepo(pkg) phead = LibGit2.head(prepo) if LibGit2.isattached(prepo) - print(io, LibGit2.ref_name(phead)) + print(io, LibGit2.shortname(phead)) else - print(io, LibGit2.ref_id(phead)[1:8]) + print(io, string(LibGit2.Oid(phead))[1:8]) end attrs = AbstractString[] isfile("METADATA",pkg,"url") || push!(attrs,"unregistered") diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 041bb25c1f623..0135aab66c206 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -39,26 +39,28 @@ function iscommit(id::AbstractString, repo::GitRepo) end function isdirty(repo::GitRepo, paths::AbstractString="") - tree_oid =revparse(repo, "HEAD^{tree}") + tree_oid = revparse(repo, "HEAD^{tree}") tree_oid == nothing && return true - tree_ptr = Ptr{Void}[0] + tree_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + tree_oid_ptr = Ref(tree_oid) err = ccall((:git_tree_lookup, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), tree_ptr, repo.ptr, tree_oid) + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}), + tree_ptr_ptr, repo.ptr, tree_oid_ptr) err != 0 && return true - diff_ptr = Ptr{Void}[0] + diff_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_diff_tree_to_workdir_with_index, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, Ptr{Void}), - diff_ptr, repo.ptr, tree_ptr[1], C_NULL, C_NULL) + diff_ptr_ptr, repo.ptr, tree_ptr_ptr[], C_NULL, C_NULL) err != 0 && return true if isempty(paths) - c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff_ptr[1]) + c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff_ptr_ptr[]) c > 0 && return true else # TODO look for specified path - c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff_ptr[1]) + c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff_ptr_ptr[]) c > 0 && return true end return false @@ -72,7 +74,7 @@ function merge_base(one::AbstractString, two::AbstractString, repo::GitRepo) (Ptr{Oid}, Ptr{Void}, Ptr{Oid}, Ptr{Oid}), moid_ptr, repo.ptr, oid1_ptr, oid2_ptr) err != 0 && return nothing - return moid[] + return moid_ptr[] end function is_ancestor_of(a::AbstractString, b::AbstractString, repo::GitRepo) @@ -83,14 +85,14 @@ end function set_remote_url(repo::GitRepo, url::AbstractString; remote::AbstractString="origin") cfg = GitConfig(repo) - err = set!(AbstractString, cfg, "remote.$remote.url", url) + err = set!(cfg, "remote.$remote.url", url) err !=0 && return m = match(GITHUB_REGEX,url) m == nothing && return push = "git@github.com:$(m.captures[1]).git" if push != url - err = set!(AbstractString, cfg, "remote.$remote.pushurl", push) + err = set!(cfg, "remote.$remote.pushurl", push) end end From 4ca8175e5ac2c7c504c40f4ea8705d49a1fdffe5 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sat, 9 May 2015 14:38:22 -0400 Subject: [PATCH 0289/1938] construct type pragmatically get rid off finalizers & do manual memory management everywhere added signature, commit, remote --- base/pkg/cache.jl | 34 ++++++--- base/pkg/dir.jl | 7 +- base/pkg/entry.jl | 25 ++++--- base/pkg/libgit2.jl | 127 +++++++++++++++++++-------------- base/pkg/libgit2/commit.jl | 40 +++++++++++ base/pkg/libgit2/config.jl | 25 +++---- base/pkg/libgit2/error.jl | 11 +++ base/pkg/libgit2/oid.jl | 6 +- base/pkg/libgit2/remote.jl | 28 ++++++++ base/pkg/libgit2/repository.jl | 28 ++++---- base/pkg/libgit2/signature.jl | 11 +++ base/pkg/libgit2/types.jl | 121 +++++++++---------------------- base/pkg/read.jl | 20 ++++-- 13 files changed, 288 insertions(+), 195 deletions(-) create mode 100644 base/pkg/libgit2/remote.jl diff --git a/base/pkg/cache.jl b/base/pkg/cache.jl index 83d2fd761289b..05200b427bd60 100644 --- a/base/pkg/cache.jl +++ b/base/pkg/cache.jl @@ -31,21 +31,33 @@ end function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) isdir(".cache") || mkcachedir() cache = path(pkg) - if !isdir(cache) + repo = if isdir(cache) + try + LibGit2.GitRepo(cache) + catch err + rethrow(err) + end + else info("Cloning cache of $pkg from $url") - repo = LibGit2.clone(url, path, bare = true, remote_cb = Pkg.LibGit2.mirror_cb) - if repo == noting + try + LibGit2.clone(url, cache, bare = true, remote_cb = LibGit2.mirror_cb) + catch err rm(cache, recursive=true) - error("Cannot clone $pkg from $url") + error("Cannot clone $pkg from $url\nError in LibGit2.clone: ", err) end - else - repo = LibGit2.GitRepo(cache) end - LibGit2.set_remote_url(repo, url) - if !all(sha1->LibGit2.iscommit(sha1, repo), sha1s) - info("Updating cache of $pkg...") - isa(LibGit2.fetch(repo), LibGit2.GitError) && - error("couldn't update $cache using `git remote update`") + try + LibGit2.set_remote_url(repo, url) + if !all(sha1->LibGit2.iscommit(sha1, repo), sha1s) + info("Updating cache of $pkg...") + try + LibGit2.fetch(repo) + catch + error("couldn't update $cache using `git remote update`") + end + end + catch err + rethrow(err) end filter(sha1->!LibGit2.iscommit(sha1, repo), sha1s) end diff --git a/base/pkg/dir.jl b/base/pkg/dir.jl index c8f573d2adb8f..3d853a4604bb7 100644 --- a/base/pkg/dir.jl +++ b/base/pkg/dir.jl @@ -47,8 +47,11 @@ function init(meta::AbstractString=DEFAULT_META, branch::AbstractString=META_BRA Base.cd(temp_dir) do info("Cloning METADATA from $meta") metadata_repo = LibGit2.clone(meta, "METADATA", branch = branch) - LibGit2.set_remote_url(metadata_repo, meta) - LibGit2.free!(metadata_repo) + try + LibGit2.set_remote_url(metadata_repo, meta) + finally + LibGit2.free!(metadata_repo) + end touch("REQUIRE") touch("META_BRANCH") open("META_BRANCH", "w") do io diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 283bd54e3541a..16e19ad35f565 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -149,17 +149,22 @@ function status(io::IO, pkg::AbstractString, ver::VersionNumber, fix::Bool) @printf io "%-19s" ver if ispath(pkg,".git") prepo = LibGit2.GitRepo(pkg) - phead = LibGit2.head(prepo) - if LibGit2.isattached(prepo) - print(io, LibGit2.shortname(phead)) - else - print(io, string(LibGit2.Oid(phead))[1:8]) + try + phead = LibGit2.head(prepo) + if LibGit2.isattached(prepo) + print(io, LibGit2.shortname(phead)) + else + print(io, string(LibGit2.Oid(phead))[1:8]) + end + attrs = AbstractString[] + isfile("METADATA",pkg,"url") || push!(attrs,"unregistered") + LibGit2.isdirty(prepo) && push!(attrs,"dirty") + isempty(attrs) || print(io, " (",join(attrs,", "),")") + catch + print(io, "broken-repo (unregistered)") + finally + LibGit2.free!(prepo) end - attrs = AbstractString[] - isfile("METADATA",pkg,"url") || push!(attrs,"unregistered") - LibGit2.isdirty(prepo) && push!(attrs,"dirty") - isempty(attrs) || print(io, " (",join(attrs,", "),")") - LibGit2.free!(prepo) else print(io, "non-repo (unregistered)") end diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 0135aab66c206..8100416430400 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -22,6 +22,7 @@ include("libgit2/commit.jl") include("libgit2/repository.jl") include("libgit2/config.jl") include("libgit2/walker.jl") +include("libgit2/remote.jl") function need_update(repo::GitRepo) if !isbare(repo) @@ -35,46 +36,59 @@ end function iscommit(id::AbstractString, repo::GitRepo) need_update(repo) - return !isa(get(GitCommit, repo, id), GitError) + res = true + try + get(GitCommit, repo, id) + catch + res = false + end + return res end function isdirty(repo::GitRepo, paths::AbstractString="") tree_oid = revparse(repo, "HEAD^{tree}") tree_oid == nothing && return true - tree_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - tree_oid_ptr = Ref(tree_oid) - err = ccall((:git_tree_lookup, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}), - tree_ptr_ptr, repo.ptr, tree_oid_ptr) - err != 0 && return true - - diff_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - err = ccall((:git_diff_tree_to_workdir_with_index, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, Ptr{Void}), - diff_ptr_ptr, repo.ptr, tree_ptr_ptr[], C_NULL, C_NULL) - err != 0 && return true - - if isempty(paths) - c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff_ptr_ptr[]) - c > 0 && return true - else - # TODO look for specified path - c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff_ptr_ptr[]) - c > 0 && return true + result = false + tree = get(GitTree, repo, tree_oid) + try + diff_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_diff_tree_to_workdir_with_index, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, Ptr{Void}), + diff_ptr_ptr, repo.ptr, tree.ptr, C_NULL, C_NULL) + diff = GitDiff(diff_ptr_ptr[]) + + if isempty(paths) + c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff.ptr) + result = c > 0 + else + # TODO look for specified path + c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff.ptr) + result = c > 0 + end + free!(diff) + catch + result = true + finally + free!(tree) end - return false + return result end function merge_base(one::AbstractString, two::AbstractString, repo::GitRepo) oid1_ptr = Ref(Oid(one)) oid2_ptr = Ref(Oid(two)) moid_ptr = Ref(Oid()) - err = ccall((:git_merge_base, :libgit2), Cint, + moid = try + @check ccall((:git_merge_base, :libgit2), Cint, (Ptr{Oid}, Ptr{Void}, Ptr{Oid}, Ptr{Oid}), moid_ptr, repo.ptr, oid1_ptr, oid2_ptr) - err != 0 && return nothing - return moid_ptr[] + moid_ptr[] + catch e + warn("merge_base: ", e.msg) + Oid() + end + return moid end function is_ancestor_of(a::AbstractString, b::AbstractString, repo::GitRepo) @@ -84,54 +98,61 @@ end function set_remote_url(repo::GitRepo, url::AbstractString; remote::AbstractString="origin") cfg = GitConfig(repo) - - err = set!(cfg, "remote.$remote.url", url) - err !=0 && return - - m = match(GITHUB_REGEX,url) - m == nothing && return - push = "git@github.com:$(m.captures[1]).git" - if push != url - err = set!(cfg, "remote.$remote.pushurl", push) + try + set!(cfg, "remote.$remote.url", url) + + m = match(GITHUB_REGEX,url) + if m != nothing + push = "git@github.com:$(m.captures[1]).git" + if push != url + set!(cfg, "remote.$remote.pushurl", push) + end + end + catch e + warn("set_remote_url: ", e.msg) + finally + free!(cfg) end end function set_remote_url(path::AbstractString, url::AbstractString; remote::AbstractString="origin") repo = GitRepo(path) set_remote_url(repo, url, remote=remote) - LibGit2.free!(prepo) + free!(prepo) end -function mirror_callback(remote::Ptr{Ptr{Void}}, repo::Ptr{Void}, name::Ptr{UInt8}, url::Ptr{UInt8}, payload::Ptr{Void}) +function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, name::Ptr{UInt8}, url::Ptr{UInt8}, payload::Ptr{Void}) # Create the remote with a mirroring url fetch_spec = "+refs/*:refs/*" err = ccall((:git_remote_create_with_fetchspec, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}, Ptr{UInt8}), - remote, repo, name, url, fetch_spec) + remote, repo_ptr, name, url, fetch_spec) err != 0 && return Cint(err) # And set the configuration option to true for the push command - config = GitConfig(GitRepo(repo, false)) + config = GitConfig(GitRepo(repo_ptr)) name_str = bytestring(name) - err = set!(config, "remote.$name_str.mirror", true) - free!(config) + err= try set!(config, "remote.$name_str.mirror", true) + catch -1 + finally free!(config) + end err != 0 && return Cint(err) - return Cint(0) end const mirror_cb = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}, Ptr{Void})) function fetch(repo::GitRepo, remote::AbstractString="origin") - remote_ptr = [C_NULL] - err = ccall((:git_remote_lookup, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), - remote_ptr, repo.ptr, remote) - err != 0 && return GitError(err) + rmt = get(GitRemote, repo, remote) - err = ccall((:git_remote_fetch, :libgit2), Cint, + try + @check ccall((:git_remote_fetch, :libgit2), Cint, (Ptr{Void}, Ptr{Void}, Ptr{UInt8}), - remote_ptr[1], C_NULL, C_NULL) - err != 0 && return GitError(err) + rmt.ptr, C_NULL, C_NULL) + catch err + rethrow(err) + finally + free!(rmt) + end end function clone(url::AbstractString, path::AbstractString; @@ -150,17 +171,15 @@ function clone(url::AbstractString, path::AbstractString; clone_opts_ref = Ref(clone_opts) repo_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - err = ccall((:git_clone, :libgit2), Cint, + @check ccall((:git_clone, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{UInt8}, Ptr{UInt8}, Ref{CloneOptionsStruct}), repo_ptr_ptr, url, path, clone_opts_ref) - err != 0 && return nothing - return GitRepo(repo_ptr_ptr[]) end function authors(repo::GitRepo) - athrs = Pkg.LibGit2.map( - (oid,repo)->author(get(GitCommit, repo, oid)), + athrs = map( + (oid,repo)->author(get(GitCommit, repo, oid))::Signature, repo) #, by = Pkg.LibGit2.GitConst.SORT_TIME) return athrs end diff --git a/base/pkg/libgit2/commit.jl b/base/pkg/libgit2/commit.jl index 8b175d6a75563..19cbe7aad866c 100644 --- a/base/pkg/libgit2/commit.jl +++ b/base/pkg/libgit2/commit.jl @@ -18,4 +18,44 @@ function committer(c::GitCommit) ptr = ccall((:git_commit_committer, :libgit2), Ptr{SignatureStruct}, (Ptr{Void},), c.ptr) @assert ptr != C_NULL return Signature(ptr) +end + +function commit(repo::GitRepo, + refname::AbstractString, + msg::AbstractString, + author::Signature, + committer::Signature, + tree::GitTree, + parents::GitCommit...) + id_ptr = Ref(Oid()) + nparents = length(parents) + parentptrs = Ptr{Void}[c.ptr for c in parents] + err = ccall((:git_commit_create, :libgit2), Cint, + (Ptr{Oid}, Ptr{Void}, Ptr{Uint8}, + Ptr{SignatureStruct}, Ptr{SignatureStruct}, + Ptr{Uint8}, Ptr{Uint8}, Ptr{Void}, + Csize_t, Ptr{Ptr{Void}}), + id_ptr, repo.ptr, isempty(refname) ? C_NULL : refname, + author, committer, + C_NULL, msg, tree, + nparents, nparents > 0 ? parentptrs : C_NULL) + err !=0 && return GitError(err) + return id_ptr[] +end + +function commit(repo::GitRepo, msg::AbstractString, + refname::AbstractString="", + author::Signature = Signature(repo), + committer::Signature = Signature(repo), + tree_id::Oid = Oid(), + parent_ids::Vector{Oid}=Oid[]) + tree = if isempty(tree_id) + get(GitTree, repo) + else + get(GitTree, repo, tree_id) + end + isa(err, GitError)&& return err + + parents = [get(GitCommit, repo, parent) for parent in parents] + return commit(repo, refname, msg, author, committer, tree, parents) end \ No newline at end of file diff --git a/base/pkg/libgit2/config.jl b/base/pkg/libgit2/config.jl index b125d08c1c944..5e1f88c3c8a45 100644 --- a/base/pkg/libgit2/config.jl +++ b/base/pkg/libgit2/config.jl @@ -24,24 +24,24 @@ end function get{T}(::Type{T}, c::GitConfig, name::AbstractString) if T<:AbstractString str_ptr = Ref{Ptr{UInt8}}(C_NULL) - err = ccall((:git_config_get_string, :libgit2), Cint, + @check ccall((:git_config_get_string, :libgit2), Cint, (Ptr{Ptr{UInt8}}, Ptr{Void}, Ptr{UInt8}), str_ptr, c.ptr, name) - err == GitErrorConst.GIT_OK && return bytestring(str_ptr[]) + return bytestring(str_ptr[]) elseif is(T, Bool) val_ptr = Ref(Cint(0)) - ccall((:git_config_get_bool, :libgit2), Cint, + @check ccall((:git_config_get_bool, :libgit2), Cint, (Ptr{Cint}, Ptr{Void}, Ptr{UInt8}), val_ptr, c.ptr, name) - err == GitErrorConst.GIT_OK && return Bool(val_ptr[]) + return Bool(val_ptr[]) elseif is(T, Int32) val_ptr = Ref(Cint(0)) - ccall((:git_config_get_bool, :libgit2), Cint, + @check ccall((:git_config_get_bool, :libgit2), Cint, (Ptr{Cint}, Ptr{Void}, Ptr{UInt8}), val_ptr, c.ptr, name) - err == GitErrorConst.GIT_OK && return val_ptr[] + return val_ptr[] elseif is(T, Int64) val_ptr = Ref(Cintmax_t(0)) - ccall((:git_config_get_bool, :libgit2), Cint, + @check ccall((:git_config_get_bool, :libgit2), Cint, (Ptr{Cintmax_t}, Ptr{Void}, Ptr{UInt8}), val_ptr, c.ptr, name) - err == GitErrorConst.GIT_OK && return Bool(val_ptr[]) + return val_ptr[] else return nothing end @@ -49,20 +49,21 @@ end function set!{T}(c::GitConfig, name::AbstractString, value::T) err = if T<:AbstractString - ccall((:git_config_set_string, :libgit2), Cint, + @check ccall((:git_config_set_string, :libgit2), Cint, (Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}), c.ptr, name, value) elseif is(T, Bool) bval = Int32(value) - ccall((:git_config_set_bool, :libgit2), Cint, + @check ccall((:git_config_set_bool, :libgit2), Cint, (Ptr{Void}, Ptr{UInt8}, Cint), c.ptr, name, bval) elseif is(T, Int32) - ccall((:git_config_set_int32, :libgit2), Cint, + @check ccall((:git_config_set_int32, :libgit2), Cint, (Ptr{Void}, Ptr{UInt8}, Cint), c.ptr, name, value) elseif is(T, Int64) - ccall((:git_config_set_int64, :libgit2), Cint, + @check ccall((:git_config_set_int64, :libgit2), Cint, (Ptr{Void}, Ptr{UInt8}, Cintmax_t), c.ptr, name, value) else warn("Type $T is not supported") + -1 end return err end diff --git a/base/pkg/libgit2/error.jl b/base/pkg/libgit2/error.jl index bec7ad2d8645f..8d054fa0acd6f 100644 --- a/base/pkg/libgit2/error.jl +++ b/base/pkg/libgit2/error.jl @@ -93,4 +93,15 @@ function GitError(code::Integer) err_code = git_error_code[Int(code)] err_class, err_msg = last_error() return GitError{err_class, err_code}(err_msg) +end + +macro check(git_func) + quote + local err::Cint + err = $(esc(git_func::Expr)) + if err < 0 + throw(GitError(err)) + end + err + end end \ No newline at end of file diff --git a/base/pkg/libgit2/oid.jl b/base/pkg/libgit2/oid.jl index 6f45eb16b2452..b7e53d782291d 100644 --- a/base/pkg/libgit2/oid.jl +++ b/base/pkg/libgit2/oid.jl @@ -48,7 +48,7 @@ function Oid(id::AbstractString) ccall((:git_oid_fromstrp, :libgit2), Cint, (Ptr{Oid}, Ptr{Cchar}), oid_ptr, bstr) end - err != 0 && return nothing + err != 0 && return Oid() return oid_ptr[] end @@ -58,13 +58,13 @@ function Oid(ref::GitReference) typ = ccall((:git_reference_type, :libgit2), Cint, (Ptr{Void},), ref.ptr) typ != 1 && return Oid() oid_ptr = ccall((:git_reference_target, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) - oid_ptr == C_NULL && return "" + oid_ptr == C_NULL && return Oid() return Oid(oid_ptr) end function Oid(obj::Ptr{Void}) oid_ptr = ccall((:git_object_id, :libgit2), Ptr{UInt8}, (Ptr{Void},), obj) - oid_ptr == C_NULL && return "" + oid_ptr == C_NULL && return Oid() return Oid(oid_ptr) end diff --git a/base/pkg/libgit2/remote.jl b/base/pkg/libgit2/remote.jl new file mode 100644 index 0000000000000..97ba3d91c9cdb --- /dev/null +++ b/base/pkg/libgit2/remote.jl @@ -0,0 +1,28 @@ +function GitRemote(repo::GitRepo, rmt_name::AbstractString, rmt_url::AbstractString) + rmt_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + err = ccall((:git_remote_create, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}), + rmt_ptr_ptr, repo.ptr, rmt_name, rmt_url) + err != 0 && return GitError(err) + return GitRemote(rmt_ptr_ptr[]) +end + +function get(::Type{GitRemote}, repo::GitRepo, rmt_name::AbstractString) + rmt_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + err = ccall((:git_remote_lookup, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), + rmt_ptr_ptr, repo.ptr, rmt_name) + err != 0 && return GitError(err) + return GitRemote(rmt_ptr_ptr[]) +end + +function save(rmt::GitRemote) + err = ccall((:git_remote_save, :libgit2), Cint, (Ptr{Void}, ), rmt.ptr) + err != 0 && return GitError(err) +end + +function url(rmt::GitRemote) + url_ptr = ccall((:git_remote_url, :libgit2), Ptr{UInt8}, (Ptr{Void}, ), rmt.ptr) + url_ptr == C_NULL && return "" + return bytestring(url_ptr) +end diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index b795fdd4a454a..34cca9a78ec6d 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -1,14 +1,14 @@ function GitRepo(path::AbstractString) - repo_ptr = Ptr{Void}[0] + repo_ptr_ptr = Ptr{Void}[0] err = ccall((:git_repository_open, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{UInt8}), repo_ptr, path) + (Ptr{Ptr{Void}}, Ptr{UInt8}), repo_ptr_ptr, path) if err != GitErrorConst.GIT_OK - if repo_ptr[1] != C_NULL - ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), repo_ptr[1]) + if repo_ptr_ptr[] != C_NULL + free!(GitRepo(repo_ptr_ptr[])) end - return nothing + throw(GitError(err)) end - return GitRepo(repo_ptr[1]) + return GitRepo(repo_ptr_ptr[]) end function close(r::GitRepo) @@ -17,11 +17,17 @@ function close(r::GitRepo) end end +function init(path::AbstractString, bare::Cuint = Cuint(0)) + repo_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_repository_init, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{UInt8}, Cuint), repo_ptr_ptr, path, bare) + return GitRepo(repo_ptr_ptr[]) +end + function head(repo::GitRepo) head_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - err = ccall((:git_repository_head, :libgit2), Cint, + @check ccall((:git_repository_head, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}), head_ptr_ptr, repo.ptr) - (err != 0) && return nothing return GitReference(head_ptr_ptr[]) end head_oid(repo::GitRepo) = Oid(head(repo)) @@ -32,9 +38,8 @@ end function revparse(repo::GitRepo, obj::AbstractString) obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - err = ccall((:git_revparse_single, :libgit2), Cint, + @check ccall((:git_revparse_single, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), obj_ptr_ptr, repo.ptr, obj) - err != 0 && return nothing return Oid(obj_ptr_ptr[]) end @@ -50,7 +55,7 @@ function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::Oid, prefix::Bool=false error("Type $T is not supported") end - err = if prefix + @check if prefix ccall((:git_object_lookup_prefix, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}, Csize_t, Cint), obj_ptr_ptr, r.ptr, id_ptr, len, git_otype) @@ -59,7 +64,6 @@ function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::Oid, prefix::Bool=false (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}, Cint), obj_ptr_ptr, r.ptr, id_ptr, git_otype) end - err != 0 && return GitError(err) return T(obj_ptr_ptr[]) end diff --git a/base/pkg/libgit2/signature.jl b/base/pkg/libgit2/signature.jl index 6cb1f0c74dd60..b7b209c8fe784 100644 --- a/base/pkg/libgit2/signature.jl +++ b/base/pkg/libgit2/signature.jl @@ -17,3 +17,14 @@ function Signature(name::AbstractString, email::AbstractString) ccall((:git_signature_free, :libgit2), Void, (Ptr{SignatureStruct},), sig_ptr) return s end + +function Signature(repo::GitRepo) + sig_ptr_ptr = Ref{Ptr{SignatureStruct}}(C_NULL) + err = ccall((:git_signature_default, libgit2), Cint, + (Ptr{Ptr{SignatureStruct}}, Ptr{Void}), sig_ptr_ptr, repo.ptr) + err != 0 && return GitError(err) + sig_ptr = sig_ptr_ptr[] + s = Signature(sig_ptr) + ccall((:git_signature_free, :libgit2), Void, (Ptr{SignatureStruct},), sig_ptr) + return s +end \ No newline at end of file diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index d08726df1df7e..4c6c746243379 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -104,24 +104,34 @@ CloneOptionsStruct() = CloneOptionsStruct(one(Cuint), Ptr{Void}(0), Ptr{Void}(0) ) -type GitRepo - ptr::Ptr{Void} - - function GitRepo(ptr::Ptr{Void}, own::Bool=true) - @assert ptr != C_NULL - r = new(ptr) - own && finalizer(r, free!) - return r +# Common types +for (typ, fnc) in ((:GitRemote, :(:git_remote_free)), + (:GitRevWalker, :(:git_revwalk_free)), + (:GitConfig, :(:git_config_free)), + (:GitReference, :(:git_reference_free)), + (:GitDiff, :(:git_diff_free)), + (:GitRepo, :(:git_repository_free))) + @eval type $typ + ptr::Ptr{Void} + function $typ(ptr::Ptr{Void}) + @assert ptr != C_NULL + obj = new(ptr) + return obj + end + $typ() = new(C_NULL) end -end -function free!(r::GitRepo) - if r.ptr != C_NULL - ccall((:git_repository_free, :libgit2), Void, (Ptr{Void},), r.ptr) - r.ptr = C_NULL + @eval function free!(obj::$typ) + if obj.ptr != C_NULL + ccall(($fnc, :libgit2), Void, (Ptr{Void},), obj.ptr) + obj.ptr = C_NULL + end end + + @eval Base.isempty(obj::$typ) = (obj.ptr == C_NULL) end +# Object types abstract GitObject function free!(o::GitObject) @@ -130,82 +140,21 @@ function free!(o::GitObject) o.ptr = C_NULL end end - -type GitCommit <: GitObject - ptr::Ptr{Void} - - function GitCommit(ptr::Ptr{Void}) - @assert ptr != C_NULL - this = new(ptr) - finalizer(this, free!) - return this - end -end - -type GitTree <: GitObject - ptr::Ptr{Void} - - function GitTree(ptr::Ptr{Void}) - @assert ptr != C_NULL - this = new(ptr) - finalizer(this, free!) - return this - end -end - -type GitReference - ptr::Ptr{Void} - - function GitReference(ptr::Ptr{Void}) - r = new(ptr) - finalizer(r, free!) - return r - end -end - -function free!(r::GitReference) - if r.ptr != C_NULL - ccall((:git_reference_free, :libgit2), Void, (Ptr{Void},), r.ptr) - r.ptr = C_NULL - end -end - -type GitConfig - ptr::Ptr{Void} - - function GitConfig(ptr::Ptr{Void}) - @assert ptr != C_NULL - cfg = new(ptr) - finalizer(cfg, free!) - return cfg - end -end - -function free!(cfg::GitConfig) - if cfg.ptr != C_NULL - ccall((:git_config_free, :libgit2), Void, (Ptr{Void},), cfg.ptr) - cfg.ptr = C_NULL - end -end - -type GitRevWalker - ptr::Ptr{Void} - - function GitRevWalker(ptr::Ptr{Void}) - @assert ptr != C_NULL - w = new(ptr) - finalizer(w, free!) - return w - end -end - -function free!(w::GitRevWalker) - if w.ptr != C_NULL - ccall((:git_revwalk_free, :libgit2), Void, (Ptr{Void},), w.ptr) - w.ptr = C_NULL +Base.isempty(obj::GitObject) = (obj.ptr == C_NULL) + +for typ in [:GitCommit, :GitTree] + @eval type $typ <: GitObject + ptr::Ptr{Void} + function $typ(ptr::Ptr{Void}) + @assert ptr != C_NULL + obj = new(ptr) + return obj + end + $typ() = new(C_NULL) end end +# Misc types type Signature name::UTF8String email::UTF8String diff --git a/base/pkg/read.jl b/base/pkg/read.jl index b6d85d24eb2cc..c24e68797b71e 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -101,6 +101,7 @@ function installed_version(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::D # get package repo head hash head = string(LibGit2.head_oid(prepo)) + isempty(head) && return typemin(VersionNumber) vers = collect(keys(filter((ver,info)->info.sha1==head, avail))) !isempty(vers) && return maximum(vers) @@ -169,11 +170,20 @@ function installed(avail::Dict=available()) for pkg in readdir() isinstalled(pkg) || continue ap = get(avail,pkg,Dict{VersionNumber,Available}()) - prepo = LibGit2.GitRepo(pkg) - ver = installed_version(pkg, prepo, ap) - fixed = isfixed(pkg, prepo, ap) - pkgs[pkg] = (ver, fixed) - LibGit2.free!(prepo) + try + prepo = LibGit2.GitRepo(pkg) + try + ver = installed_version(pkg, prepo, ap) + fixed = isfixed(pkg, prepo, ap) + pkgs[pkg] = (ver, fixed) + catch e + pkgs[pkg] = (typemin(VersionNumber), true) + finally + LibGit2.free!(prepo) + end + catch + pkgs[pkg] = (typemin(VersionNumber), true) + end end return pkgs end From 25a7d0aaaf22ca6b484cd962b29d569abae63d51 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sat, 9 May 2015 19:43:47 -0400 Subject: [PATCH 0290/1938] added `with_libgit2` helper function better memory resource handling `finalize` instead of `free` --- base/pkg/dir.jl | 7 ++-- base/pkg/entry.jl | 10 ++++-- base/pkg/libgit2.jl | 27 ++++++--------- base/pkg/libgit2/commit.jl | 60 ++++++++++++++++++++++++---------- base/pkg/libgit2/remote.jl | 9 ++--- base/pkg/libgit2/repository.jl | 2 +- base/pkg/libgit2/signature.jl | 27 +++++++++------ base/pkg/libgit2/types.jl | 55 +++++++++++++++++++++---------- base/pkg/read.jl | 18 ++++++---- 9 files changed, 132 insertions(+), 83 deletions(-) diff --git a/base/pkg/dir.jl b/base/pkg/dir.jl index 3d853a4604bb7..2d00570b6b9f3 100644 --- a/base/pkg/dir.jl +++ b/base/pkg/dir.jl @@ -3,7 +3,7 @@ module Dir import ..Pkg: DEFAULT_META, META_BRANCH -import ..LibGit2 +import ..LibGit2, ..LibGit2.with_libgit2 const DIR_NAME = ".julia" @@ -46,11 +46,8 @@ function init(meta::AbstractString=DEFAULT_META, branch::AbstractString=META_BRA temp_dir = mktempdir(dir) Base.cd(temp_dir) do info("Cloning METADATA from $meta") - metadata_repo = LibGit2.clone(meta, "METADATA", branch = branch) - try + with_libgit2(LibGit2.clone(meta, "METADATA", branch = branch)) do metadata_repo LibGit2.set_remote_url(metadata_repo, meta) - finally - LibGit2.free!(metadata_repo) end touch("REQUIRE") touch("META_BRANCH") diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 16e19ad35f565..2abc6a9c83fda 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -107,9 +107,13 @@ end function installed(pkg::AbstractString) avail = Read.available(pkg) if Read.isinstalled(pkg) + res = typemin(VersionNumber) prepo = LibGit2.GitRepo(pkg) - res = Read.installed_version(pkg, prepo, avail) - LibGit2.free!(prepo) + try + res = Read.installed_version(pkg, prepo, avail) + finally + LibGit2.finalize(prepo) + end return res end isempty(avail) && error("$pkg is not a package (not registered or installed)") @@ -163,7 +167,7 @@ function status(io::IO, pkg::AbstractString, ver::VersionNumber, fix::Bool) catch print(io, "broken-repo (unregistered)") finally - LibGit2.free!(prepo) + LibGit2.finalize(prepo) end else print(io, "non-repo (unregistered)") diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 8100416430400..70b7f4db49ed4 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -1,5 +1,7 @@ module LibGit2 +export with_libgit2 + const GITHUB_REGEX = r"^(?:git@|git://|https://(?:[\w\.\+\-]+@)?)github.com[:/](([^/].+)/(.+?))(?:\.git)?$"i @@ -66,11 +68,11 @@ function isdirty(repo::GitRepo, paths::AbstractString="") c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff.ptr) result = c > 0 end - free!(diff) + finalize(diff) catch result = true finally - free!(tree) + finalize(tree) end return result end @@ -97,8 +99,7 @@ function is_ancestor_of(a::AbstractString, b::AbstractString, repo::GitRepo) end function set_remote_url(repo::GitRepo, url::AbstractString; remote::AbstractString="origin") - cfg = GitConfig(repo) - try + with_libgit2(GitConfig, repo) do cfg set!(cfg, "remote.$remote.url", url) m = match(GITHUB_REGEX,url) @@ -108,17 +109,13 @@ function set_remote_url(repo::GitRepo, url::AbstractString; remote::AbstractStri set!(cfg, "remote.$remote.pushurl", push) end end - catch e - warn("set_remote_url: ", e.msg) - finally - free!(cfg) end end function set_remote_url(path::AbstractString, url::AbstractString; remote::AbstractString="origin") - repo = GitRepo(path) - set_remote_url(repo, url, remote=remote) - free!(prepo) + with_libgit2(GitRepo, path) do repo + set_remote_url(repo, url, remote=remote) + end end function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, name::Ptr{UInt8}, url::Ptr{UInt8}, payload::Ptr{Void}) @@ -134,7 +131,7 @@ function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, name::Ptr{ name_str = bytestring(name) err= try set!(config, "remote.$name_str.mirror", true) catch -1 - finally free!(config) + finally finalize(config) end err != 0 && return Cint(err) return Cint(0) @@ -144,14 +141,10 @@ const mirror_cb = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, P function fetch(repo::GitRepo, remote::AbstractString="origin") rmt = get(GitRemote, repo, remote) - try + with_libgit2(rmt, warn_on_exception=true) do rmt @check ccall((:git_remote_fetch, :libgit2), Cint, (Ptr{Void}, Ptr{Void}, Ptr{UInt8}), rmt.ptr, C_NULL, C_NULL) - catch err - rethrow(err) - finally - free!(rmt) end end diff --git a/base/pkg/libgit2/commit.jl b/base/pkg/libgit2/commit.jl index 19cbe7aad866c..c71e4eb20c462 100644 --- a/base/pkg/libgit2/commit.jl +++ b/base/pkg/libgit2/commit.jl @@ -20,42 +20,68 @@ function committer(c::GitCommit) return Signature(ptr) end +""" Wrapper around `git_commit_create` """ function commit(repo::GitRepo, refname::AbstractString, msg::AbstractString, - author::Signature, - committer::Signature, + author::GitSignature, + committer::GitSignature, tree::GitTree, parents::GitCommit...) - id_ptr = Ref(Oid()) + commit_id_ptr = Ref(Oid()) nparents = length(parents) parentptrs = Ptr{Void}[c.ptr for c in parents] - err = ccall((:git_commit_create, :libgit2), Cint, + @check ccall((:git_commit_create, :libgit2), Cint, (Ptr{Oid}, Ptr{Void}, Ptr{Uint8}, Ptr{SignatureStruct}, Ptr{SignatureStruct}, Ptr{Uint8}, Ptr{Uint8}, Ptr{Void}, Csize_t, Ptr{Ptr{Void}}), - id_ptr, repo.ptr, isempty(refname) ? C_NULL : refname, - author, committer, - C_NULL, msg, tree, + commit_id_ptr, repo.ptr, isempty(refname) ? C_NULL : refname, + author.ptr, committer.ptr, + C_NULL, msg, tree.ptr, nparents, nparents > 0 ? parentptrs : C_NULL) - err !=0 && return GitError(err) - return id_ptr[] + return commit_id_ptr[] end +"""Commit changes to repository""" function commit(repo::GitRepo, msg::AbstractString, - refname::AbstractString="", + refname::AbstractString="HEAD", author::Signature = Signature(repo), committer::Signature = Signature(repo), tree_id::Oid = Oid(), parent_ids::Vector{Oid}=Oid[]) - tree = if isempty(tree_id) - get(GitTree, repo) - else - get(GitTree, repo, tree_id) + # Retrieve tree identifier + if iszero(tree_id) + idx = GitIndex(repo) + try + tree_id = write_tree!(idx) + catch err + rethrow(err) + finally + finalize(idx) + end end - isa(err, GitError)&& return err - parents = [get(GitCommit, repo, parent) for parent in parents] - return commit(repo, refname, msg, author, committer, tree, parents) + # return commit id + commit_id = Oid() + + # get necessary objects + tree = get(GitTree, repo, tree_id) + auth_sig = convert(GitSignature, author) + comm_sig = convert(GitSignature, committer) + parents = GitCommit[] + try + for parent in parents + push!(parents, get(GitCommit, repo, parent)) + end + commit_id = commit(repo, refname, msg, auth_sig, comm_sig, tree, parents...) + finally + for parent in parents + finalize(parent) + end + finalize(tree) + finalize(auth_sig) + finalize(auth_sig) + end + return commit_id end \ No newline at end of file diff --git a/base/pkg/libgit2/remote.jl b/base/pkg/libgit2/remote.jl index 97ba3d91c9cdb..691e53a9fcc1d 100644 --- a/base/pkg/libgit2/remote.jl +++ b/base/pkg/libgit2/remote.jl @@ -1,24 +1,21 @@ function GitRemote(repo::GitRepo, rmt_name::AbstractString, rmt_url::AbstractString) rmt_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - err = ccall((:git_remote_create, :libgit2), Cint, + @check ccall((:git_remote_create, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}), rmt_ptr_ptr, repo.ptr, rmt_name, rmt_url) - err != 0 && return GitError(err) return GitRemote(rmt_ptr_ptr[]) end function get(::Type{GitRemote}, repo::GitRepo, rmt_name::AbstractString) rmt_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - err = ccall((:git_remote_lookup, :libgit2), Cint, + @check ccall((:git_remote_lookup, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), rmt_ptr_ptr, repo.ptr, rmt_name) - err != 0 && return GitError(err) return GitRemote(rmt_ptr_ptr[]) end function save(rmt::GitRemote) - err = ccall((:git_remote_save, :libgit2), Cint, (Ptr{Void}, ), rmt.ptr) - err != 0 && return GitError(err) + @check ccall((:git_remote_save, :libgit2), Cint, (Ptr{Void}, ), rmt.ptr) end function url(rmt::GitRemote) diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index 34cca9a78ec6d..9f52c7cd0ec3b 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -4,7 +4,7 @@ function GitRepo(path::AbstractString) (Ptr{Ptr{Void}}, Ptr{UInt8}), repo_ptr_ptr, path) if err != GitErrorConst.GIT_OK if repo_ptr_ptr[] != C_NULL - free!(GitRepo(repo_ptr_ptr[])) + finalize(GitRepo(repo_ptr_ptr[])) end throw(GitError(err)) end diff --git a/base/pkg/libgit2/signature.jl b/base/pkg/libgit2/signature.jl index b7b209c8fe784..3af9d1d974236 100644 --- a/base/pkg/libgit2/signature.jl +++ b/base/pkg/libgit2/signature.jl @@ -6,25 +6,32 @@ function Signature(ptr::Ptr{SignatureStruct}) offset = sig.when.offset return Signature(name, email, time, offset) end +Signature(sig::GitSignature) = Signature(sig.ptr) function Signature(name::AbstractString, email::AbstractString) sig_ptr_ptr = Ref{Ptr{SignatureStruct}}(C_NULL) - err = ccall((:git_signature_now, :libgit2), Cint, + @check ccall((:git_signature_now, :libgit2), Cint, (Ptr{Ptr{SignatureStruct}}, Ptr{UInt8}, Ptr{UInt8}), sig_ptr_ptr, name, email) - err != 0 && return GitError(err) - sig_ptr = sig_ptr_ptr[] - s = Signature(sig_ptr) - ccall((:git_signature_free, :libgit2), Void, (Ptr{SignatureStruct},), sig_ptr) + sig = GitSignature(sig_ptr_ptr[]) + s = Signature(sig.ptr) + finalize(sig) return s end function Signature(repo::GitRepo) sig_ptr_ptr = Ref{Ptr{SignatureStruct}}(C_NULL) - err = ccall((:git_signature_default, libgit2), Cint, + @check ccall((:git_signature_default, :libgit2), Cint, (Ptr{Ptr{SignatureStruct}}, Ptr{Void}), sig_ptr_ptr, repo.ptr) - err != 0 && return GitError(err) - sig_ptr = sig_ptr_ptr[] - s = Signature(sig_ptr) - ccall((:git_signature_free, :libgit2), Void, (Ptr{SignatureStruct},), sig_ptr) + sig = GitSignature(sig_ptr_ptr[]) + s = Signature(sig.ptr) + finalize(sig) return s +end + +function Base.convert(::Type{GitSignature}, sig::Signature) + sig_ptr_ptr = Ref{Ptr{SignatureStruct}}(C_NULL) + @check ccall((:git_signature_new, :libgit2), Cint, + (Ptr{Ptr{SignatureStruct}}, Ptr{Uint8}, Ptr{Uint8}, Cint, Cint), + sig_ptr_ptr, sig.name, sig.email, sig.time, sig.time_offset) + return GitSignature(sig_ptr_ptr[]) end \ No newline at end of file diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index 4c6c746243379..e517079c4684a 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -105,25 +105,26 @@ CloneOptionsStruct() = CloneOptionsStruct(one(Cuint), ) # Common types -for (typ, fnc) in ((:GitRemote, :(:git_remote_free)), - (:GitRevWalker, :(:git_revwalk_free)), - (:GitConfig, :(:git_config_free)), - (:GitReference, :(:git_reference_free)), - (:GitDiff, :(:git_diff_free)), - (:GitRepo, :(:git_repository_free))) +for (typ, ref, fnc) in ((:GitRemote, :Void, :(:git_remote_free)), + (:GitRevWalker, :Void, :(:git_revwalk_free)), + (:GitConfig, :Void, :(:git_config_free)), + (:GitReference, :Void, :(:git_reference_free)), + (:GitDiff, :Void, :(:git_diff_free)), + (:GitIndex, :Void, :(:git_index_free)), + (:GitSignature, :SignatureStruct, :(:git_signature_free)), + (:GitRepo, :Void, :(:git_repository_free))) @eval type $typ - ptr::Ptr{Void} - function $typ(ptr::Ptr{Void}) + ptr::Ptr{$ref} + function $typ(ptr::Ptr{$ref}) @assert ptr != C_NULL obj = new(ptr) return obj end - $typ() = new(C_NULL) end - @eval function free!(obj::$typ) + @eval function finalize(obj::$typ) if obj.ptr != C_NULL - ccall(($fnc, :libgit2), Void, (Ptr{Void},), obj.ptr) + ccall(($fnc, :libgit2), Void, (Ptr{$ref},), obj.ptr) obj.ptr = C_NULL end end @@ -134,7 +135,7 @@ end # Object types abstract GitObject -function free!(o::GitObject) +function finalize(o::GitObject) if o.ptr != C_NULL ccall((:git_object_free, :libgit2), Void, (Ptr{Void},), o.ptr) o.ptr = C_NULL @@ -150,14 +151,34 @@ for typ in [:GitCommit, :GitTree] obj = new(ptr) return obj end - $typ() = new(C_NULL) end end -# Misc types +# Structure has the same layout as SignatureStruct type Signature - name::UTF8String - email::UTF8String - time::Int32 + name::AbstractString + email::AbstractString + time::Int64 time_offset::Int32 end + +""" Resource management helper function +""" +function with_libgit2(f::Function, obj) + try + f(obj) + catch err + rethrow(err) + finally + finalize(obj) + end +end + +function with_libgit2{T}(f::Function, ::Type{T}, args...; warn_on_exception::Bool=true) + obj = T(args...) + try + with_libgit2(f, obj) + catch err + warn_on_exception ? warn("$(string(T)) thrown exception: $err") : rethrow(err) + end +end \ No newline at end of file diff --git a/base/pkg/read.jl b/base/pkg/read.jl index c24e68797b71e..6c4bfb66d4b50 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -92,7 +92,7 @@ function isfixed(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::Dict=availa Base.warn_once("unknown $pkg commit $(info.sha1[1:8]), metadata may be ahead of package cache") end end - cache_has_head && LibGit2.free!(crepo) + cache_has_head && LibGit2.finalize(crepo) return res end @@ -128,7 +128,7 @@ function installed_version(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::D base == sha1 && push!(ancestors,ver) base == head && push!(descendants,ver) end - cache_has_head && LibGit2.free!(crepo) + cache_has_head && LibGit2.finalize(crepo) both = sort!(intersect(ancestors,descendants)) isempty(both) || warn("$pkg: some versions are both ancestors and descendants of head: $both") @@ -147,10 +147,14 @@ function requires_path(pkg::AbstractString, avail::Dict=available(pkg)) pkgreq = joinpath(pkg,"REQUIRE") ispath(pkg,".git") || return pkgreq repo = LibGit2.GitRepo(pkg) - LibGit2.isdirty(repo, "REQUIRE") && return pkgreq - LibGit2.revparse(prepo, "HEAD:REQUIRE") == nothing && isfile(pkgreq) && return pkgreq - head = string(LibGit2.head_oid(prepo)) - LibGit2.free!(repo) + head = "" + try + LibGit2.isdirty(repo, "REQUIRE") && return pkgreq + LibGit2.revparse(repo, "HEAD:REQUIRE") == nothing && isfile(pkgreq) && return pkgreq + head = string(LibGit2.head_oid(repo)) + finally + LibGit2.finalize(repo) + end for (ver,info) in avail if head == info.sha1 return joinpath("METADATA", pkg, "versions", string(ver), "requires") @@ -179,7 +183,7 @@ function installed(avail::Dict=available()) catch e pkgs[pkg] = (typemin(VersionNumber), true) finally - LibGit2.free!(prepo) + LibGit2.finalize(prepo) end catch pkgs[pkg] = (typemin(VersionNumber), true) From 3516046acfb82451d25afb6c4389a04b8201ee89 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sun, 10 May 2015 03:57:59 -0400 Subject: [PATCH 0291/1938] package generator works!!! added `index` & `strarray` related functions --- base/pkg/generate.jl | 126 ++++++++++++++++++++------------- base/pkg/libgit2.jl | 69 ++++++++++++++++-- base/pkg/libgit2/commit.jl | 20 +++--- base/pkg/libgit2/const.jl | 2 +- base/pkg/libgit2/index.jl | 99 ++++++++++++++++++++++++++ base/pkg/libgit2/oid.jl | 11 ++- base/pkg/libgit2/reference.jl | 8 +++ base/pkg/libgit2/repository.jl | 5 ++ base/pkg/libgit2/strarray.jl | 29 ++++++++ base/pkg/libgit2/types.jl | 13 +++- 10 files changed, 312 insertions(+), 70 deletions(-) create mode 100644 base/pkg/libgit2/index.jl create mode 100644 base/pkg/libgit2/strarray.jl diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index 1323fad7fedb8..475c7bd1c528c 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -2,15 +2,20 @@ module Generate -import ..Git, ..LibGit2, ..Read +import ..Git, ..Read, ..LibGit2 +importall ..LibGit2 copyright_year() = Dates.year(Dates.today()) -copyright_name(dir::AbstractString) = LibGit2.get(AbstractString, LibGit2.GitConfig(LibGit2.GitRepo(dir)), "user.name") -github_user() = LibGit2.get(AbstractString, LibGit2.GitConfig(), "github.user") +copyright_name(repo::GitRepo) = + with(GitConfig, repo) do cfg + LibGit2.get(AbstractString, cfg, "user.name") + end +github_user() = with(GitConfig) do cfg + LibGit2.get(AbstractString, cfg, "github.user") + end -function git_contributors(dir::AbstractString, n::Int=typemax(Int)) +function git_contributors(repo::GitRepo, n::Int=typemax(Int)) contrib = Dict() - repo = LibGit2.GitRepo(dir) for sig in LibGit2.authors(repo) if haskey(contrib, sig.email) contrib[sig.email][1] += 1 @@ -18,7 +23,6 @@ function git_contributors(dir::AbstractString, n::Int=typemax(Int)) contrib[sig.email] = [1, sig.name] end end - LibGit2.free!(repo) names = Dict() for (commits,name) in values(contrib) @@ -39,25 +43,31 @@ function package( ) isnew = !ispath(pkg) try - if isnew + repo = if isnew url = isempty(user) ? "" : "git://github.com/$user/$pkg.jl.git" Generate.init(pkg,url,config=config) else - Git.dirty(dir=pkg) && error("$pkg is dirty – commit or stash your changes") + repo = GitRepo(pkg) + if LibGit2.isdirty(repo) + LibGit2.finalize(repo) + error("$pkg is dirty – commit or stash your changes") + end + repo end - Git.transact(dir=pkg) do + LibGit2.transact(repo, dir=pkg) do repo if isempty(authors) - authors = isnew ? copyright_name(pkg) : git_contributors(pkg,5) + authors = isnew ? copyright_name(repo) : git_contributors(repo,5) end - Generate.license(pkg,license,years,authors,force=force) - Generate.readme(pkg,user,force=force) - Generate.entrypoint(pkg,force=force) - Generate.tests(pkg,force=force) - Generate.require(pkg,force=force) - Generate.travis(pkg,force=force) - Generate.appveyor(pkg,force=force) - Generate.gitignore(pkg,force=force) + + files = [Generate.license(pkg,license,years,authors,force=force), + Generate.readme(pkg,user,force=force), + Generate.entrypoint(pkg,force=force), + Generate.tests(pkg,force=force), + Generate.require(pkg,force=force), + Generate.travis(pkg,force=force), + Generate.appveyor(pkg,force=force), + Generate.gitignore(pkg,force=force) ] msg = """ $pkg.jl $(isnew ? "generated" : "regenerated") files. @@ -69,14 +79,14 @@ function package( Julia Version $VERSION [$(Base.GIT_VERSION_INFO.commit_short)] """ - + LibGit2.add!(repo, files..., flags = LibGit2.GitConst.INDEX_ADD_FORCE) if isnew info("Committing $pkg generated files") - Git.run(`commit -q -m $msg`, dir=pkg) - elseif Git.dirty(dir=pkg) - Git.run(`reset -q --`, dir=pkg) + LibGit2.commit(repo, msg) + elseif LibGit2.isdirty(repo) + LibGit2.remove!(repo, files...) info("Regenerated files left unstaged, use `git add -p` to select") - open(io->print(io,msg), joinpath(Git.dir(pkg),"MERGE_MSG"), "w") + open(io->print(io,msg), joinpath(LibGit2.gitdir(pkg, repo),"MERGE_MSG"), "w") else info("Regenerated files are unchanged") end @@ -85,37 +95,63 @@ function package( isnew && rm(pkg, recursive=true) rethrow() end + return end function init(pkg::AbstractString, url::AbstractString=""; config::Dict=Dict()) if !ispath(pkg) info("Initializing $pkg repo: $(abspath(pkg))") - Git.run(`init -q $pkg`) - - for (key,val) in config - Git.run(`config $key $val`, dir=pkg) + repo = LibGit2.init(pkg) + try + with(GitConfig, repo) do cfg + for (key,val) in config + LibGit2.set!(cfg, key, val) + end + end + LibGit2.commit(repo, "initial empty commit") + catch err + error("Unable to initialize $pkg package: $err") + end + else + repo = GitRepo(pkg) + end + try + if !isempty(url) + info("Origin: $url") + with(LibGit2.GitRemote, repo, "origin", url) do rmt + LibGit2.save(rmt) + end + LibGit2.set_remote_url(repo, url) end - Git.run(`commit -q --allow-empty -m "initial empty commit"`, dir=pkg) end - isempty(url) && return - info("Origin: $url") - Git.run(`remote add origin $url`,dir=pkg) - Git.set_remote_url(url,dir=pkg) - Git.run(`config branch.master.remote origin`, dir=pkg) - Git.run(`config branch.master.merge refs/heads/master`, dir=pkg) + return repo end -function license(pkg::AbstractString, license::AbstractString, - years::Union{Int,AbstractString}, - authors::Union{AbstractString,Array}; +function genfile(f::Function, pkg::AbstractString, file::AbstractString, force::Bool=false) + path = joinpath(pkg,file) + if force || !ispath(path) + info("Generating $file") + mkpath(dirname(path)) + open(f, path, "w") + return file + end + return "" +end + +function license(pkg::AbstractString, + license::AbstractString, + years::Union(Int,AbstractString), + authors::Union(AbstractString,Array); force::Bool=false) - genfile(pkg,"LICENSE.md",force) do io + file = genfile(pkg,"LICENSE.md",force) do io if !haskey(LICENSES,license) licenses = join(sort!(collect(keys(LICENSES)), by=lowercase), ", ") error("$license is not a known license choice, choose one of: $licenses.") end print(io, LICENSES[license](pkg, string(years), authors)) - end || info("License file exists, leaving unmodified; use `force=true` to overwrite") + end + isempty(file) || info("License file exists, leaving unmodified; use `force=true` to overwrite") + file end function readme(pkg::AbstractString, user::AbstractString=""; force::Bool=false) @@ -253,18 +289,6 @@ function entrypoint(pkg::AbstractString; force::Bool=false) end end -function genfile(f::Function, pkg::AbstractString, file::AbstractString, force::Bool=false) - path = joinpath(pkg,file) - if force || !ispath(path) - info("Generating $file") - mkpath(dirname(path)) - open(f, path, "w") - Git.run(`add -f $file`, dir=pkg) - return true - end - return false -end - copyright(years::AbstractString, authors::AbstractString) = "> Copyright (c) $years: $authors." function copyright(years::AbstractString, authors::Array) diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 70b7f4db49ed4..3138e6e5b4866 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -1,6 +1,7 @@ module LibGit2 -export with_libgit2 +export with_libgit2, with, with_warn +export GitRepo, GitConfig, GitIndex const GITHUB_REGEX = r"^(?:git@|git://|https://(?:[\w\.\+\-]+@)?)github.com[:/](([^/].+)/(.+?))(?:\.git)?$"i @@ -25,6 +26,8 @@ include("libgit2/repository.jl") include("libgit2/config.jl") include("libgit2/walker.jl") include("libgit2/remote.jl") +include("libgit2/strarray.jl") +include("libgit2/index.jl") function need_update(repo::GitRepo) if !isbare(repo) @@ -99,7 +102,7 @@ function is_ancestor_of(a::AbstractString, b::AbstractString, repo::GitRepo) end function set_remote_url(repo::GitRepo, url::AbstractString; remote::AbstractString="origin") - with_libgit2(GitConfig, repo) do cfg + with(GitConfig, repo) do cfg set!(cfg, "remote.$remote.url", url) m = match(GITHUB_REGEX,url) @@ -113,7 +116,7 @@ function set_remote_url(repo::GitRepo, url::AbstractString; remote::AbstractStri end function set_remote_url(path::AbstractString, url::AbstractString; remote::AbstractString="origin") - with_libgit2(GitRepo, path) do repo + with(GitRepo, path) do repo set_remote_url(repo, url, remote=remote) end end @@ -141,7 +144,7 @@ const mirror_cb = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, P function fetch(repo::GitRepo, remote::AbstractString="origin") rmt = get(GitRemote, repo, remote) - with_libgit2(rmt, warn_on_exception=true) do rmt + with_warn(rmt) do rmt @check ccall((:git_remote_fetch, :libgit2), Cint, (Ptr{Void}, Ptr{Void}, Ptr{UInt8}), rmt.ptr, C_NULL, C_NULL) @@ -182,4 +185,60 @@ function normalize_url(url::AbstractString) m == nothing ? url : "git://github.com/$(m.captures[1]).git" end -end \ No newline at end of file +immutable State + head::Oid + index::Oid + work::Oid +end + +function snapshot(repo::GitRepo; dir="") + head = Oid(repo, "HEAD") + index = with(GitIndex, repo) do idx; write_tree!(idx) end + work = try + with(GitIndex, repo) do idx + content = readdir(abspath(dir)) + if length(content) > 1 + files = [utf8(bytestring(c))::UTF8String for c in content] + push!(files, utf8(".")) + + add!(idx, files..., flags = GitConst.INDEX_ADD_CHECK_PATHSPEC) + write!(idx) + end + write_tree!(idx) + end + finally + # restore index + with(GitIndex, repo) do idx + read_tree!(idx, index) + write!(idx) + end + end + State(head, index, work) +end + +function restore(s::State, repo::GitRepo; dir="") + run(`reset -q --`, dir=dir) # unstage everything + run(`read-tree $(s.work)`, dir=dir) # move work tree to index + run(`checkout-index -fa`, dir=dir) # check the index out to work + run(`clean -qdf`, dir=dir) # remove everything else + run(`read-tree $(s.index)`, dir=dir) # restore index + run(`reset -q --soft $(s.head)`, dir=dir) # restore head +end + +function transact(f::Function, repo::GitRepo; dir="") + #state = snapshot(repo, dir=dir) + try f(repo) catch + #restore(state, repo, dir=dir) + rethrow() + finally + finalize(repo) + end +end + +function gitdir(d, repo::GitRepo) + g = joinpath(d,".git") + isdir(g) && return g + path(repo) +end + +end # module \ No newline at end of file diff --git a/base/pkg/libgit2/commit.jl b/base/pkg/libgit2/commit.jl index c71e4eb20c462..4ee01f36de66e 100644 --- a/base/pkg/libgit2/commit.jl +++ b/base/pkg/libgit2/commit.jl @@ -44,7 +44,7 @@ function commit(repo::GitRepo, end """Commit changes to repository""" -function commit(repo::GitRepo, msg::AbstractString, +function commit(repo::GitRepo, msg::AbstractString; refname::AbstractString="HEAD", author::Signature = Signature(repo), committer::Signature = Signature(repo), @@ -52,13 +52,13 @@ function commit(repo::GitRepo, msg::AbstractString, parent_ids::Vector{Oid}=Oid[]) # Retrieve tree identifier if iszero(tree_id) - idx = GitIndex(repo) - try - tree_id = write_tree!(idx) - catch err - rethrow(err) - finally - finalize(idx) + tree_id = with(GitIndex, repo) do idx; write_tree!(idx) end + end + + # Retrieve parents from HEAD + if length(parent_ids) == 0 + try # if throws then HEAD not found -> empty repo + push!(parent_ids, Oid(repo, refname)) end end @@ -71,7 +71,7 @@ function commit(repo::GitRepo, msg::AbstractString, comm_sig = convert(GitSignature, committer) parents = GitCommit[] try - for parent in parents + for parent in parent_ids push!(parents, get(GitCommit, repo, parent)) end commit_id = commit(repo, refname, msg, auth_sig, comm_sig, tree, parents...) @@ -81,7 +81,7 @@ function commit(repo::GitRepo, msg::AbstractString, end finalize(tree) finalize(auth_sig) - finalize(auth_sig) + finalize(comm_sig) end return commit_id end \ No newline at end of file diff --git a/base/pkg/libgit2/const.jl b/base/pkg/libgit2/const.jl index 43f442410d469..0e36203099c56 100644 --- a/base/pkg/libgit2/const.jl +++ b/base/pkg/libgit2/const.jl @@ -160,7 +160,7 @@ module GitConst const INDEXCAP_NO_SYMLINKS = Cuint(4) const INDEXCAP_FROM_OWNER = ~(Cuint(0)) - const INDEX_ADD_DEFAULT = Cint(0) + const INDEX_ADD_DEFAULT = Cuint(0) const INDEX_ADD_FORCE = Cuint(1) << Cint(0) const INDEX_ADD_DISABLE_PATHSPEC_MATCH = Cuint(1) << Cint(1) const INDEX_ADD_CHECK_PATHSPEC = Cuint(1) << Cint(2) diff --git a/base/pkg/libgit2/index.jl b/base/pkg/libgit2/index.jl new file mode 100644 index 0000000000000..ceb8a5660d2a7 --- /dev/null +++ b/base/pkg/libgit2/index.jl @@ -0,0 +1,99 @@ +function GitIndex(repo::GitRepo) + idx_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_repository_index, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}), idx_ptr_ptr, repo.ptr) + return GitIndex(idx_ptr_ptr[]) +end + +function write!(idx::GitIndex) + @check ccall((:git_index_write, :libgit2), Cint, (Ptr{Void},), idx.ptr) + return idx +end + +function write_tree!(idx::GitIndex) + oid_ptr = Ref(Oid()) + @check ccall((:git_index_write_tree, :libgit2), Cint, + (Ptr{Oid}, Ptr{Void}), oid_ptr, idx.ptr) + return oid_ptr[] +end + +function owner(idx::GitIndex) + repo_ptr = ccall((:git_index_owner, :libgit2), Ptr{Void}, + (Ptr{Void},), idx.ptr) + return GitRepo(repo_ptr) +end + +function read_tree!(idx::GitIndex, tree_id::Oid) + repo = owner(idx) + tree = get(GitTree, repo, tree_id) + try + @check ccall((:git_index_read_tree, :libgit2), Cint, + (Ptr{Void}, Ptr{Void}), idx.ptr, tree.ptr) + catch err + rethrow(err) + finally + finalize(tree) + end +end + +function add!{T<:AbstractString}(idx::GitIndex, files::T...; + flags::Cuint = GitConst.INDEX_ADD_DEFAULT) + sa = StrArrayStruct(files...) + try + @check ccall((:git_index_add_all, :libgit2), Cint, + (Ptr{Void}, Ptr{StrArrayStruct}, Cuint, Ptr{Void}, Ptr{Void}), + idx.ptr, Ref(sa), flags, C_NULL, C_NULL) + catch err + rethrow(err) + finally + finalize(sa) + end +end + +function update!{T<:AbstractString}(idx::GitIndex, files::T...) + sa = StrArrayStruct(files...) + try + @check ccall((:git_index_update_all, :libgit2), Cint, + (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{Void}, Ptr{Void}), + idx.ptr, Ref(sa), C_NULL, C_NULL) + catch err + rethrow(err) + finally + finalize(sa) + end +end + +function remove!{T<:AbstractString}(idx::GitIndex, files::T...) + sa = StrArrayStruct(files...) + try + @check ccall((:git_index_remove_all, :libgit2), Cint, + (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{Void}, Ptr{Void}), + idx.ptr, Ref(sa), C_NULL, C_NULL) + catch err + rethrow(err) + finally + finalize(sa) + end +end + +function add!{T<:AbstractString}(repo::GitRepo, files::T...; + flags::Cuint = GitConst.INDEX_ADD_DEFAULT) + with(GitIndex, repo) do idx + add!(idx, files..., flags = flags) + write!(idx) + end +end + +function update!{T<:AbstractString}(repo::GitRepo, files::T...) + with(GitIndex, repo) do idx + update!(idx, files...) + write!(idx) + end +end + +function remove!{T<:AbstractString}(repo::GitRepo, files::T...) + with(GitIndex, repo) do idx + remove!(idx, files...) + write!(idx) + end +end \ No newline at end of file diff --git a/base/pkg/libgit2/oid.jl b/base/pkg/libgit2/oid.jl index b7e53d782291d..60675082c6176 100644 --- a/base/pkg/libgit2/oid.jl +++ b/base/pkg/libgit2/oid.jl @@ -53,7 +53,7 @@ function Oid(id::AbstractString) end function Oid(ref::GitReference) - ref == nothing && return Oid() + isempty(ref) && return Oid() typ = ccall((:git_reference_type, :libgit2), Cint, (Ptr{Void},), ref.ptr) typ != 1 && return Oid() @@ -62,6 +62,15 @@ function Oid(ref::GitReference) return Oid(oid_ptr) end +function Oid(repo::GitRepo, ref_name::AbstractString) + isempty(repo) && return Oid() + oid_ptr = Ref(Oid()) + @check ccall((:git_reference_name_to_id, :libgit2), Cint, + (Ptr{Oid}, Ptr{Void}, Ptr{UInt8}), + oid_ptr, repo.ptr, ref_name) + return oid_ptr[] +end + function Oid(obj::Ptr{Void}) oid_ptr = ccall((:git_object_id, :libgit2), Ptr{UInt8}, (Ptr{Void},), obj) oid_ptr == C_NULL && return Oid() diff --git a/base/pkg/libgit2/reference.jl b/base/pkg/libgit2/reference.jl index 0560b9d4f0c4f..a0856328bbf43 100644 --- a/base/pkg/libgit2/reference.jl +++ b/base/pkg/libgit2/reference.jl @@ -14,4 +14,12 @@ function fullname(ref::GitReference) rname = ccall((:git_reference_symbolic_target, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) rname == C_NULL && return "" return bytestring(rname) +end + +function GitReference(repo::GitRepo, ref_name::AbstractString) + ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_reference_lookup, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), + ref_ptr_ptr, repo.ptr, ref_name) + return GitReference(ref_ptr_ptr[]) end \ No newline at end of file diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index 9f52c7cd0ec3b..3b615d46c8b83 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -69,4 +69,9 @@ end function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::AbstractString) return get(T, r, Oid(oid), length(oid) != OID_HEXSZ) +end + +function path(repo::GitRepo) + return bytestring(ccall((:git_repository_path, :libgit2), Ptr{UInt8}, + (Ptr{Void},), repo.ptr)) end \ No newline at end of file diff --git a/base/pkg/libgit2/strarray.jl b/base/pkg/libgit2/strarray.jl new file mode 100644 index 0000000000000..b604f2d9b2951 --- /dev/null +++ b/base/pkg/libgit2/strarray.jl @@ -0,0 +1,29 @@ +function StrArrayStruct{T<:AbstractString}(strs::T...) + count = length(strs) + strings = convert(Ptr{Ptr{Uint8}}, Libc.malloc(sizeof(Ptr{Uint8}) * count)) + for i=1:count + len = length(strs[i]) + #in_ptr = convert(Ptr{Uint8}, bytestring(strs[i])) + in_ptr = pointer(bytestring(strs[i])) + out_ptr = convert(Ptr{Uint8}, Libc.malloc(sizeof(Uint8) * (len + 1))) + unsafe_copy!(out_ptr, in_ptr, len) + unsafe_store!(out_ptr, zero(Uint8), len + 1) # NULL byte + unsafe_store!(strings, out_ptr, i) + end + return StrArrayStruct(strings, count) +end +StrArrayStruct{T<:AbstractString}(strs::Vector{T}) = StrArrayStruct(strs...) + +function Base.convert(::Type{Vector{AbstractString}}, sa::StrArrayStruct) + arr = Array(AbstractString, sa.count) + for i=1:sa.count + arr[i] = bytestring(unsafe_load(sa.strings, i)) + end + return arr +end + +function finalize(sa::StrArrayStruct) + sa_ptr = Ref(sa) + ccall((:git_strarray_free, :libgit2), Void, (Ptr{StrArrayStruct},), sa_ptr) + return sa_ptr[] +end \ No newline at end of file diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index e517079c4684a..bbdea9bff9a4c 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -174,11 +174,20 @@ function with_libgit2(f::Function, obj) end end -function with_libgit2{T}(f::Function, ::Type{T}, args...; warn_on_exception::Bool=true) +function with{T}(f::Function, ::Type{T}, args...; warn_on_exception::Bool=true) obj = T(args...) try with_libgit2(f, obj) catch err - warn_on_exception ? warn("$(string(T)) thrown exception: $err") : rethrow(err) + rethrow(err) + end +end + +function with_warn{T}(f::Function, ::Type{T}, args...) + obj = T(args...) + try + with_libgit2(f, obj) + catch err + warn("$(string(T)) thrown exception: $err") end end \ No newline at end of file From 319851962f3c4b6ca4bbadedbd6e7ca178205b57 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sun, 10 May 2015 14:51:30 -0400 Subject: [PATCH 0292/1938] added default value to GitConfig 'get' to prevent exceptions --- base/pkg/generate.jl | 6 +++--- base/pkg/github.jl | 6 ++++-- base/pkg/libgit2/config.jl | 4 ++++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index 475c7bd1c528c..8792f83c757d9 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -2,16 +2,16 @@ module Generate -import ..Git, ..Read, ..LibGit2 +import ..LibGit2, ..Read importall ..LibGit2 copyright_year() = Dates.year(Dates.today()) copyright_name(repo::GitRepo) = with(GitConfig, repo) do cfg - LibGit2.get(AbstractString, cfg, "user.name") + LibGit2.get(cfg, "user.name", "") end github_user() = with(GitConfig) do cfg - LibGit2.get(AbstractString, cfg, "github.user") + LibGit2.get(cfg, "github.user", "") end function git_contributors(repo::GitRepo, n::Int=typemax(Int)) diff --git a/base/pkg/github.jl b/base/pkg/github.jl index 3ca072eba5129..f827def0af915 100644 --- a/base/pkg/github.jl +++ b/base/pkg/github.jl @@ -12,8 +12,10 @@ const AUTH_DATA = Dict{Any,Any}( ) function user() - usr = LibGit2.get(AbstractString, LibGit2.GitConfig(), "github.user") - if usr == nothing + usr = LibGit2.with(LibGit2.GitConfig) do cfg + LibGit2.get(cfg, "github.user", "") + end + if isempty(usr) error(""" no GitHub user name configured; please configure it with: diff --git a/base/pkg/libgit2/config.jl b/base/pkg/libgit2/config.jl index 5e1f88c3c8a45..6fb84ffaa5804 100644 --- a/base/pkg/libgit2/config.jl +++ b/base/pkg/libgit2/config.jl @@ -47,6 +47,10 @@ function get{T}(::Type{T}, c::GitConfig, name::AbstractString) end end +function get{T}(c::GitConfig, name::AbstractString, default::T) + return try get(T,c,name) catch default end +end + function set!{T}(c::GitConfig, name::AbstractString, value::T) err = if T<:AbstractString @check ccall((:git_config_set_string, :libgit2), Cint, From 3305ed62d7e105e9c232391a62764bbc59eb52cd Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sun, 10 May 2015 14:54:29 -0400 Subject: [PATCH 0293/1938] added anyobject type and `checkout` functions --- base/pkg/libgit2/repository.jl | 39 ++++++++++++++++++++++++++++++++-- base/pkg/libgit2/types.jl | 2 +- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index 3b615d46c8b83..96958def06079 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -40,7 +40,9 @@ function revparse(repo::GitRepo, obj::AbstractString) obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_revparse_single, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), obj_ptr_ptr, repo.ptr, obj) - return Oid(obj_ptr_ptr[]) + oid = Oid(obj_ptr_ptr[]) + finalize(GitAnyObject(obj_ptr_ptr[])) + return oid end function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::Oid, prefix::Bool=false) @@ -51,6 +53,8 @@ function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::Oid, prefix::Bool=false GitConst.OBJ_COMMIT elseif T == GitTree GitConst.OBJ_TREE + elseif T == GitAnyObject + GitConst.OBJ_ANY else error("Type $T is not supported") end @@ -74,4 +78,35 @@ end function path(repo::GitRepo) return bytestring(ccall((:git_repository_path, :libgit2), Ptr{UInt8}, (Ptr{Void},), repo.ptr)) -end \ No newline at end of file +end + +function checkout(repo::GitRepo, spec::AbstractString) + try + obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_revparse_single, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), + obj_ptr_ptr, repo.ptr, spec) + obj = GitAnyObject(obj_ptr_ptr[]) + try + checkout_tree(repo, obj) + catch err + rethrow(err) + finally + finalize(obj) + end + catch err + warn("'checkout' thrown exception: $err") + end +end + +function checkout_tree(repo::GitRepo, obj::GitAnyObject) + @check ccall((:git_checkout_tree, :libgit2), Cint, + (Ptr{Void}, Ptr{Void}, Ptr{Void}), + repo.ptr, obj.ptr, C_NULL) +end + +function checkout_index(repo::GitRepo, idx::GitIndex) + @check ccall((:git_checkout_index, :libgit2), Cint, + (Ptr{Void}, Ptr{Void}, Ptr{Void}), + repo.ptr, obj.ptr, C_NULL) +end diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index bbdea9bff9a4c..dfaa43f0aa092 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -143,7 +143,7 @@ function finalize(o::GitObject) end Base.isempty(obj::GitObject) = (obj.ptr == C_NULL) -for typ in [:GitCommit, :GitTree] +for typ in [:GitAnyObject, :GitCommit, :GitTree] @eval type $typ <: GitObject ptr::Ptr{Void} function $typ(ptr::Ptr{Void}) From 071a872600f376180d04a7d83b6e8515945333bb Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sun, 10 May 2015 14:55:54 -0400 Subject: [PATCH 0294/1938] no more `git` in Write --- base/pkg/libgit2.jl | 47 ++++++++++++++++++++++++++++++-------- base/pkg/libgit2/remote.jl | 8 +++++++ base/pkg/write.jl | 24 +++++++++++++------ 3 files changed, 62 insertions(+), 17 deletions(-) diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 3138e6e5b4866..93fd868a1723b 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -141,13 +141,31 @@ function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, name::Ptr{ end const mirror_cb = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}, Ptr{Void})) +function fetch(rmt::GitRemote) + @check ccall((:git_remote_fetch, :libgit2), Cint, + (Ptr{Void}, Ptr{Void}, Ptr{UInt8}), + rmt.ptr, C_NULL, C_NULL) +end + function fetch(repo::GitRepo, remote::AbstractString="origin") rmt = get(GitRemote, repo, remote) + try + fetch(rmt) + catch err + warn("'fetch' thrown exception: $err") + finally + finalize(rmt) + end +end - with_warn(rmt) do rmt - @check ccall((:git_remote_fetch, :libgit2), Cint, - (Ptr{Void}, Ptr{Void}, Ptr{UInt8}), - rmt.ptr, C_NULL, C_NULL) +function fetch(repo::GitRepo, remote_path::AbstractString, refspecs::AbstractString) + rmt = GitRemoteAnon(repo, remote_path, refspecs) + try + fetch(rmt) + catch err + warn("'fetch' thrown exception: $err") + finally + finalize(rmt) end end @@ -217,14 +235,23 @@ function snapshot(repo::GitRepo; dir="") end function restore(s::State, repo::GitRepo; dir="") - run(`reset -q --`, dir=dir) # unstage everything - run(`read-tree $(s.work)`, dir=dir) # move work tree to index - run(`checkout-index -fa`, dir=dir) # check the index out to work - run(`clean -qdf`, dir=dir) # remove everything else - run(`read-tree $(s.index)`, dir=dir) # restore index - run(`reset -q --soft $(s.head)`, dir=dir) # restore head + with(GitIndex, repo) do idx + #TODO: reset -q -- # unstage everything + + read_tree!(idx, s.work) # move work tree to index + write!(idx) + + #TODO: checkout-index -fa # check the index out to work + #TODO: clean -qdf # remove everything else + + read_tree!(idx, s.index) # restore index + write!(idx) + + #TODO: reset -q --soft $(s.head) # restore head + end end +# TODO: restore required function transact(f::Function, repo::GitRepo; dir="") #state = snapshot(repo, dir=dir) try f(repo) catch diff --git a/base/pkg/libgit2/remote.jl b/base/pkg/libgit2/remote.jl index 691e53a9fcc1d..a5bd95625d6d1 100644 --- a/base/pkg/libgit2/remote.jl +++ b/base/pkg/libgit2/remote.jl @@ -6,6 +6,14 @@ function GitRemote(repo::GitRepo, rmt_name::AbstractString, rmt_url::AbstractStr return GitRemote(rmt_ptr_ptr[]) end +function GitRemoteAnon(repo::GitRepo, url::AbstractString, refspec::AbstractString) + rmt_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_remote_create_anonymous, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}), + rmt_ptr_ptr, repo.ptr, url, refspec) + return GitRemote(rmt_ptr_ptr[]) +end + function get(::Type{GitRemote}, repo::GitRepo, rmt_name::AbstractString) rmt_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_remote_lookup, :libgit2), Cint, diff --git a/base/pkg/write.jl b/base/pkg/write.jl index 4f9c68c7bd16a..02d67a64714cb 100644 --- a/base/pkg/write.jl +++ b/base/pkg/write.jl @@ -2,7 +2,8 @@ module Write -import ..Git, ..Cache, ..Read +import ..LibGit2, ..Cache, ..Read +importall ..LibGit2 function prefetch(pkg::AbstractString, sha1::AbstractString) isempty(Cache.prefetch(pkg, Read.url(pkg), sha1)) && return @@ -11,9 +12,14 @@ end function fetch(pkg::AbstractString, sha1::AbstractString) refspec = "+refs/*:refs/remotes/cache/*" - Git.run(`fetch -q $(Cache.path(pkg)) $refspec`, dir=pkg) - Git.iscommit(sha1, dir=pkg) && return - f = Git.iscommit(sha1, dir=Cache.path(pkg)) ? "fetch" : "prefetch" + cache = Cache.path(pkg) + with(GitRepo, pkg) do repo + LibGit2.fetch(repo, cache, refspec) + LibGit2.iscommit(sha1, repo) + end && return + f = with(GitRepo, cache) do repo + LibGit2.iscommit(sha1, repo) + end ? "fetch" : "prefetch" url = Read.issue_url(pkg) if isempty(url) error("$pkg: $f failed to get commit $(sha1[1:10]), please file a bug report with the package author.") @@ -23,8 +29,10 @@ function fetch(pkg::AbstractString, sha1::AbstractString) end function checkout(pkg::AbstractString, sha1::AbstractString) - Git.set_remote_url(Read.url(pkg), dir=pkg) - Git.run(`checkout -q $sha1`, dir=pkg) + with(GitRepo, pkg) do repo + LibGit2.set_remote_url(Read.url(pkg)) + LibGit2.checkout(repo, sha1) + end end function install(pkg::AbstractString, sha1::AbstractString) @@ -32,7 +40,9 @@ function install(pkg::AbstractString, sha1::AbstractString) if isdir(".trash/$pkg") mv(".trash/$pkg", "./$pkg") else - Git.run(`clone -q $(Cache.path(pkg)) $pkg`) + with(GitRepo, pkg) do repo + LibGit2.clone(repo, Cache.path(pkg), pkg) + end end fetch(pkg, sha1) checkout(pkg, sha1) From 87e7fbf0fe72dd6f8490c16d1804094a978aab67 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sun, 10 May 2015 18:36:27 -0400 Subject: [PATCH 0295/1938] fixed `revparse`, `isdirty` & errors in `Write` --- base/pkg/libgit2.jl | 24 +++++++++++++----------- base/pkg/libgit2/const.jl | 4 ++-- base/pkg/libgit2/repository.jl | 5 +++-- base/pkg/libgit2/types.jl | 32 ++++++++++++++++++++++++++++++++ base/pkg/read.jl | 2 +- base/pkg/write.jl | 6 ++---- 6 files changed, 53 insertions(+), 20 deletions(-) diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 93fd868a1723b..4cb4f34d5f553 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -53,28 +53,30 @@ end function isdirty(repo::GitRepo, paths::AbstractString="") tree_oid = revparse(repo, "HEAD^{tree}") tree_oid == nothing && return true + emptypathspec = isempty(paths) result = false tree = get(GitTree, repo, tree_oid) + if !emptypathspec + sa = StrArrayStruct(paths) + diff_opts = DiffOptionsStruct() + diff_opts.pathspec = sa + end try diff_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_diff_tree_to_workdir_with_index, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, Ptr{Void}), - diff_ptr_ptr, repo.ptr, tree.ptr, C_NULL, C_NULL) + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{DiffOptionsStruct}), + diff_ptr_ptr, repo.ptr, tree.ptr, emptypathspec ? C_NULL : Ref(diff_opts)) diff = GitDiff(diff_ptr_ptr[]) - if isempty(paths) - c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff.ptr) - result = c > 0 - else - # TODO look for specified path - c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff.ptr) - result = c > 0 - end + c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff.ptr) + result = c > 0 finalize(diff) - catch + catch err + warn(err) result = true finally + !emptypathspec && finalize(sa) finalize(tree) end return result diff --git a/base/pkg/libgit2/const.jl b/base/pkg/libgit2/const.jl index 0e36203099c56..54534bfb66ee0 100644 --- a/base/pkg/libgit2/const.jl +++ b/base/pkg/libgit2/const.jl @@ -75,9 +75,9 @@ module GitConst const GIT_PATH_MAX = Cint(4096) - const DIFF_OPTIONS_VERSION = Cint(1) + const DIFF_OPTIONS_VERSION = Cuint(1) - const DIFF_NORMAL = Cint(0) + const DIFF_NORMAL = Cuint(0) const DIFF_REVERSE = Cuint(1) << Cint(0) const DIFF_INCLUDE_IGNORED = Cuint(1) << Cint(1) const DIFF_RECURSE_IGNORED_DIRS = Cuint(1) << Cint(2) diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index 96958def06079..2ef3255bf046f 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -38,8 +38,9 @@ end function revparse(repo::GitRepo, obj::AbstractString) obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - @check ccall((:git_revparse_single, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), obj_ptr_ptr, repo.ptr, obj) + err = ccall((:git_revparse_single, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), obj_ptr_ptr, repo.ptr, obj) + err != 0 && return Oid() oid = Oid(obj_ptr_ptr[]) finalize(GitAnyObject(obj_ptr_ptr[])) return oid diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index dfaa43f0aa092..6b234861c5a63 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -104,6 +104,38 @@ CloneOptionsStruct() = CloneOptionsStruct(one(Cuint), Ptr{Void}(0), Ptr{Void}(0) ) +# git diff option struct +type DiffOptionsStruct + version::Cuint + flags::UInt32 + + # options controlling which files are in the diff + ignore_submodules::Cint + pathspec::StrArrayStruct + notify_cb::Ptr{Void} + notify_payload::Ptr{Void} + + # options controlling how the diff text is generated + context_lines::UInt32 + interhunk_lines::UInt32 + id_abbrev::UInt16 + max_size::Coff_t + old_prefix::Ptr{UInt8} + new_prefix::Ptr{UInt8} +end +DiffOptionsStruct() = DiffOptionsStruct(GitConst.DIFF_OPTIONS_VERSION, + GitConst.DIFF_NORMAL, + GitConst.SUBMODULE_IGNORE_DEFAULT, + StrArrayStruct(), + Ptr{Void}(0), + Ptr{Void}(0), + UInt32(3), + zero(UInt32), + UInt16(7), + Coff_t(512*1024*1024), #zero(Coff_t), + Ptr{UInt8}(0), + Ptr{UInt8}(0)) + # Common types for (typ, ref, fnc) in ((:GitRemote, :Void, :(:git_remote_free)), (:GitRevWalker, :Void, :(:git_revwalk_free)), diff --git a/base/pkg/read.jl b/base/pkg/read.jl index 6c4bfb66d4b50..29b43e66bbc4f 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -150,7 +150,7 @@ function requires_path(pkg::AbstractString, avail::Dict=available(pkg)) head = "" try LibGit2.isdirty(repo, "REQUIRE") && return pkgreq - LibGit2.revparse(repo, "HEAD:REQUIRE") == nothing && isfile(pkgreq) && return pkgreq + LibGit2.iszero(LibGit2.revparse(repo, "HEAD:REQUIRE")) && isfile(pkgreq) && return pkgreq head = string(LibGit2.head_oid(repo)) finally LibGit2.finalize(repo) diff --git a/base/pkg/write.jl b/base/pkg/write.jl index 02d67a64714cb..f49c92e3d3a27 100644 --- a/base/pkg/write.jl +++ b/base/pkg/write.jl @@ -30,7 +30,7 @@ end function checkout(pkg::AbstractString, sha1::AbstractString) with(GitRepo, pkg) do repo - LibGit2.set_remote_url(Read.url(pkg)) + LibGit2.set_remote_url(repo, Read.url(pkg)) LibGit2.checkout(repo, sha1) end end @@ -40,9 +40,7 @@ function install(pkg::AbstractString, sha1::AbstractString) if isdir(".trash/$pkg") mv(".trash/$pkg", "./$pkg") else - with(GitRepo, pkg) do repo - LibGit2.clone(repo, Cache.path(pkg), pkg) - end + LibGit2.clone(Cache.path(pkg), pkg) end fetch(pkg, sha1) checkout(pkg, sha1) From 409bb27934fe946fed6e25dc709c6b2dfca3cf2a Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sun, 10 May 2015 19:18:15 -0400 Subject: [PATCH 0296/1938] improved git types description --- base/pkg/libgit2/types.jl | 69 +++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index 6b234861c5a63..322394a0fb512 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -136,16 +136,34 @@ DiffOptionsStruct() = DiffOptionsStruct(GitConst.DIFF_OPTIONS_VERSION, Ptr{UInt8}(0), Ptr{UInt8}(0)) +# Abstract object types +abstract AbstractGitObject +Base.isempty(obj::AbstractGitObject) = (obj.ptr == C_NULL) + +abstract GitObject <: AbstractGitObject +function finalize(obj::GitObject) + if obj.ptr != C_NULL + ccall((:git_object_free, :libgit2), Void, (Ptr{Void},), obj.ptr) + obj.ptr = C_NULL + end +end + # Common types -for (typ, ref, fnc) in ((:GitRemote, :Void, :(:git_remote_free)), - (:GitRevWalker, :Void, :(:git_revwalk_free)), - (:GitConfig, :Void, :(:git_config_free)), - (:GitReference, :Void, :(:git_reference_free)), - (:GitDiff, :Void, :(:git_diff_free)), - (:GitIndex, :Void, :(:git_index_free)), - (:GitSignature, :SignatureStruct, :(:git_signature_free)), - (:GitRepo, :Void, :(:git_repository_free))) - @eval type $typ +for (typ, ref, sup, fnc) in ( + (:GitRemote, :Void, :AbstractGitObject, :(:git_remote_free)), + (:GitRevWalker, :Void, :AbstractGitObject, :(:git_revwalk_free)), + (:GitConfig, :Void, :AbstractGitObject, :(:git_config_free)), + (:GitReference, :Void, :AbstractGitObject, :(:git_reference_free)), + (:GitDiff, :Void, :AbstractGitObject, :(:git_diff_free)), + (:GitIndex, :Void, :AbstractGitObject, :(:git_index_free)), + (:GitRepo, :Void, :AbstractGitObject, :(:git_repository_free)), + (:GitSignature, :SignatureStruct, :AbstractGitObject, :(:git_signature_free)), + (:GitAnyObject, :Void, :GitObject, nothing), + (:GitCommit, :Void, :GitObject, nothing), + (:GitTree, :Void, :GitObject, nothing) + ) + + @eval type $typ <: $sup ptr::Ptr{$ref} function $typ(ptr::Ptr{$ref}) @assert ptr != C_NULL @@ -154,36 +172,15 @@ for (typ, ref, fnc) in ((:GitRemote, :Void, :(:git_remote_free)), end end - @eval function finalize(obj::$typ) - if obj.ptr != C_NULL - ccall(($fnc, :libgit2), Void, (Ptr{$ref},), obj.ptr) - obj.ptr = C_NULL + if fnc != nothing + @eval function finalize(obj::$typ) + if obj.ptr != C_NULL + ccall(($fnc, :libgit2), Void, (Ptr{$ref},), obj.ptr) + obj.ptr = C_NULL + end end end - @eval Base.isempty(obj::$typ) = (obj.ptr == C_NULL) -end - -# Object types -abstract GitObject - -function finalize(o::GitObject) - if o.ptr != C_NULL - ccall((:git_object_free, :libgit2), Void, (Ptr{Void},), o.ptr) - o.ptr = C_NULL - end -end -Base.isempty(obj::GitObject) = (obj.ptr == C_NULL) - -for typ in [:GitAnyObject, :GitCommit, :GitTree] - @eval type $typ <: GitObject - ptr::Ptr{Void} - function $typ(ptr::Ptr{Void}) - @assert ptr != C_NULL - obj = new(ptr) - return obj - end - end end # Structure has the same layout as SignatureStruct From 9802b854fb82df49329b0515e859ea3c7bc3da1e Mon Sep 17 00:00:00 2001 From: Art Wild Date: Mon, 11 May 2015 12:12:07 -0400 Subject: [PATCH 0297/1938] transitioned Entry.add --- base/pkg/entry.jl | 23 +++++++++++++---------- base/pkg/libgit2.jl | 21 ++++++++++++++++++--- base/pkg/libgit2/reference.jl | 14 ++++++++++---- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 2abc6a9c83fda..510882b475c1c 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -4,6 +4,7 @@ module Entry import Base: thispatch, nextpatch, nextminor, nextmajor, check_new_version import ..LibGit2, ..Git, ..Reqs, ..Read, ..Query, ..Resolve, ..Cache, ..Write, ..GitHub, ..Dir +importall ..LibGit2 using ..Types macro recover(ex) @@ -50,18 +51,20 @@ function add(pkg::AbstractString, vers::VersionSet) info("Nothing to be done") end branch = Dir.getmetabranch() - if Git.branch(dir="METADATA") == branch - if !Git.success(`diff --quiet origin/$branch`, dir="METADATA") - outdated = :yes - else - try - run(pipeline(Git.cmd(`fetch -q --all`, dir="METADATA"),stdout=DevNull,stderr=DevNull)) - outdated = Git.success(`diff --quiet origin/$branch`, dir="METADATA") ? - (:no) : (:yes) + outdated = with(GitRepo, "METADATA") do repo + current_barnch = LibGit2.branch_name(repo) + if current_barnch == branch + if LibGit2.isdiff(repo, "origin/$branch") # diff origin/metadata-v2 + outdated = :yes + else + try + LibGit2.fetch(repo) + outdated = LibGit2.isdiff(repo, "origin/$branch") ? (:yes) : (:no) + end end + else + :no # user is doing something funky with METADATA end - else - outdated = :no # user is doing something funky with METADATA end end if outdated != :no diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 4cb4f34d5f553..e3a822aceb15f 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -50,9 +50,13 @@ function iscommit(id::AbstractString, repo::GitRepo) return res end -function isdirty(repo::GitRepo, paths::AbstractString="") - tree_oid = revparse(repo, "HEAD^{tree}") - tree_oid == nothing && return true +""" git diff-index HEAD [-- ]""" +isdirty(repo::GitRepo, paths::AbstractString="") = isdiff(repo, "HEAD", paths) + +""" git diff-index [-- ]""" +function isdiff(repo::GitRepo, treeish::AbstractString, paths::AbstractString="") + tree_oid = revparse(repo, "$treeish^{tree}") + iszero(tree_oid) && return true emptypathspec = isempty(paths) result = false @@ -200,6 +204,17 @@ function authors(repo::GitRepo) return athrs end +function branch_name(repo::GitRepo) + head_ref = head(repo) + brnch = "" + try + brnch = branch(head_ref) + finally + finalize(head_ref) + end + return brnch +end + function normalize_url(url::AbstractString) m = match(GITHUB_REGEX,url) m == nothing ? url : "git://github.com/$(m.captures[1]).git" diff --git a/base/pkg/libgit2/reference.jl b/base/pkg/libgit2/reference.jl index a0856328bbf43..3ec0dc263d360 100644 --- a/base/pkg/libgit2/reference.jl +++ b/base/pkg/libgit2/reference.jl @@ -1,14 +1,12 @@ function shortname(ref::GitReference) - ref == nothing && return "" - + isempty(ref) && return "" name_ptr = ccall((:git_reference_shorthand, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) name_ptr == C_NULL && return "" return bytestring(name_ptr) end function fullname(ref::GitReference) - ref == nothing && return "" - + isempty(ref) && return "" typ = ccall((:git_reference_type, :libgit2), Cint, (Ptr{Void},), ref.ptr) typ == 1 && return "" rname = ccall((:git_reference_symbolic_target, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) @@ -16,6 +14,14 @@ function fullname(ref::GitReference) return bytestring(rname) end +function branch(ref::GitReference) + isempty(ref) && return "" + str_ptr_ptr = Ref{Ptr{UInt8}}(C_NULL) + @check ccall((:git_branch_name, :libgit2), Cint, + (Ref{Ptr{UInt8}}, Ptr{Void},), str_ptr_ptr, ref.ptr) + return bytestring(str_ptr_ptr[]) +end + function GitReference(repo::GitRepo, ref_name::AbstractString) ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_reference_lookup, :libgit2), Cint, From e2c1a38a2fdb3fe64406bdae732c70fd36bdc70a Mon Sep 17 00:00:00 2001 From: Art Wild Date: Mon, 11 May 2015 12:21:39 -0400 Subject: [PATCH 0298/1938] transitioned Entry.clone --- base/pkg/entry.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 510882b475c1c..c874df2bf8715 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -182,8 +182,9 @@ function clone(url::AbstractString, pkg::AbstractString) info("Cloning $pkg from $url") ispath(pkg) && error("$pkg already exists") try - Git.run(`clone -q $url $pkg`) - Git.set_remote_url(url, dir=pkg) + repo = LibGit2.clone(url, pkg) + LibGit2.set_remote_url(repo, url) + LibGit2.finalize(repo) catch Base.rm(pkg, recursive=true) rethrow() From ff9b5d6cde914c67a9770b5a458ab117351a6322 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Mon, 11 May 2015 14:11:06 -0400 Subject: [PATCH 0299/1938] GitError is Exception subclass --- base/pkg/libgit2/error.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/pkg/libgit2/error.jl b/base/pkg/libgit2/error.jl index 8d054fa0acd6f..1ba54b6933335 100644 --- a/base/pkg/libgit2/error.jl +++ b/base/pkg/libgit2/error.jl @@ -1,4 +1,4 @@ -module GitErrorConst +module GitErrorConst #TODO: use @enum & rid of git_error_code const GIT_OK = Cint(0) const ERROR = Cint(-01) const ENOTFOUND = Cint(-03) @@ -72,7 +72,7 @@ immutable ErrorStruct class::Cint end -immutable GitError{Class, Code} +immutable GitError{Class, Code} <: Exception msg::AbstractString end From f55de933c58957f958fba44040561583d008c7a3 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Mon, 11 May 2015 14:15:28 -0400 Subject: [PATCH 0300/1938] fixed `need_update`, `head_oid`, `revparse`, `CHECKOUT_NONE`, `CheckoutOptionsStruct` added index: `read!` & repo: `checkout`, `new_branch` --- base/pkg/libgit2.jl | 89 +++++++++++++++++++++++----------- base/pkg/libgit2/const.jl | 2 +- base/pkg/libgit2/index.jl | 11 +++++ base/pkg/libgit2/repository.jl | 26 ++++++++-- base/pkg/libgit2/types.jl | 2 +- base/pkg/read.jl | 6 ++- base/pkg/write.jl | 1 + 7 files changed, 101 insertions(+), 36 deletions(-) diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index e3a822aceb15f..45749690d0ceb 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -6,15 +6,6 @@ export GitRepo, GitConfig, GitIndex const GITHUB_REGEX = r"^(?:git@|git://|https://(?:[\w\.\+\-]+@)?)github.com[:/](([^/].+)/(.+?))(?:\.git)?$"i -function __init__() - err = ccall((:git_libgit2_init, :libgit2), Cint, ()) - err > 0 || error("error initializing LibGit2 module") - atexit() do - gc() - ccall((:git_libgit2_shutdown, :libgit2), Cint, ()) - end -end - include("libgit2/const.jl") include("libgit2/types.jl") include("libgit2/error.jl") @@ -29,18 +20,33 @@ include("libgit2/remote.jl") include("libgit2/strarray.jl") include("libgit2/index.jl") +immutable State + head::Oid + index::Oid + work::Oid +end + +function normalize_url(url::AbstractString) + m = match(GITHUB_REGEX,url) + m == nothing ? url : "git://github.com/$(m.captures[1]).git" +end + +function gitdir(d, repo::GitRepo) + g = joinpath(d,".git") + isdir(g) && return g + path(repo) +end + function need_update(repo::GitRepo) if !isbare(repo) - "git update-index -q --really-refresh" #TODO: update-index + # read updates index from filesystem + # i.e. "git update-index -q --really-refresh" + read!(repo, true) end end -function isattached(repo::GitRepo) - ccall((:git_repository_head_detached, :libgit2), Cint, (Ptr{Void},), repo.ptr) != 1 -end - +""" Checks if commit is in repository """ function iscommit(id::AbstractString, repo::GitRepo) - need_update(repo) res = true try get(GitCommit, repo, id) @@ -55,7 +61,7 @@ isdirty(repo::GitRepo, paths::AbstractString="") = isdiff(repo, "HEAD", paths) """ git diff-index [-- ]""" function isdiff(repo::GitRepo, treeish::AbstractString, paths::AbstractString="") - tree_oid = revparse(repo, "$treeish^{tree}") + tree_oid = revparseid(repo, "$treeish^{tree}") iszero(tree_oid) && return true emptypathspec = isempty(paths) @@ -103,7 +109,7 @@ function merge_base(one::AbstractString, two::AbstractString, repo::GitRepo) end function is_ancestor_of(a::AbstractString, b::AbstractString, repo::GitRepo) - A = revparse(repo, a) + A = revparseid(repo, a) merge_base(a, b, repo) == A end @@ -215,15 +221,38 @@ function branch_name(repo::GitRepo) return brnch end -function normalize_url(url::AbstractString) - m = match(GITHUB_REGEX,url) - m == nothing ? url : "git://github.com/$(m.captures[1]).git" +function checkout(repo::GitRepo, branch::AbstractString="master"; + strategy::Cuint = GitConst.CHECKOUT_SAFE) + treeish_obj = revparse(repo, branch) + try + opts = CheckoutOptionsStruct() + opts.checkout_strategy = strategy + error = git_checkout_tree(repo, treeish, &opts); + ccall((:git_checkout_tree, :libgit2), Cint, + (Ptr{Void}, Ptr{Void}, Ref{CheckoutOptionsStruct}), + repo.ptr, treeish_obj.ptr, Ref(opts)) + finally + finalize(treeish_obj) + end end -immutable State - head::Oid - index::Oid - work::Oid +function create_branch(repo::GitRepo, branch::AbstractString, + commit_oid::AbstractString, force::Bool=false) + cmt = get(CitCommit, repo, commit_oid) + frc = CInt(force) + oid = Oid() + try + ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_branch_create, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ref{Void}, Cint), + ref_ptr_ptr, repo.ptr, branch, cmt.ptr, CInt(force)) + ref = GitReference(ref_ptr_ptr[]) + oid = Oid(ref.ptr) + finalize(ref) + finally + finalize(treeish_obj) + end + return oid end function snapshot(repo::GitRepo; dir="") @@ -279,10 +308,14 @@ function transact(f::Function, repo::GitRepo; dir="") end end -function gitdir(d, repo::GitRepo) - g = joinpath(d,".git") - isdir(g) && return g - path(repo) + +function __init__() + err = ccall((:git_libgit2_init, :libgit2), Cint, ()) + err > 0 || error("error initializing LibGit2 module") + atexit() do + gc() + ccall((:git_libgit2_shutdown, :libgit2), Cint, ()) + end end end # module \ No newline at end of file diff --git a/base/pkg/libgit2/const.jl b/base/pkg/libgit2/const.jl index 54534bfb66ee0..60f9946429391 100644 --- a/base/pkg/libgit2/const.jl +++ b/base/pkg/libgit2/const.jl @@ -27,7 +27,7 @@ module GitConst const FILEMODE_LINK = Cint(40960) const FILEMODE_COMMIT = Cint(57344) - const CHECKOUT_NONE = Cint(0) + const CHECKOUT_NONE = Cuint(0) const CHECKOUT_SAFE = Cuint(1) << Cint(0) const CHECKOUT_SAFE_CREATE = Cuint(1) << Cint(1) const CHECKOUT_FORCE = Cuint(1) << Cint(2) diff --git a/base/pkg/libgit2/index.jl b/base/pkg/libgit2/index.jl index ceb8a5660d2a7..7b1ed84603f42 100644 --- a/base/pkg/libgit2/index.jl +++ b/base/pkg/libgit2/index.jl @@ -5,6 +5,11 @@ function GitIndex(repo::GitRepo) return GitIndex(idx_ptr_ptr[]) end +function read!(idx::GitIndex, force::Bool = false) + @check ccall((:git_index_read, :libgit2), Cint, (Ptr{Void}, Cint), idx.ptr, Cint(force)) + return idx +end + function write!(idx::GitIndex) @check ccall((:git_index_write, :libgit2), Cint, (Ptr{Void},), idx.ptr) return idx @@ -96,4 +101,10 @@ function remove!{T<:AbstractString}(repo::GitRepo, files::T...) remove!(idx, files...) write!(idx) end +end + +function read!(repo::GitRepo, force::Bool = false) + with(GitIndex, repo) do idx + read!(idx, force) + end end \ No newline at end of file diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index 2ef3255bf046f..f5f3eaaaf18c3 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -30,19 +30,37 @@ function head(repo::GitRepo) (Ptr{Ptr{Void}}, Ptr{Void}), head_ptr_ptr, repo.ptr) return GitReference(head_ptr_ptr[]) end -head_oid(repo::GitRepo) = Oid(head(repo)) + +function head_oid(repo::GitRepo) + head_ref = head(repo) + oid = Oid(head_ref) + finalize(head_ref) + return oid +end function isbare(repo::GitRepo) return ccall((:git_repository_is_bare, :libgit2), Cint, (Ptr{Void},), repo.ptr) == 1 end +function isattached(repo::GitRepo) + ccall((:git_repository_head_detached, :libgit2), Cint, (Ptr{Void},), repo.ptr) != 1 +end + +""" Returns a found object """ function revparse(repo::GitRepo, obj::AbstractString) obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_revparse_single, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), obj_ptr_ptr, repo.ptr, obj) - err != 0 && return Oid() - oid = Oid(obj_ptr_ptr[]) - finalize(GitAnyObject(obj_ptr_ptr[])) + err != 0 && return GitAnyObject(C_NULL) + return GitAnyObject(obj_ptr_ptr[]) +end + +""" Returns id of a found object """ +function revparseid(repo::GitRepo, obj::AbstractString) + obj = revparse(repo, objname) + isempty(obj) && return Oid() + oid = Oid(obj.ptr) + finalize(obj) return oid end diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index 322394a0fb512..f11c2dabc6a83 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -19,7 +19,7 @@ immutable StrArrayStruct end StrArrayStruct() = StrArrayStruct(Ptr{UInt8}(0), zero(Csize_t)) -immutable CheckoutOptionsStruct +type CheckoutOptionsStruct version::Cuint checkout_strategy::Cuint disable_filters::Cint diff --git a/base/pkg/read.jl b/base/pkg/read.jl index 29b43e66bbc4f..74cc7c1854c90 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -62,7 +62,8 @@ function isfixed(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::Dict=availa LibGit2.isdirty(prepo) && return true LibGit2.isattached(prepo) && return true - LibGit2.revparse(prepo, "HEAD:REQUIRE") == nothing && isfile(pkg,"REQUIRE") && return true + LibGit2.need_update(prepo) + LibGit2.iszero(LibGit2.revparseid(prepo, "HEAD:REQUIRE")) && isfile(pkg,"REQUIRE") && return true head = string(LibGit2.head_oid(prepo)) for (ver,info) in avail @@ -150,7 +151,8 @@ function requires_path(pkg::AbstractString, avail::Dict=available(pkg)) head = "" try LibGit2.isdirty(repo, "REQUIRE") && return pkgreq - LibGit2.iszero(LibGit2.revparse(repo, "HEAD:REQUIRE")) && isfile(pkgreq) && return pkgreq + LibGit2.need_update(repo) + LibGit2.iszero(LibGit2.revparseid(repo, "HEAD:REQUIRE")) && isfile(pkgreq) && return pkgreq head = string(LibGit2.head_oid(repo)) finally LibGit2.finalize(repo) diff --git a/base/pkg/write.jl b/base/pkg/write.jl index f49c92e3d3a27..1398382965e8b 100644 --- a/base/pkg/write.jl +++ b/base/pkg/write.jl @@ -15,6 +15,7 @@ function fetch(pkg::AbstractString, sha1::AbstractString) cache = Cache.path(pkg) with(GitRepo, pkg) do repo LibGit2.fetch(repo, cache, refspec) + LibGit2.need_update(repo) LibGit2.iscommit(sha1, repo) end && return f = with(GitRepo, cache) do repo From 6dedbb29d459f65ad79ca7b3ab2a6cc75e794a3b Mon Sep 17 00:00:00 2001 From: Art Wild Date: Mon, 11 May 2015 14:21:19 -0400 Subject: [PATCH 0301/1938] transitioned Entry: `free` & `pin` --- base/pkg/entry.jl | 49 +++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index c874df2bf8715..bd72a9e81092e 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -111,11 +111,11 @@ function installed(pkg::AbstractString) avail = Read.available(pkg) if Read.isinstalled(pkg) res = typemin(VersionNumber) - prepo = LibGit2.GitRepo(pkg) + repo = LibGit2.GitRepo(pkg) try - res = Read.installed_version(pkg, prepo, avail) + res = Read.installed_version(pkg, repo, avail) finally - LibGit2.finalize(prepo) + LibGit2.finalize(repo) end return res end @@ -163,6 +163,7 @@ function status(io::IO, pkg::AbstractString, ver::VersionNumber, fix::Bool) else print(io, string(LibGit2.Oid(phead))[1:8]) end + LibGit2.finalize(phead) attrs = AbstractString[] isfile("METADATA",pkg,"url") || push!(attrs,"unregistered") LibGit2.isdirty(prepo) && push!(attrs,"dirty") @@ -212,7 +213,7 @@ function clone(url_or_pkg::AbstractString) end function _checkout(pkg::AbstractString, what::AbstractString, merge::Bool=false, pull::Bool=false, branch::Bool=false) - Git.transact(dir=pkg) do + Git.transact(dir=pkg) do #TODO: finish restore() Git.dirty(dir=pkg) && error("$pkg is dirty, bailing") branch ? Git.run(`checkout -q -B $what -t origin/$what`, dir=pkg) : Git.run(`checkout -q $what`, dir=pkg) merge && Git.run(`merge -q --ff-only $what`, dir=pkg) @@ -235,17 +236,22 @@ function free(pkg::AbstractString) Read.isinstalled(pkg) || error("$pkg cannot be freed – not an installed package") avail = Read.available(pkg) isempty(avail) && error("$pkg cannot be freed – not a registered package") - Git.dirty(dir=pkg) && error("$pkg cannot be freed – repo is dirty") - info("Freeing $pkg") - vers = sort!(collect(keys(avail)), rev=true) - while true - for ver in vers - sha1 = avail[ver].sha1 - Git.iscommit(sha1, dir=pkg) || continue - return _checkout(pkg,sha1) + repo = LibGit2.GitRepo(pkg) + try + LibGit2.isdirty(repo) && error("$pkg cannot be freed – repo is dirty") + info("Freeing $pkg") + vers = sort!(collect(keys(avail)), rev=true) + while true + for ver in vers + sha1 = avail[ver].sha1 + LibGit2.iscommit(sha1, repo) || continue + return _checkout(pkg, sha1) + end + isempty(Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail])) && continue + error("can't find any registered versions of $pkg to checkout") end - isempty(Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail])) && continue - error("can't find any registered versions of $pkg to checkout") + finally + LibGit2.finalize(repo) end end @@ -275,13 +281,18 @@ end function pin(pkg::AbstractString, head::AbstractString) ispath(pkg,".git") || error("$pkg is not a git repo") - branch = "pinned.$(head[1:8]).tmp" - rslv = (head != Git.head(dir=pkg)) - info("Creating $pkg branch $branch") - Git.run(`checkout -q -B $branch $head`, dir=pkg) + rslv = !isempty(head) # no need to resolve, branch will be from HEAD + with(GitRepo, pkg) do repo + if isempty(head) # get HEAD oid + head = head_oid(repo) + end + branch = "pinned.$(head[1:8]).tmp" + info("Creating $pkg branch $branch") + LibGit2.create_branch(repo, branch, head) + end rslv ? resolve() : nothing end -pin(pkg::AbstractString) = pin(pkg,Git.head(dir=pkg)) +pin(pkg::AbstractString) = pin(pkg, "") function pin(pkg::AbstractString, ver::VersionNumber) ispath(pkg,".git") || error("$pkg is not a git repo") From 8fbf905c0cf18e0ea90923faa5c9906bcbd7f1a0 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Tue, 12 May 2015 21:02:31 -0400 Subject: [PATCH 0302/1938] refactored error handling --- base/pkg/libgit2/error.jl | 138 ++++++++++++++++----------------- base/pkg/libgit2/repository.jl | 4 +- base/pkg/libgit2/walker.jl | 44 ++++++----- 3 files changed, 91 insertions(+), 95 deletions(-) diff --git a/base/pkg/libgit2/error.jl b/base/pkg/libgit2/error.jl index 1ba54b6933335..b77c9a1524271 100644 --- a/base/pkg/libgit2/error.jl +++ b/base/pkg/libgit2/error.jl @@ -1,106 +1,100 @@ -module GitErrorConst #TODO: use @enum & rid of git_error_code - const GIT_OK = Cint(0) - const ERROR = Cint(-01) - const ENOTFOUND = Cint(-03) - const EEXISTS = Cint(-04) - const EAMBIGUOUS = Cint(-05) - const EBUFS = Cint(-06) - const EUSER = Cint(-07) - const EBAREREPO = Cint(-08) - const EUNBORNBRANCH = Cint(-09) - const EUNMERGED = Cint(-10) - const ENONFASTFORWARD = Cint(-11) - const EINVALIDSPEC = Cint(-12) - const EMERGECONFLICT = Cint(-13) - const ELOCKED = Cint(-14) - const PASSTHROUGH = Cint(-30) - const ITEROVER = Cint(-31) -end +module Error + +@enum(Code, GIT_OK = Cint(0), # no error + ERROR = Cint(-01), # generic error + ENOTFOUND = Cint(-03), # requested object could not be found + EEXISTS = Cint(-04), # object exits preventing op + EAMBIGUOUS = Cint(-05), # more than one object matches + EBUFS = Cint(-06), # output buffer too small to hold data + EUSER = Cint(-07), # user callback generated error + EBAREREPO = Cint(-08), # operation not allowed on bare repo + EUNBORNBRANCH = Cint(-09), # HEAD refers to branch with 0 commits + EUNMERGED = Cint(-10), # merge in progress prevented op + ENONFASTFORWARD = Cint(-11), # ref not fast-forwardable + EINVALIDSPEC = Cint(-12), # name / ref not in valid format + EMERGECONFLICT = Cint(-13), # merge conflict prevented op + ELOCKED = Cint(-14), # lock file prevented op + EMODIFIED = Cint(-15), # ref value does not match expected + EAUTH = Cint(-16), # authentication error + ECERTIFICATE = Cint(-17), # server certificate is invalid + EAPPLIED = Cint(-18), # patch/merge has already been applied + EPEEL = Cint(-19), # the requested peel operation is not possible + PASSTHROUGH = Cint(-30), # internal only + ITEROVER = Cint(-31)) # signals end of iteration -const git_error_code = Dict{Int,Symbol}( - 00 => :OK, # no error - -01 => :Error, # generic error - -03 => :NotFound, # requested object could not be found - -04 => :Exists, # object exits preventing op - -05 => :Ambiguous, # more than one object matches - -06 => :Bufs, # output buffer too small to hold data - -07 => :User, # user callback generated error - -08 => :BareRepo, # operation not allowed on bare repo - -09 => :UnbornBranch, # HEAD refers to branch with 0 commits - -10 => :Unmerged, # merge in progress prevented op - -11 => :NonFastForward, # ref not fast-forwardable - -12 => :InvalidSpec, # name / ref not in valid format - -13 => :MergeConflict, # merge conflict prevented op - -14 => :Locked, # lock file prevented op - -15 => :Modified, # ref value does not match expected - -31 => :Iterover # signals end of iteration -) +Base.(:(==))(c::Code, e::Integer) = c.val == e +Base.(:(==))(e::Integer, c::Code) = c.val == e -const git_error_class = Dict{Int,Symbol}( - 0 => :None, - 1 => :NoMemory, - 2 => :OS, - 3 => :Invalid, - 4 => :Ref, - 5 => :Zlib, - 6 => :Repo, - 7 => :Config, - 8 => :Regex, - 9 => :Odb, - 10 => :Index, - 11 => :Object, - 12 => :Net, - 13 => :Tag, - 14 => :Tree, - 15 => :Indexer, - 16 => :SSL, - 17 => :Submodule, - 18 => :Thread, - 19 => :Stash, - 20 => :Checkout, - 21 => :FetchHead, - 22 => :Merge, - 23 => :SSH, - 24 => :Filter, - 25 => :Revert, - 26 => :Callback, - 27 => :CherryPick -) +@enum(Class, None, + NoMemory, + OS, + Invalid, + Reference, + Zlib, + Repository, + Config, + Regex, + Odb, + Index, + Object, + Net, + Tag, + Tree, + Indexer, + SSL, + Submodule, + Thread, + Stash, + Checkout, + FetchHead, + Merge, + SSH, + Filter, + Revert, + Callback, + CherryPick, + Describe, + Rebase) immutable ErrorStruct message::Ptr{UInt8} class::Cint end -immutable GitError{Class, Code} <: Exception +immutable GitError <: Exception + class::Class + code::Code msg::AbstractString end +Base.show(io::IO, err::GitError) = print(io, "[Code:$(err.code), Class:$(err.class)]: $(err.msg)") function last_error() err = ccall((:giterr_last, :libgit2), Ptr{ErrorStruct}, ()) if err != C_NULL err_obj = unsafe_load(err) - err_class = git_error_class[Int(err_obj.class)] + err_class = Class[err_obj.class][] err_msg = bytestring(err_obj.message) else - err_class = git_error_class[0] + err_class = Class[0][] err_msg = "No errors" end return (err_class, err_msg) end function GitError(code::Integer) - err_code = git_error_code[Int(code)] + err_code = Code[code][] err_class, err_msg = last_error() - return GitError{err_class, err_code}(err_msg) + return GitError(err_class, err_code, err_msg) end +end # Error module + macro check(git_func) quote local err::Cint err = $(esc(git_func::Expr)) if err < 0 - throw(GitError(err)) + throw(Error.GitError(err)) end err end diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index f5f3eaaaf18c3..df7613d5b4c83 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -2,11 +2,11 @@ function GitRepo(path::AbstractString) repo_ptr_ptr = Ptr{Void}[0] err = ccall((:git_repository_open, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{UInt8}), repo_ptr_ptr, path) - if err != GitErrorConst.GIT_OK + if err != Error.GIT_OK if repo_ptr_ptr[] != C_NULL finalize(GitRepo(repo_ptr_ptr[])) end - throw(GitError(err)) + throw(Error.GitError(err)) end return GitRepo(repo_ptr_ptr[]) end diff --git a/base/pkg/libgit2/walker.jl b/base/pkg/libgit2/walker.jl index dd7a26c5a3489..c0715fe1b2e70 100644 --- a/base/pkg/libgit2/walker.jl +++ b/base/pkg/libgit2/walker.jl @@ -1,8 +1,7 @@ function GitRevWalker(r::GitRepo) w_ptr = Ref{Ptr{Void}}(C_NULL) - err = ccall((:git_revwalk_new, :libgit2), Cint, + @check ccall((:git_revwalk_new, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}), w_ptr, r.ptr) - err != GitErrorConst.GIT_OK && return nothing return GitRevWalker(w_ptr[]) end @@ -10,7 +9,7 @@ function Base.start(w::GitRevWalker) id_ptr = Ref(Oid()) err = ccall((:git_revwalk_next, :libgit2), Cint, (Ptr{Oid}, Ptr{Void}), id_ptr, w.ptr) - err != GitErrorConst.GIT_OK && return (nothing, true) + err != Error.GIT_OK && return (nothing, true) return (id_ptr[], false) end @@ -20,19 +19,17 @@ function Base.next(w::GitRevWalker, state) id_ptr = Ref(Oid()) err = ccall((:git_revwalk_next, :libgit2), Cint, (Ptr{Oid}, Ptr{Void}), id_ptr, w.ptr) - err != GitErrorConst.GIT_OK && return (state[1], (nothing, true)) + err != Error.GIT_OK && return (state[1], (nothing, true)) return (state[1], (id_ptr[], false)) end function push_head!(w::GitRevWalker) - err = ccall((:git_revwalk_push_head, :libgit2), Cint, (Ptr{Void},), w.ptr) - err != GitErrorConst.GIT_OK && return GitError(err) + @check ccall((:git_revwalk_push_head, :libgit2), Cint, (Ptr{Void},), w.ptr) return w end function Base.push!(w::GitRevWalker, cid::Oid) - err = ccall((:git_revwalk_push, :libgit2), Cint, (Ptr{Void}, Ptr{Oid}), w.ptr, Ref(cid)) - err != GitErrorConst.GIT_OK && return GitError(err) + @check ccall((:git_revwalk_push, :libgit2), Cint, (Ptr{Void}, Ptr{Oid}), w.ptr, Ref(cid)) return w end @@ -44,21 +41,26 @@ end function Base.map(f::Function, repo::GitRepo; oid::Oid=Oid(), by::Cint = GitConst.SORT_NONE, rev::Bool=false) walker = GitRevWalker(repo) - sort!(walker, by=by, rev=rev) - if iszero(oid) - push_head!(walker) - else - push!(walker, oid) - end - s = start(walker) res = nothing - while !done(walker, s) - val = f(s[1], repo) - if res == nothing - res = Array(typeof(val),0) + try + sort!(walker, by=by, rev=rev) + if iszero(oid) + push_head!(walker) + else + push!(walker, oid) + end + s = start(walker) + + while !done(walker, s) + val = f(s[1], repo) + if res == nothing + res = Array(typeof(val),0) + end + push!(res, val) + val, s = next(walker, s) end - push!(res, val) - val, s = next(walker, s) + finally + finalize(walker) end return res end \ No newline at end of file From e4ac8bd84f55ed1125288f70dbeada2152660b2d Mon Sep 17 00:00:00 2001 From: Art Wild Date: Tue, 12 May 2015 22:22:21 -0400 Subject: [PATCH 0303/1938] error fix: nonexistent directory case, cannot create empty wrapper --- base/pkg/cache.jl | 4 ++-- base/pkg/libgit2/repository.jl | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/base/pkg/cache.jl b/base/pkg/cache.jl index 05200b427bd60..c07667deb1557 100644 --- a/base/pkg/cache.jl +++ b/base/pkg/cache.jl @@ -42,8 +42,8 @@ function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) try LibGit2.clone(url, cache, bare = true, remote_cb = LibGit2.mirror_cb) catch err - rm(cache, recursive=true) - error("Cannot clone $pkg from $url\nError in LibGit2.clone: ", err) + isdir(cache) && rm(cache, recursive=true) + error("Cannot clone $pkg from $url\n", err) end end try diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index df7613d5b4c83..441666ef4fcf4 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -47,18 +47,18 @@ function isattached(repo::GitRepo) end """ Returns a found object """ -function revparse(repo::GitRepo, obj::AbstractString) +function revparse(repo::GitRepo, objname::AbstractString) obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_revparse_single, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), obj_ptr_ptr, repo.ptr, obj) - err != 0 && return GitAnyObject(C_NULL) + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), obj_ptr_ptr, repo.ptr, objname) + err != 0 && return nothing return GitAnyObject(obj_ptr_ptr[]) end """ Returns id of a found object """ -function revparseid(repo::GitRepo, obj::AbstractString) +function revparseid(repo::GitRepo, objname::AbstractString) obj = revparse(repo, objname) - isempty(obj) && return Oid() + obj == nothing && return Oid() oid = Oid(obj.ptr) finalize(obj) return oid From 8ff8a258189884bd4893ca823fffc8b38df8c2cb Mon Sep 17 00:00:00 2001 From: Art Wild Date: Tue, 12 May 2015 22:40:35 -0400 Subject: [PATCH 0304/1938] made options structures immutable with convinient constructors --- base/pkg/libgit2.jl | 20 +++---- base/pkg/libgit2/types.jl | 122 +++++++++++++++++++++++++------------- 2 files changed, 89 insertions(+), 53 deletions(-) diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 45749690d0ceb..b30c4355ee815 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -185,16 +185,14 @@ function clone(url::AbstractString, path::AbstractString; branch::AbstractString="", bare::Bool = false, remote_cb::Ptr{Void} = C_NULL) - # start cloning - clone_opts = CloneOptionsStruct() - clone_opts.bare = Int32(bare) - if !isempty(branch) - clone_opts.checkout_branch = pointer(branch) - end - if remote_cb != C_NULL - clone_opts.remote_cb = remote_cb - end + # setup colne options + clone_opts = CloneOptionsStruct( + bare = Int32(bare), + checkout_branch = isempty(branch) ? Ptr{UInt8}(C_NULL) : pointer(branch), + remote_cb = remote_cb + ) + # start cloning clone_opts_ref = Ref(clone_opts) repo_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_clone, :libgit2), Cint, @@ -225,9 +223,7 @@ function checkout(repo::GitRepo, branch::AbstractString="master"; strategy::Cuint = GitConst.CHECKOUT_SAFE) treeish_obj = revparse(repo, branch) try - opts = CheckoutOptionsStruct() - opts.checkout_strategy = strategy - error = git_checkout_tree(repo, treeish, &opts); + opts = CheckoutOptionsStruct(checkout_strategy = strategy) ccall((:git_checkout_tree, :libgit2), Cint, (Ptr{Void}, Ptr{Void}, Ref{CheckoutOptionsStruct}), repo.ptr, treeish_obj.ptr, Ref(opts)) diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index f11c2dabc6a83..6535144ee0de0 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -19,7 +19,7 @@ immutable StrArrayStruct end StrArrayStruct() = StrArrayStruct(Ptr{UInt8}(0), zero(Csize_t)) -type CheckoutOptionsStruct +immutable CheckoutOptionsStruct version::Cuint checkout_strategy::Cuint disable_filters::Cint @@ -38,21 +38,39 @@ type CheckoutOptionsStruct our_label::Ptr{UInt8} their_label::Ptr{UInt8} end -CheckoutOptionsStruct() = CheckoutOptionsStruct(one(Cuint), - GitConst.CHECKOUT_SAFE_CREATE, - zero(Cint), - zero(Cuint), # Cuint(0o755), # - zero(Cuint), # Cuint(0o755), # - zero(Cint), - GitConst.CHECKOUT_NOTIFY_NONE, - Ptr{Void}(0), Ptr{Void}(0), - Ptr{Void}(0), Ptr{Void}(0), - StrArrayStruct(), - Ptr{Void}(0), - Ptr{UInt8}(0), - Ptr{UInt8}(0), - Ptr{UInt8}(0), - Ptr{UInt8}(0)) +CheckoutOptionsStruct(; checkout_strategy::Cuint = GitConst.CHECKOUT_SAFE_CREATE, + disable_filters::Cint = zero(Cint), + dir_mode::Cuint = Cuint(0o755), + file_mode::Cuint = Cuint(0o755), + file_open_flags::Cint = zero(Cint), + notify_flags::Cuint = GitConst.CHECKOUT_NOTIFY_NONE, + notify_cb::Ptr{Void} = Ptr{Void}(0), + notify_payload::Ptr{Void} = Ptr{Void}(0), + progress_cb::Ptr{Void} = Ptr{Void}(0), + progress_payload::Ptr{Void} = Ptr{Void}(0), + paths::StrArrayStruct = StrArrayStruct(), + baseline::Ptr{Void} = Ptr{Void}(0), + target_directory::Ptr{UInt8} = Ptr{UInt8}(0), + ancestor_label::Ptr{UInt8} =Ptr{UInt8}(0), + our_label::Ptr{UInt8} = Ptr{UInt8}(0), + their_label::Ptr{UInt8} = Ptr{UInt8}(0) +)=CheckoutOptionsStruct(one(Cuint), + checkout_strategy, + disable_filters, + dir_mode, + file_mode, + file_open_flags, + notify_flags, + notify_cb, + notify_payload, + progress_cb, + progress_payload, + paths, + baseline, + target_directory, + ancestor_label, + our_label, + their_label) immutable RemoteCallbacksStruct version::Cuint @@ -79,7 +97,7 @@ RemoteCallbacksStruct() = RemoteCallbacksStruct(one(Cuint), Ptr{Void}(0), Ptr{Void}(0)) -type CloneOptionsStruct +immutable CloneOptionsStruct version::Cuint checkout_opts::CheckoutOptionsStruct remote_callbacks::RemoteCallbacksStruct @@ -92,20 +110,30 @@ type CloneOptionsStruct remote_cb::Ptr{Void} remote_cb_payload::Ptr{Void} end - -CloneOptionsStruct() = CloneOptionsStruct(one(Cuint), - CheckoutOptionsStruct(), - RemoteCallbacksStruct(), - zero(Cint), - zero(Cint), - Ptr{UInt8}(0), - Ptr{Void}(0), - Ptr{Void}(0), Ptr{Void}(0), - Ptr{Void}(0), Ptr{Void}(0) - ) +CloneOptionsStruct(; checkout_opts::CheckoutOptionsStruct = CheckoutOptionsStruct(), + remote_callbacks::RemoteCallbacksStruct = RemoteCallbacksStruct(), + bare::Cint = zero(Cint), + localclone::Cint = zero(Cint), + checkout_branch::Ptr{UInt8} = Ptr{UInt8}(0), + signature::Ptr{Void} = Ptr{Void}(0), + repository_cb::Ptr{Void} = Ptr{Void}(0), + repository_cb_payload::Ptr{Void} = Ptr{Void}(0), + remote_cb::Ptr{Void} = Ptr{Void}(0), + remote_cb_payload::Ptr{Void} = Ptr{Void}(0) +)=CloneOptionsStruct(one(Cuint), + checkout_opts, + remote_callbacks, + bare, + localclone, + checkout_branch, + signature, + repository_cb, + repository_cb_payload, + remote_cb, + remote_cb_payload) # git diff option struct -type DiffOptionsStruct +immutable DiffOptionsStruct version::Cuint flags::UInt32 @@ -123,18 +151,30 @@ type DiffOptionsStruct old_prefix::Ptr{UInt8} new_prefix::Ptr{UInt8} end -DiffOptionsStruct() = DiffOptionsStruct(GitConst.DIFF_OPTIONS_VERSION, - GitConst.DIFF_NORMAL, - GitConst.SUBMODULE_IGNORE_DEFAULT, - StrArrayStruct(), - Ptr{Void}(0), - Ptr{Void}(0), - UInt32(3), - zero(UInt32), - UInt16(7), - Coff_t(512*1024*1024), #zero(Coff_t), - Ptr{UInt8}(0), - Ptr{UInt8}(0)) +DiffOptionsStruct(; flags::UInt32 = GitConst.DIFF_NORMAL, + ignore_submodules::Cint = GitConst.SUBMODULE_IGNORE_DEFAULT, + pathspec::StrArrayStruct = StrArrayStruct(), + notify_cb::Ptr{Void} = Ptr{Void}(0), + notify_payload::Ptr{Void} = Ptr{Void}(0), + context_lines::UInt32 = UInt32(3), + interhunk_lines::UInt32 = zero(UInt32), + id_abbrev::UInt16 = UInt16(7), + max_size::Coff_t = Coff_t(512*1024*1024), #zero(Coff_t), #512Mb + old_prefix::Ptr{UInt8} = Ptr{UInt8}(0), + new_prefix::Ptr{UInt8} = Ptr{UInt8}(0) +)=DiffOptionsStruct(GitConst.DIFF_OPTIONS_VERSION, + flags, + ignore_submodules, + pathspec, + notify_cb, + notify_payload, + context_lines, + interhunk_lines, + id_abbrev, + max_size, + old_prefix, + new_prefix + ) # Abstract object types abstract AbstractGitObject From 273ecfa512256f1ae3333d57b9a8a79270042208 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Wed, 13 May 2015 02:16:44 -0400 Subject: [PATCH 0305/1938] rewrote Write module: changed resource management and commit checkout. some minor fixes and refactoring --- base/pkg/cache.jl | 7 ++++-- base/pkg/libgit2.jl | 29 ++++++++++++----------- base/pkg/libgit2/repository.jl | 42 +++++++++++++++++++--------------- base/pkg/libgit2/types.jl | 4 ++-- base/pkg/write.jl | 36 ++++++++++++++++------------- 5 files changed, 64 insertions(+), 54 deletions(-) diff --git a/base/pkg/cache.jl b/base/pkg/cache.jl index c07667deb1557..3b50d2e9deac6 100644 --- a/base/pkg/cache.jl +++ b/base/pkg/cache.jl @@ -33,13 +33,14 @@ function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) cache = path(pkg) repo = if isdir(cache) try - LibGit2.GitRepo(cache) + LibGit2.GitRepo(cache) # open repo, free it at the end catch err rethrow(err) end else info("Cloning cache of $pkg from $url") try + # clone repo, free it at the end LibGit2.clone(url, cache, bare = true, remote_cb = LibGit2.mirror_cb) catch err isdir(cache) && rm(cache, recursive=true) @@ -56,10 +57,12 @@ function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) error("couldn't update $cache using `git remote update`") end end + filter(sha1->!LibGit2.iscommit(sha1, repo), sha1s) catch err rethrow(err) + finally + LibGit2.finalize(repo) # closing repo opened/created above end - filter(sha1->!LibGit2.iscommit(sha1, repo), sha1s) end prefetch(pkg::AbstractString, url::AbstractString, sha1::AbstractString...) = prefetch(pkg, url, AbstractString[sha1...]) diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index b30c4355ee815..8e1c2e06423e7 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -37,6 +37,19 @@ function gitdir(d, repo::GitRepo) path(repo) end +"""Return HEAD Oid as string""" +function head(pkg::AbstractString) + head_id ="" + repo = GitRepo(pkg) + try + head_id = string(head_oid(repo)) + catch err + warn(err) + finally + finalize(repo) + end +end + function need_update(repo::GitRepo) if !isbare(repo) # read updates index from filesystem @@ -69,8 +82,7 @@ function isdiff(repo::GitRepo, treeish::AbstractString, paths::AbstractString="" tree = get(GitTree, repo, tree_oid) if !emptypathspec sa = StrArrayStruct(paths) - diff_opts = DiffOptionsStruct() - diff_opts.pathspec = sa + diff_opts = DiffOptionsStruct(pathspec = sa) end try diff_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @@ -219,19 +231,6 @@ function branch_name(repo::GitRepo) return brnch end -function checkout(repo::GitRepo, branch::AbstractString="master"; - strategy::Cuint = GitConst.CHECKOUT_SAFE) - treeish_obj = revparse(repo, branch) - try - opts = CheckoutOptionsStruct(checkout_strategy = strategy) - ccall((:git_checkout_tree, :libgit2), Cint, - (Ptr{Void}, Ptr{Void}, Ref{CheckoutOptionsStruct}), - repo.ptr, treeish_obj.ptr, Ref(opts)) - finally - finalize(treeish_obj) - end -end - function create_branch(repo::GitRepo, branch::AbstractString, commit_oid::AbstractString, force::Bool=false) cmt = get(CitCommit, repo, commit_oid) diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index 441666ef4fcf4..31615513d68b9 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -99,33 +99,37 @@ function path(repo::GitRepo) (Ptr{Void},), repo.ptr)) end -function checkout(repo::GitRepo, spec::AbstractString) + +function checkout(repo::GitRepo, spec::AbstractString="master"; + strategy::Cuint = GitConst.CHECKOUT_SAFE_CREATE) + treeish_obj = revparse(repo, spec) + treeish_obj == nothing && return try - obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - @check ccall((:git_revparse_single, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), - obj_ptr_ptr, repo.ptr, spec) - obj = GitAnyObject(obj_ptr_ptr[]) - try - checkout_tree(repo, obj) - catch err - rethrow(err) - finally - finalize(obj) - end + opts = CheckoutOptionsStruct(checkout_strategy = strategy) + checkout_tree(repo, treeish_obj, opts) catch err warn("'checkout' thrown exception: $err") + finally + finalize(treeish_obj) end end -function checkout_tree(repo::GitRepo, obj::GitAnyObject) +function checkout_tree(repo::GitRepo, obj::GitAnyObject, + opts::CheckoutOptionsStruct = CheckoutOptionsStruct()) @check ccall((:git_checkout_tree, :libgit2), Cint, - (Ptr{Void}, Ptr{Void}, Ptr{Void}), - repo.ptr, obj.ptr, C_NULL) + (Ptr{Void}, Ptr{Void}, Ptr{CheckoutOptionsStruct}), + repo.ptr, obj.ptr, opts == CheckoutOptionsStruct() ? C_NULL : Ref(opts)) end -function checkout_index(repo::GitRepo, idx::GitIndex) +function checkout_index(repo::GitRepo, idx::GitIndex, + opts::CheckoutOptionsStruct = CheckoutOptionsStruct()) @check ccall((:git_checkout_index, :libgit2), Cint, - (Ptr{Void}, Ptr{Void}, Ptr{Void}), - repo.ptr, obj.ptr, C_NULL) + (Ptr{Void}, Ptr{Void}, Ptr{CheckoutOptionsStruct}), + repo.ptr, obj.ptr, opts == CheckoutOptionsStruct() ? C_NULL : Ref(opts)) +end + +function set_head_detached(repo::GitRepo, oid_str::AbstractString) + oid = Ref(Oid(oid_str)) + @check ccall((:git_repository_set_head_detached, :libgit2), Cint, + (Ptr{Void}, Ptr{Oid}), repo.ptr, oid) end diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index 6535144ee0de0..22482c0d3a7ef 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -40,8 +40,8 @@ immutable CheckoutOptionsStruct end CheckoutOptionsStruct(; checkout_strategy::Cuint = GitConst.CHECKOUT_SAFE_CREATE, disable_filters::Cint = zero(Cint), - dir_mode::Cuint = Cuint(0o755), - file_mode::Cuint = Cuint(0o755), + dir_mode::Cuint = Cuint(0), # Cuint(0o755), + file_mode::Cuint = Cuint(0), #Cuint(0o644), file_open_flags::Cint = zero(Cint), notify_flags::Cuint = GitConst.CHECKOUT_NOTIFY_NONE, notify_cb::Ptr{Void} = Ptr{Void}(0), diff --git a/base/pkg/write.jl b/base/pkg/write.jl index 1398382965e8b..197345ec94845 100644 --- a/base/pkg/write.jl +++ b/base/pkg/write.jl @@ -10,14 +10,12 @@ function prefetch(pkg::AbstractString, sha1::AbstractString) error("$pkg: couldn't find commit $(sha1[1:10])") end -function fetch(pkg::AbstractString, sha1::AbstractString) +function fetch(repo::GitRepo, pkg::AbstractString, sha1::AbstractString) refspec = "+refs/*:refs/remotes/cache/*" cache = Cache.path(pkg) - with(GitRepo, pkg) do repo - LibGit2.fetch(repo, cache, refspec) - LibGit2.need_update(repo) - LibGit2.iscommit(sha1, repo) - end && return + LibGit2.fetch(repo, cache, refspec) + LibGit2.need_update(repo) + LibGit2.iscommit(sha1, repo) && return f = with(GitRepo, cache) do repo LibGit2.iscommit(sha1, repo) end ? "fetch" : "prefetch" @@ -29,28 +27,34 @@ function fetch(pkg::AbstractString, sha1::AbstractString) end end -function checkout(pkg::AbstractString, sha1::AbstractString) - with(GitRepo, pkg) do repo - LibGit2.set_remote_url(repo, Read.url(pkg)) - LibGit2.checkout(repo, sha1) - end +function checkout(repo::GitRepo, pkg::AbstractString, sha1::AbstractString) + LibGit2.set_remote_url(repo, Read.url(pkg)) + LibGit2.set_head_detached(repo, sha1) + LibGit2.checkout(repo, sha1, strategy = LibGit2.GitConst.CHECKOUT_FORCE) end function install(pkg::AbstractString, sha1::AbstractString) prefetch(pkg, sha1) - if isdir(".trash/$pkg") + repo = if isdir(".trash/$pkg") mv(".trash/$pkg", "./$pkg") + LibGit2.GitRepo(pkg) else LibGit2.clone(Cache.path(pkg), pkg) end - fetch(pkg, sha1) - checkout(pkg, sha1) + try + fetch(repo, pkg, sha1) + checkout(repo, pkg, sha1) + finally + LibGit2.finalize(repo) + end end function update(pkg::AbstractString, sha1::AbstractString) prefetch(pkg, sha1) - fetch(pkg, sha1) - checkout(pkg, sha1) + with(GitRepo, pkg) do repo + fetch(repo, pkg, sha1) + checkout(repo, pkg, sha1) + end end function remove(pkg::AbstractString) From c1179e78c8df9075e154131d454871a9bfc6f63f Mon Sep 17 00:00:00 2001 From: Art Wild Date: Wed, 13 May 2015 15:36:38 -0400 Subject: [PATCH 0306/1938] added proper `checkout` & `branch` functions errors fixed & refactored code --- base/pkg/entry.jl | 6 +- base/pkg/libgit2.jl | 120 ++++++++++++++++++++++++--------- base/pkg/libgit2/commit.jl | 2 +- base/pkg/libgit2/const.jl | 2 + base/pkg/libgit2/reference.jl | 66 ++++++++++++++++-- base/pkg/libgit2/repository.jl | 77 ++++++++++----------- base/pkg/libgit2/signature.jl | 13 ++-- base/pkg/libgit2/types.jl | 25 +++++++ base/pkg/write.jl | 3 +- 9 files changed, 225 insertions(+), 89 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index bd72a9e81092e..1877ab108e1bb 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -52,9 +52,9 @@ function add(pkg::AbstractString, vers::VersionSet) end branch = Dir.getmetabranch() outdated = with(GitRepo, "METADATA") do repo - current_barnch = LibGit2.branch_name(repo) + current_barnch = LibGit2.branch(repo) if current_barnch == branch - if LibGit2.isdiff(repo, "origin/$branch") # diff origin/metadata-v2 + if LibGit2.isdiff(repo, "origin/$branch") outdated = :yes else try @@ -465,7 +465,7 @@ function resolve( missing = [] for (pkg,(ver1,ver2)) in changes vers = ASCIIString[] - ver1 !== nothing && push!(vers,Git.head(dir=pkg)) + ver1 !== nothing && push!(vers,LibGit2.head(pkg)) ver2 !== nothing && push!(vers,Read.sha1(pkg,ver2)) append!(missing, map(sha1->(pkg,(ver1,ver2),sha1), diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 8e1c2e06423e7..ccb7da11d6040 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -70,7 +70,7 @@ function iscommit(id::AbstractString, repo::GitRepo) end """ git diff-index HEAD [-- ]""" -isdirty(repo::GitRepo, paths::AbstractString="") = isdiff(repo, "HEAD", paths) +isdirty(repo::GitRepo, paths::AbstractString="") = isdiff(repo, GitConst.HEAD_FILE, paths) """ git diff-index [-- ]""" function isdiff(repo::GitRepo, treeish::AbstractString, paths::AbstractString="") @@ -193,6 +193,92 @@ function fetch(repo::GitRepo, remote_path::AbstractString, refspecs::AbstractStr end end +""" git branch """ +function branch(repo::GitRepo) + head_ref = head(repo) + brnch = "" + try + brnch = branch(head_ref) + finally + finalize(head_ref) + end + return brnch +end + +""" git branch -b [] """ +function branch!(repo::GitRepo, branch::AbstractString, + commit::AbstractString = ""; + force::Bool=false) + # if commit is empty get head commit oid + commit_oid = if isempty(commit) + head_ref = head(repo) + commit_id = peel(head_ref, GitConst.OBJ_COMMIT) + else + Oid(commit) + end + iszero(commit_id) && return + + cmt = get(GitCommit, repo, commit_oid) + try + create_branch(repo, cmt, branch, + force=force, + msg="pkg.libgit2.branch: moving to $branch") + ref = create_branch(repo, cmt, branch, force=force) + finalize(ref) + finally + finalize(cmt) + end +end + +""" git checkout [-f] --detach """ +function checkout!(repo::GitRepo, commit::AbstractString = ""; + force::Bool = true) + # nothing to do + isempty(commit) && return + + # grab head name + head_name = GitConst.HEAD_FILE + try + head_ref = head(repo) + head_name = shortname(head_ref) + # if it is HEAD use short OID instead + if head_name == GitConst.HEAD_FILE + head_name = string(Oid(head_ref)) + end + finalize(head_ref) + catch err + warn(err) + end + + # search for commit or revparse branch to get a commit object + obj = get(GitAnyObject, repo, Oid(commit)) + obj == nothing && return + try + peeled = peel(obj, GitConst.OBJ_COMMIT) + peeled == nothing && return + opts = force ? CheckoutOptionsStruct(checkout_strategy = GitConst.CHECKOUT_FORCE) : + CheckoutOptionsStruct() + try + # detach commit + obj_oid = Oid(peeled) + ref = create_reference(repo, obj_oid, + force=force, + msg="pkg.libgit2.checkout: moving from $head_name to $(string(obj_oid))") + finalize(ref) + + # checkout branch or commit + checkout_tree(repo, peeled, options = opts) + finally + finalize(peeled) + end + catch err + rethrow(err) + #warn("Checkout: $err") + finally + finalize(obj) + end +end + function clone(url::AbstractString, path::AbstractString; branch::AbstractString="", bare::Bool = false, @@ -220,38 +306,8 @@ function authors(repo::GitRepo) return athrs end -function branch_name(repo::GitRepo) - head_ref = head(repo) - brnch = "" - try - brnch = branch(head_ref) - finally - finalize(head_ref) - end - return brnch -end - -function create_branch(repo::GitRepo, branch::AbstractString, - commit_oid::AbstractString, force::Bool=false) - cmt = get(CitCommit, repo, commit_oid) - frc = CInt(force) - oid = Oid() - try - ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - @check ccall((:git_branch_create, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ref{Void}, Cint), - ref_ptr_ptr, repo.ptr, branch, cmt.ptr, CInt(force)) - ref = GitReference(ref_ptr_ptr[]) - oid = Oid(ref.ptr) - finalize(ref) - finally - finalize(treeish_obj) - end - return oid -end - function snapshot(repo::GitRepo; dir="") - head = Oid(repo, "HEAD") + head = Oid(repo, GitConst.HEAD_FILE) index = with(GitIndex, repo) do idx; write_tree!(idx) end work = try with(GitIndex, repo) do idx diff --git a/base/pkg/libgit2/commit.jl b/base/pkg/libgit2/commit.jl index 4ee01f36de66e..b2e80bbede555 100644 --- a/base/pkg/libgit2/commit.jl +++ b/base/pkg/libgit2/commit.jl @@ -45,7 +45,7 @@ end """Commit changes to repository""" function commit(repo::GitRepo, msg::AbstractString; - refname::AbstractString="HEAD", + refname::AbstractString=GitConst.HEAD_FILE, author::Signature = Signature(repo), committer::Signature = Signature(repo), tree_id::Oid = Oid(), diff --git a/base/pkg/libgit2/const.jl b/base/pkg/libgit2/const.jl index 60f9946429391..175b0fa12ce65 100644 --- a/base/pkg/libgit2/const.jl +++ b/base/pkg/libgit2/const.jl @@ -1,5 +1,7 @@ module GitConst + const HEAD_FILE = "HEAD" + const OBJ_ANY = Cint(-2) const OBJ_BAD = Cint(-1) const OBJ_COMMIT = Cint(1) diff --git a/base/pkg/libgit2/reference.jl b/base/pkg/libgit2/reference.jl index 3ec0dc263d360..11ea54cbf6fac 100644 --- a/base/pkg/libgit2/reference.jl +++ b/base/pkg/libgit2/reference.jl @@ -1,3 +1,18 @@ +function GitReference(repo::GitRepo, ref_name::AbstractString) + ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_reference_lookup, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), + ref_ptr_ptr, repo.ptr, ref_name) + return GitReference(ref_ptr_ptr[]) +end + +function head(repo::GitRepo) + head_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_repository_head, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}), head_ptr_ptr, repo.ptr) + return GitReference(head_ptr_ptr[]) +end + function shortname(ref::GitReference) isempty(ref) && return "" name_ptr = ccall((:git_reference_shorthand, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) @@ -22,10 +37,53 @@ function branch(ref::GitReference) return bytestring(str_ptr_ptr[]) end -function GitReference(repo::GitRepo, ref_name::AbstractString) +function peel(ref::GitReference, obj_type::Cint) + obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + err = ccall((:git_reference_peel, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Cint), obj_ptr_ptr, ref.ptr, obj_type) + if err == Error.ENOTFOUND + return Oid() + elseif err != Error.GIT_OK + if obj_ptr_ptr[] != C_NULL + finalize(GitAnyObject(obj_ptr_ptr[])) + end + throw(Error.GitError(err)) + end + id = Oid(obj_ptr_ptr[]) + finalize(GitAnyObject(obj_ptr_ptr[])) + return id +end + +function create_reference(repo::GitRepo, obj_oid::Oid, refname::AbstractString = GitConst.HEAD_FILE; + force::Bool=false, msg::AbstractString="") + sig = default_signature(repo) ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - @check ccall((:git_reference_lookup, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), - ref_ptr_ptr, repo.ptr, ref_name) + try + @check ccall((:git_reference_create, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{Oid}, Cint, Ptr{SignatureStruct}, Ptr{UInt8}), + ref_ptr_ptr, repo.ptr, refname, Ref(obj_oid), Cint(force), + sig.ptr, isempty(msg) ? Ptr{UInt8}(C_NULL) : msg) + catch err + rethrow(err) + finally + finalize(sig) + end + return GitReference(ref_ptr_ptr[]) +end + +function create_branch(repo::GitRepo, commit_obj::GitCommit, bname::AbstractString; + force::Bool=false, msg::AbstractString="") + sig = default_signature(repo) + ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + try + @check ccall((:git_branch_create, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{Void}, Cint, Ptr{SignatureStruct}, Ptr{UInt8}), + ref_ptr_ptr, repo.ptr, bname, commit_obj.ptr, Cint(force), + sig.ptr, isempty(msg) ? Ptr{UInt8}(C_NULL) : msg) + catch err + rethrow(err) + finally + finalize(sig) + end return GitReference(ref_ptr_ptr[]) end \ No newline at end of file diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index 31615513d68b9..834e138c85c7c 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -1,5 +1,5 @@ function GitRepo(path::AbstractString) - repo_ptr_ptr = Ptr{Void}[0] + repo_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_repository_open, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{UInt8}), repo_ptr_ptr, path) if err != Error.GIT_OK @@ -24,13 +24,6 @@ function init(path::AbstractString, bare::Cuint = Cuint(0)) return GitRepo(repo_ptr_ptr[]) end -function head(repo::GitRepo) - head_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - @check ccall((:git_repository_head, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}), head_ptr_ptr, repo.ptr) - return GitReference(head_ptr_ptr[]) -end - function head_oid(repo::GitRepo) head_ref = head(repo) oid = Oid(head_ref) @@ -67,26 +60,25 @@ end function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::Oid, prefix::Bool=false) id_ptr = Ref(oid) obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + git_otype = getobjecttype(T) - git_otype = if T == GitCommit - GitConst.OBJ_COMMIT - elseif T == GitTree - GitConst.OBJ_TREE - elseif T == GitAnyObject - GitConst.OBJ_ANY - else - error("Type $T is not supported") - end - - @check if prefix + err = if prefix ccall((:git_object_lookup_prefix, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}, Csize_t, Cint), - obj_ptr_ptr, r.ptr, id_ptr, len, git_otype) + obj_ptr_ptr, r.ptr, id_ptr, len, git_otype) #TODO: length else ccall((:git_object_lookup, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}, Cint), obj_ptr_ptr, r.ptr, id_ptr, git_otype) end + if err == Error.ENOTFOUND + return nothing + elseif err != Error.GIT_OK + if obj_ptr_ptr[] != C_NULL + finalize(GitAnyObject(obj_ptr_ptr[])) + end + throw(Error.GitError(err)) + end return T(obj_ptr_ptr[]) end @@ -100,36 +92,35 @@ function path(repo::GitRepo) end -function checkout(repo::GitRepo, spec::AbstractString="master"; - strategy::Cuint = GitConst.CHECKOUT_SAFE_CREATE) - treeish_obj = revparse(repo, spec) - treeish_obj == nothing && return - try - opts = CheckoutOptionsStruct(checkout_strategy = strategy) - checkout_tree(repo, treeish_obj, opts) - catch err - warn("'checkout' thrown exception: $err") - finally - finalize(treeish_obj) +function peel(obj::GitObject, obj_type::Cint) + peeled_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + git_otype = getobjecttype(obj_type) + err = ccall((:git_object_peel, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Cint), peeled_ptr_ptr, obj.ptr, obj_type) + if err == Error.ENOTFOUND + return Oid() + elseif err != Error.GIT_OK + if peeled_ptr_ptr[] != C_NULL + finalize(GitAnyObject(peeled_ptr_ptr[])) + end + throw(Error.GitError(err)) end + return git_otype(peeled_ptr_ptr[]) end -function checkout_tree(repo::GitRepo, obj::GitAnyObject, - opts::CheckoutOptionsStruct = CheckoutOptionsStruct()) +function checkout_tree(repo::GitRepo, obj::GitObject; + options::CheckoutOptionsStruct = CheckoutOptionsStruct()) @check ccall((:git_checkout_tree, :libgit2), Cint, (Ptr{Void}, Ptr{Void}, Ptr{CheckoutOptionsStruct}), - repo.ptr, obj.ptr, opts == CheckoutOptionsStruct() ? C_NULL : Ref(opts)) + repo.ptr, obj.ptr, + options == CheckoutOptionsStruct() ? C_NULL : Ref(options)) end -function checkout_index(repo::GitRepo, idx::GitIndex, - opts::CheckoutOptionsStruct = CheckoutOptionsStruct()) +function checkout_index(repo::GitRepo, idx::Nullable{GitIndex} = Nullable{GitIndex}(); + options::CheckoutOptionsStruct = CheckoutOptionsStruct()) @check ccall((:git_checkout_index, :libgit2), Cint, (Ptr{Void}, Ptr{Void}, Ptr{CheckoutOptionsStruct}), - repo.ptr, obj.ptr, opts == CheckoutOptionsStruct() ? C_NULL : Ref(opts)) -end - -function set_head_detached(repo::GitRepo, oid_str::AbstractString) - oid = Ref(Oid(oid_str)) - @check ccall((:git_repository_set_head_detached, :libgit2), Cint, - (Ptr{Void}, Ptr{Oid}), repo.ptr, oid) + repo.ptr, + isnull(idx) ? C_NULL : idx.ptr, + options == CheckoutOptionsStruct() ? C_NULL : Ref(options)) end diff --git a/base/pkg/libgit2/signature.jl b/base/pkg/libgit2/signature.jl index 3af9d1d974236..2f79117f95f28 100644 --- a/base/pkg/libgit2/signature.jl +++ b/base/pkg/libgit2/signature.jl @@ -19,10 +19,7 @@ function Signature(name::AbstractString, email::AbstractString) end function Signature(repo::GitRepo) - sig_ptr_ptr = Ref{Ptr{SignatureStruct}}(C_NULL) - @check ccall((:git_signature_default, :libgit2), Cint, - (Ptr{Ptr{SignatureStruct}}, Ptr{Void}), sig_ptr_ptr, repo.ptr) - sig = GitSignature(sig_ptr_ptr[]) + sig = default_signature(repo) s = Signature(sig.ptr) finalize(sig) return s @@ -34,4 +31,12 @@ function Base.convert(::Type{GitSignature}, sig::Signature) (Ptr{Ptr{SignatureStruct}}, Ptr{Uint8}, Ptr{Uint8}, Cint, Cint), sig_ptr_ptr, sig.name, sig.email, sig.time, sig.time_offset) return GitSignature(sig_ptr_ptr[]) +end + +"""Return signature object. Free it after use.""" +function default_signature(repo::GitRepo) + sig_ptr_ptr = Ref{Ptr{SignatureStruct}}(C_NULL) + @check ccall((:git_signature_default, :libgit2), Cint, + (Ptr{Ptr{SignatureStruct}}, Ptr{Void}), sig_ptr_ptr, repo.ptr) + return GitSignature(sig_ptr_ptr[]) end \ No newline at end of file diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index 22482c0d3a7ef..b35695b9a0728 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -259,4 +259,29 @@ function with_warn{T}(f::Function, ::Type{T}, args...) catch err warn("$(string(T)) thrown exception: $err") end +end + + +function getobjecttype{T<:GitObject}(::Type{T}) + return if T == GitCommit + GitConst.OBJ_COMMIT + elseif T == GitTree + GitConst.OBJ_TREE + elseif T == GitAnyObject + GitConst.OBJ_ANY + else + error("Type $T is not supported") + end +end + +function getobjecttype(obj_type::Cint) + return if obj_type == GitConst.OBJ_COMMIT + GitCommit + elseif obj_type == GitConst.OBJ_TREE + GitTree + elseif obj_type == GitConst.OBJ_ANY + GitAnyObject + else + error("Type $T is not supported") + end end \ No newline at end of file diff --git a/base/pkg/write.jl b/base/pkg/write.jl index 197345ec94845..0d5064ba92499 100644 --- a/base/pkg/write.jl +++ b/base/pkg/write.jl @@ -29,8 +29,7 @@ end function checkout(repo::GitRepo, pkg::AbstractString, sha1::AbstractString) LibGit2.set_remote_url(repo, Read.url(pkg)) - LibGit2.set_head_detached(repo, sha1) - LibGit2.checkout(repo, sha1, strategy = LibGit2.GitConst.CHECKOUT_FORCE) + LibGit2.checkout!(repo, sha1) end function install(pkg::AbstractString, sha1::AbstractString) From 67435f68e607fa6c8a003648451930ac7e2f7629 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Fri, 15 May 2015 23:47:04 -0400 Subject: [PATCH 0307/1938] added tests for `libgit2` added functions: `reset` & `checkout_head` merged multiple `fetch` functions into one fixed `restore` & `transact` --- base/pkg/libgit2.jl | 84 +++++++++------- base/pkg/libgit2/const.jl | 4 + base/pkg/libgit2/repository.jl | 46 ++++++++- base/pkg/write.jl | 3 +- test/libgit2.jl | 172 ++++++++++++++++++++++++++++++++- 5 files changed, 271 insertions(+), 38 deletions(-) diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index ccb7da11d6040..ea2f456fbe7da 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -165,25 +165,15 @@ function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, name::Ptr{ end const mirror_cb = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}, Ptr{Void})) -function fetch(rmt::GitRemote) - @check ccall((:git_remote_fetch, :libgit2), Cint, - (Ptr{Void}, Ptr{Void}, Ptr{UInt8}), - rmt.ptr, C_NULL, C_NULL) -end - -function fetch(repo::GitRepo, remote::AbstractString="origin") - rmt = get(GitRemote, repo, remote) - try - fetch(rmt) - catch err - warn("'fetch' thrown exception: $err") - finally - finalize(rmt) +""" git fetch []""" +function fetch(repo::GitRepo, remote::AbstractString="origin"; + refspecs::AbstractString="") + rmt = if isempty(refspecs) + get(GitRemote, repo, remote) + else + GitRemoteAnon(repo, remote, refspecs) end -end -function fetch(repo::GitRepo, remote_path::AbstractString, refspecs::AbstractString) - rmt = GitRemoteAnon(repo, remote_path, refspecs) try fetch(rmt) catch err @@ -279,6 +269,7 @@ function checkout!(repo::GitRepo, commit::AbstractString = ""; end end +""" git clone [-b ] [--bare] """ function clone(url::AbstractString, path::AbstractString; branch::AbstractString="", bare::Bool = false, @@ -299,6 +290,36 @@ function clone(url::AbstractString, path::AbstractString; return GitRepo(repo_ptr_ptr[]) end +""" git reset [] [--] ... """ +function reset!(repo::GitRepo, committish::AbstractString, pathspecs::AbstractString...) + target_obj = isempty(committish) ? Nullable{GitAnyObject}() : + Nullable(revparse(repo, committish)) + try + reset!(repo, target_obj, pathspecs...) + catch err + rethrow(err) + #warn("Reset: $err") + finally + !isnull(target_obj) && finalize(Base.get(target_obj)) + end +end + +""" git reset [--soft | --mixed | --hard] """ +function reset!(repo::GitRepo, commit::Oid, mode::Cint = GitConst.GIT_RESET_MIXED) + + obj = get(GitAnyObject, repo, commit) + obj == nothing && return + try + reset!(repo, obj, mode) + catch err + rethrow(err) + #warn("Reset: $err") + finally + finalize(obj) + end +end + +""" Returns all commit authors """ function authors(repo::GitRepo) athrs = map( (oid,repo)->author(get(GitCommit, repo, oid))::Signature, @@ -316,7 +337,7 @@ function snapshot(repo::GitRepo; dir="") files = [utf8(bytestring(c))::UTF8String for c in content] push!(files, utf8(".")) - add!(idx, files..., flags = GitConst.INDEX_ADD_CHECK_PATHSPEC) + add!(idx, files...) write!(idx) end write_tree!(idx) @@ -331,28 +352,24 @@ function snapshot(repo::GitRepo; dir="") State(head, index, work) end -function restore(s::State, repo::GitRepo; dir="") +function restore(s::State, repo::GitRepo) + reset!(repo, GitConst.HEAD_FILE, "*") # unstage everything with(GitIndex, repo) do idx - #TODO: reset -q -- # unstage everything - - read_tree!(idx, s.work) # move work tree to index - write!(idx) - - #TODO: checkout-index -fa # check the index out to work - #TODO: clean -qdf # remove everything else - - read_tree!(idx, s.index) # restore index - write!(idx) + read_tree!(idx, s.work) # move work tree to index + opts = CheckoutOptionsStruct( + checkout_strategy = GitConst.CHECKOUT_FORCE | # check the index out to work + GitConst.CHECKOUT_REMOVE_UNTRACKED) # remove everything else + checkout_index(repo, Nullable(idx), options = opts) - #TODO: reset -q --soft $(s.head) # restore head + read_tree!(idx, s.index) # restore index end + reset!(repo, s.head, GitConst.GIT_RESET_SOFT) # restore head end -# TODO: restore required function transact(f::Function, repo::GitRepo; dir="") - #state = snapshot(repo, dir=dir) + state = snapshot(repo, dir=dir) try f(repo) catch - #restore(state, repo, dir=dir) + restore(state, repo) rethrow() finally finalize(repo) @@ -364,7 +381,6 @@ function __init__() err = ccall((:git_libgit2_init, :libgit2), Cint, ()) err > 0 || error("error initializing LibGit2 module") atexit() do - gc() ccall((:git_libgit2_shutdown, :libgit2), Cint, ()) end end diff --git a/base/pkg/libgit2/const.jl b/base/pkg/libgit2/const.jl index 175b0fa12ce65..d2d1e62df3b11 100644 --- a/base/pkg/libgit2/const.jl +++ b/base/pkg/libgit2/const.jl @@ -209,4 +209,8 @@ module GitConst const GIT_REPOSITORY_STATE_REBASE_MERGE = Cint(7) const GIT_REPOSITORY_STATE_APPLY_MAILBOX = Cint(8) const GIT_REPOSITORY_STATE_APPLY_MAILBOX_OR_REBASE = Cint(9) + + const GIT_RESET_SOFT = Cint(1) # Move the head to the given commit + const GIT_RESET_MIXED = Cint(2) # SOFT plus reset index to the commit + const GIT_RESET_HARD = Cint(3) # MIXED plus changes in working tree discarded end \ No newline at end of file diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index 834e138c85c7c..c1806df5ac1aa 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -121,6 +121,50 @@ function checkout_index(repo::GitRepo, idx::Nullable{GitIndex} = Nullable{GitInd @check ccall((:git_checkout_index, :libgit2), Cint, (Ptr{Void}, Ptr{Void}, Ptr{CheckoutOptionsStruct}), repo.ptr, - isnull(idx) ? C_NULL : idx.ptr, + isnull(idx) ? C_NULL : Base.get(idx).ptr, options == CheckoutOptionsStruct() ? C_NULL : Ref(options)) end + +function checkout_head(repo::GitRepo; options::CheckoutOptionsStruct = CheckoutOptionsStruct()) + @check ccall((:git_checkout_head, :libgit2), Cint, + (Ptr{Void}, Ptr{CheckoutOptionsStruct}), + repo.ptr, options == CheckoutOptionsStruct() ? C_NULL : Ref(options)) +end + +function fetch(rmt::GitRemote) + @check ccall((:git_remote_fetch, :libgit2), Cint, + (Ptr{Void}, Ptr{Void}, Ptr{UInt8}), + rmt.ptr, C_NULL, C_NULL) +end + +function reset!(repo::GitRepo, obj::Nullable{GitAnyObject}, pathspecs::AbstractString...) + sa = StrArrayStruct(pathspecs...) + try + @check ccall((:git_reset_default, :libgit2), Cint, + (Ptr{Void}, Ptr{Void}, Ptr{StrArrayStruct}), + repo.ptr, + isnull(obj) ? C_NULL: Base.get(obj).ptr, + Ref(sa)) + catch err + rethrow(err) + finally + finalize(sa) + end +end + +function reset!(repo::GitRepo, obj::GitObject, mode::Cint; + checkout_opts::CheckoutOptionsStruct = CheckoutOptionsStruct()) + sig = default_signature(repo) + msg = "pkg.libgit2.reset: moving to $(string(Oid(obj)))" + try + @check ccall((:git_reset, :libgit2), Cint, + (Ptr{Void}, Ptr{Void}, Cint, Ptr{CheckoutOptionsStruct}, Ptr{SignatureStruct}, Ptr{UInt8}), + repo.ptr, obj.ptr, mode, Ref(checkout_opts), sig.ptr, msg) + catch err + rethrow(err) + finally + finalize(sig) + end + return +end + diff --git a/base/pkg/write.jl b/base/pkg/write.jl index 0d5064ba92499..ea4e1cdde30f1 100644 --- a/base/pkg/write.jl +++ b/base/pkg/write.jl @@ -11,9 +11,8 @@ function prefetch(pkg::AbstractString, sha1::AbstractString) end function fetch(repo::GitRepo, pkg::AbstractString, sha1::AbstractString) - refspec = "+refs/*:refs/remotes/cache/*" cache = Cache.path(pkg) - LibGit2.fetch(repo, cache, refspec) + LibGit2.fetch(repo, cache, refspecs = "+refs/*:refs/remotes/cache/*") LibGit2.need_update(repo) LibGit2.iscommit(sha1, repo) && return f = with(GitRepo, cache) do repo diff --git a/test/libgit2.jl b/test/libgit2.jl index 27054bc7c331c..90ef1353c2237 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -2,7 +2,7 @@ # check that libgit2 has been installed correctly -const LIBGIT2_VER = v"0.21+" +const LIBGIT2_VER = v"0.22+" function check_version() major, minor, patch = Cint[0], Cint[0], Cint[0] @@ -17,3 +17,173 @@ function check_version() end @test check_version() + +function temp_dir(fn::Function, remove_tmp_dir::Bool=true) + tmpdir = joinpath(tempdir(),randstring()) + @test !isdir(tmpdir) + try + mkdir(tmpdir) + @test isdir(tmpdir) + fn(tmpdir) + finally + remove_tmp_dir && rm(tmpdir, recursive=true) + end +end + +# clone bare +temp_dir() do dir + url = "https://github.com/JuliaLang/Example.jl" + path = joinpath(dir, "Example.Bare") + repo = Pkg.LibGit2.clone(url, path, bare = true, remote_cb = Pkg.LibGit2.mirror_cb) + Pkg.LibGit2.finalize(repo) + @test isdir(path) + @test isfile(joinpath(path, Pkg.LibGit2.GitConst.HEAD_FILE)) +end + +# clone +temp_dir() do dir + url = "https://github.com/JuliaLang/Example.jl" + path = joinpath(dir, "Example") + repo = Pkg.LibGit2.clone(url, path) + Pkg.LibGit2.finalize(repo) + @test isdir(path) + @test isdir(joinpath(path, ".git")) +end + +# init +temp_dir() do dir + path = joinpath(dir, "Example") + repo = Pkg.LibGit2.init(path) + + @test isdir(path) + @test isdir(joinpath(path, ".git")) + + branch = "upstream" + url = "https://github.com/JuliaLang/julia.git" + remote = Pkg.LibGit2.GitRemote(repo, branch, url) + Pkg.LibGit2.save(remote) + Pkg.LibGit2.finalize(remote) + + config = joinpath(path, ".git", "config") + lines = split(open(readall, config, "r"), "\n") + @test any(map(x->x == "[remote \"upstream\"]", lines)) + + remote = Pkg.LibGit2.get(Pkg.LibGit2.GitRemote, repo, branch) + @test Pkg.LibGit2.url(remote) == url + + Pkg.LibGit2.finalize(repo) +end + +# fetch +temp_dir() do dir_cache + # create cache + url = "https://github.com/JuliaLang/Example.jl" + path_cache = joinpath(dir_cache, "Example.Bare") + repo = Pkg.LibGit2.clone(url, path_cache, bare = true, remote_cb = Pkg.LibGit2.mirror_cb) + Pkg.LibGit2.finalize(repo) + + + # fetch + temp_dir() do dir + # clone repo + path = joinpath(dir, "Example") + repo = Pkg.LibGit2.clone(path_cache, path) + + # fetch + Pkg.LibGit2.fetch(repo) + refs1 = parse(Int, readchomp(pipe(`find $(joinpath(path, ".git/refs"))`,`wc -l`))) + + Pkg.LibGit2.fetch(repo, path_cache, refspecs = "+refs/*:refs/remotes/cache/*") + refs2 = parse(Int, readchomp(pipe(`find $(joinpath(path, ".git/refs"))`,`wc -l`))) + + Pkg.LibGit2.finalize(repo) + @test refs1 > 0 + @test refs2 > 0 + @test refs2 > refs1 + end + + # revwalk + temp_dir() do dir + path = joinpath(dir, "Example") + repo = Pkg.LibGit2.clone(path_cache, path) + oids = Pkg.LibGit2.map((oid,repo)->string(oid), repo, by = Pkg.LibGit2.GitConst.SORT_TIME) + @test length(oids) > 0 + Pkg.LibGit2.finalize(repo) + + Pkg.LibGit2.with(Pkg.LibGit2.GitRepo, path) do repo + oid = Pkg.LibGit2.Oid(oids[end]) + oids = Pkg.LibGit2.map((oid,repo)->(oid,repo), repo, oid=oid, by=Pkg.LibGit2.GitConst.SORT_TIME) + @test length(oids) > 0 + end + end + + # signature + temp_dir() do dir + path = joinpath(dir, "Example") + repo = Pkg.LibGit2.clone(path_cache, path) + oid = "129eb39c8e0817c616296d1ac5f2cd1cf4f8b312" + c = Pkg.LibGit2.get(Pkg.LibGit2.GitCommit, repo, oid) + auth = Pkg.LibGit2.author(c) + cmtr = Pkg.LibGit2.committer(c) + cmsg = Pkg.LibGit2.message(c) + @test isa(auth, Pkg.LibGit2.Signature) + @test length(auth.name) > 0 + @test isa(cmtr, Pkg.LibGit2.Signature) + @test length(cmtr.email) > 0 + @test length(cmsg) > 0 + Pkg.LibGit2.finalize(repo) + + sig = Pkg.LibGit2.Signature("AAA", "AAA@BBB.COM", round(time(), 0), 0) + git_sig = convert(Pkg.LibGit2.GitSignature, sig) + sig2 = Pkg.LibGit2.Signature(git_sig) + Pkg.LibGit2.finalize(git_sig) + @test sig.name == sig2.name + @test sig.email == sig2.email + @test sig.time == sig2.time + end + + # transact + temp_dir() do dir + # clone repo + path = joinpath(dir, "Example") + repo = Pkg.LibGit2.clone(path_cache, path) + cp(joinpath(path, "REQUIRE"), joinpath(path, "CCC")) + cp(joinpath(path, "REQUIRE"), joinpath(path, "AAA")) + Pkg.LibGit2.add!(repo, "AAA") + @test_throws ErrorException Pkg.LibGit2.transact(repo, dir=path) do repo + mv(joinpath(path, "REQUIRE"), joinpath(path, "BBB")) + Pkg.LibGit2.add!(repo, "BBB") + oid = Pkg.LibGit2.commit(repo, "test commit") + error("Force recovery") + end + @test isfile(joinpath(path, "AAA")) + @test isfile(joinpath(path, "CCC")) + @test !isfile(joinpath(path, "BBB")) + @test isfile(joinpath(path, "REQUIRE")) + end + +end + +# strarray +begin + p1 = "XXX" + p2 = "YYY" + sa1 = Pkg.LibGit2.StrArrayStruct(p1) + try + arr = convert(Vector{AbstractString}, sa1) + @test arr[1] == p1 + finally + Pkg.LibGit2.finalize(sa1) + end + + sa2 = Pkg.LibGit2.StrArrayStruct(p1, p2) + try + arr = convert(Vector{AbstractString}, sa2) + @test arr[1] == p1 + @test arr[2] == p2 + finally + Pkg.LibGit2.finalize(sa2) + end +end + + From 740194ad6c341271e7ab935bd47cfef7718e0a2b Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sat, 16 May 2015 13:54:28 -0400 Subject: [PATCH 0308/1938] removed unnecessary `rethrow` calls improved `branch!` removed `dir` parameter from `transact` --- base/pkg/cache.jl | 8 +--- base/pkg/generate.jl | 2 +- base/pkg/libgit2.jl | 87 ++++++++++++++++++++-------------- base/pkg/libgit2/const.jl | 1 + base/pkg/libgit2/index.jl | 8 ---- base/pkg/libgit2/reference.jl | 42 ++++++++++++++-- base/pkg/libgit2/repository.jl | 5 -- base/pkg/libgit2/types.jl | 11 +---- 8 files changed, 93 insertions(+), 71 deletions(-) diff --git a/base/pkg/cache.jl b/base/pkg/cache.jl index 3b50d2e9deac6..08d95bcf0247b 100644 --- a/base/pkg/cache.jl +++ b/base/pkg/cache.jl @@ -32,11 +32,7 @@ function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) isdir(".cache") || mkcachedir() cache = path(pkg) repo = if isdir(cache) - try - LibGit2.GitRepo(cache) # open repo, free it at the end - catch err - rethrow(err) - end + LibGit2.GitRepo(cache) # open repo, free it at the end else info("Cloning cache of $pkg from $url") try @@ -58,8 +54,6 @@ function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) end end filter(sha1->!LibGit2.iscommit(sha1, repo), sha1s) - catch err - rethrow(err) finally LibGit2.finalize(repo) # closing repo opened/created above end diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index 8792f83c757d9..a93985b84c4ad 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -55,7 +55,7 @@ function package( repo end - LibGit2.transact(repo, dir=pkg) do repo + LibGit2.transact(repo) do repo if isempty(authors) authors = isnew ? copyright_name(repo) : git_contributors(repo,5) end diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index ea2f456fbe7da..d48eafd3fd1fe 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -50,10 +50,10 @@ function head(pkg::AbstractString) end end +""" git update-index """ function need_update(repo::GitRepo) if !isbare(repo) # read updates index from filesystem - # i.e. "git update-index -q --really-refresh" read!(repo, true) end end @@ -62,7 +62,8 @@ end function iscommit(id::AbstractString, repo::GitRepo) res = true try - get(GitCommit, repo, id) + c = get(GitCommit, repo, id) + finalize(c) catch res = false end @@ -195,29 +196,51 @@ function branch(repo::GitRepo) return brnch end -""" git branch -b [] """ -function branch!(repo::GitRepo, branch::AbstractString, - commit::AbstractString = ""; - force::Bool=false) - # if commit is empty get head commit oid - commit_oid = if isempty(commit) - head_ref = head(repo) - commit_id = peel(head_ref, GitConst.OBJ_COMMIT) - else - Oid(commit) - end - iszero(commit_id) && return +""" git checkout [-b|-B] [] [--track /] """ +function branch!(repo::GitRepo, branch_name::AbstractString, + commit::AbstractString = ""; # start point + track::AbstractString = "", # track remote branch + force::Bool=false, # force branch creation + set_head::Bool=true) # set as head reference on exit + # try to lookup branch first + branch_ref = force ? nothing : lookup_branch(repo, branch_name) + if branch_ref == nothing + # if commit is empty get head commit oid + commit_id = if isempty(commit) + head_ref = head(repo) + try + peel(head_ref, GitConst.OBJ_COMMIT) + finally + finalize(head_ref) + end + else + Oid(commit) + end + iszero(commit_id) && return - cmt = get(GitCommit, repo, commit_oid) + cmt = get(GitCommit, repo, commit_id) + try + branch_ref = create_branch(repo, cmt, branch_name, + force=force, + msg="pkg.libgit2.branch: moving to $branch_name") + finally + finalize(cmt) + end + end try - create_branch(repo, cmt, branch, - force=force, - msg="pkg.libgit2.branch: moving to $branch") - ref = create_branch(repo, cmt, branch, force=force) - finalize(ref) + if !isempty(track) # setup tracking + with(GitConfig, repo) do cfg + set!(cfg, "branch.$branch_name.remote", GitConst.REMOTE_ORIGIN) + set!(cfg, "branch.$branch_name.merge", name(branch_ref)) + end + end + + # switch head to the branch + set_head && head!(repo, branch_ref) finally - finalize(cmt) + finalize(branch_ref) end + return end """ git checkout [-f] --detach """ @@ -240,7 +263,7 @@ function checkout!(repo::GitRepo, commit::AbstractString = ""; warn(err) end - # search for commit or revparse branch to get a commit object + # search for commit to get a commit object obj = get(GitAnyObject, repo, Oid(commit)) obj == nothing && return try @@ -256,14 +279,11 @@ function checkout!(repo::GitRepo, commit::AbstractString = ""; msg="pkg.libgit2.checkout: moving from $head_name to $(string(obj_oid))") finalize(ref) - # checkout branch or commit + # checkout commit checkout_tree(repo, peeled, options = opts) finally finalize(peeled) end - catch err - rethrow(err) - #warn("Checkout: $err") finally finalize(obj) end @@ -296,9 +316,6 @@ function reset!(repo::GitRepo, committish::AbstractString, pathspecs::AbstractSt Nullable(revparse(repo, committish)) try reset!(repo, target_obj, pathspecs...) - catch err - rethrow(err) - #warn("Reset: $err") finally !isnull(target_obj) && finalize(Base.get(target_obj)) end @@ -311,9 +328,6 @@ function reset!(repo::GitRepo, commit::Oid, mode::Cint = GitConst.GIT_RESET_MIXE obj == nothing && return try reset!(repo, obj, mode) - catch err - rethrow(err) - #warn("Reset: $err") finally finalize(obj) end @@ -327,12 +341,13 @@ function authors(repo::GitRepo) return athrs end -function snapshot(repo::GitRepo; dir="") +function snapshot(repo::GitRepo) head = Oid(repo, GitConst.HEAD_FILE) index = with(GitIndex, repo) do idx; write_tree!(idx) end work = try + dir = splitdir(path(repo)[1:end-1])[1] # remove `/.git` part with(GitIndex, repo) do idx - content = readdir(abspath(dir)) + content = readdir(dir) if length(content) > 1 files = [utf8(bytestring(c))::UTF8String for c in content] push!(files, utf8(".")) @@ -366,8 +381,8 @@ function restore(s::State, repo::GitRepo) reset!(repo, s.head, GitConst.GIT_RESET_SOFT) # restore head end -function transact(f::Function, repo::GitRepo; dir="") - state = snapshot(repo, dir=dir) +function transact(f::Function, repo::GitRepo) + state = snapshot(repo) try f(repo) catch restore(state, repo) rethrow() diff --git a/base/pkg/libgit2/const.jl b/base/pkg/libgit2/const.jl index d2d1e62df3b11..eae8ba50404ff 100644 --- a/base/pkg/libgit2/const.jl +++ b/base/pkg/libgit2/const.jl @@ -1,6 +1,7 @@ module GitConst const HEAD_FILE = "HEAD" + const REMOTE_ORIGIN = "origin" const OBJ_ANY = Cint(-2) const OBJ_BAD = Cint(-1) diff --git a/base/pkg/libgit2/index.jl b/base/pkg/libgit2/index.jl index 7b1ed84603f42..a1132991caee8 100644 --- a/base/pkg/libgit2/index.jl +++ b/base/pkg/libgit2/index.jl @@ -34,8 +34,6 @@ function read_tree!(idx::GitIndex, tree_id::Oid) try @check ccall((:git_index_read_tree, :libgit2), Cint, (Ptr{Void}, Ptr{Void}), idx.ptr, tree.ptr) - catch err - rethrow(err) finally finalize(tree) end @@ -48,8 +46,6 @@ function add!{T<:AbstractString}(idx::GitIndex, files::T...; @check ccall((:git_index_add_all, :libgit2), Cint, (Ptr{Void}, Ptr{StrArrayStruct}, Cuint, Ptr{Void}, Ptr{Void}), idx.ptr, Ref(sa), flags, C_NULL, C_NULL) - catch err - rethrow(err) finally finalize(sa) end @@ -61,8 +57,6 @@ function update!{T<:AbstractString}(idx::GitIndex, files::T...) @check ccall((:git_index_update_all, :libgit2), Cint, (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{Void}, Ptr{Void}), idx.ptr, Ref(sa), C_NULL, C_NULL) - catch err - rethrow(err) finally finalize(sa) end @@ -74,8 +68,6 @@ function remove!{T<:AbstractString}(idx::GitIndex, files::T...) @check ccall((:git_index_remove_all, :libgit2), Cint, (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{Void}, Ptr{Void}), idx.ptr, Ref(sa), C_NULL, C_NULL) - catch err - rethrow(err) finally finalize(sa) end diff --git a/base/pkg/libgit2/reference.jl b/base/pkg/libgit2/reference.jl index 11ea54cbf6fac..44c22b71b6ddf 100644 --- a/base/pkg/libgit2/reference.jl +++ b/base/pkg/libgit2/reference.jl @@ -29,6 +29,13 @@ function fullname(ref::GitReference) return bytestring(rname) end +function name(ref::GitReference) + isempty(ref) && return "" + name_ptr = ccall((:git_reference_name, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) + name_ptr == C_NULL && return "" + return bytestring(name_ptr) +end + function branch(ref::GitReference) isempty(ref) && return "" str_ptr_ptr = Ref{Ptr{UInt8}}(C_NULL) @@ -63,8 +70,6 @@ function create_reference(repo::GitRepo, obj_oid::Oid, refname::AbstractString = (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{Oid}, Cint, Ptr{SignatureStruct}, Ptr{UInt8}), ref_ptr_ptr, repo.ptr, refname, Ref(obj_oid), Cint(force), sig.ptr, isempty(msg) ? Ptr{UInt8}(C_NULL) : msg) - catch err - rethrow(err) finally finalize(sig) end @@ -80,10 +85,39 @@ function create_branch(repo::GitRepo, commit_obj::GitCommit, bname::AbstractStri (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{Void}, Cint, Ptr{SignatureStruct}, Ptr{UInt8}), ref_ptr_ptr, repo.ptr, bname, commit_obj.ptr, Cint(force), sig.ptr, isempty(msg) ? Ptr{UInt8}(C_NULL) : msg) - catch err - rethrow(err) finally finalize(sig) end return GitReference(ref_ptr_ptr[]) +end + +function head!(repo::GitRepo, ref::GitReference; msg::AbstractString="") + ref_name = name(ref) + sig = default_signature(repo) + msg = "pkg.libgit2.head!: switched to $(branch(ref)) "*msg + try + @check ccall((:git_repository_set_head, :libgit2), Cint, + (Ptr{Void}, Ptr{UInt8}, Ptr{SignatureStruct}, Ptr{UInt8}), + repo.ptr, ref_name, sig.ptr, isempty(msg) ? Ptr{UInt8}(C_NULL) : msg) + finally + finalize(sig) + end + return ref +end + +function lookup_branch(repo::GitRepo, branch_name::AbstractString, remote::Bool=false) + ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + branch_type = remote ? GitConst.BRANCH_REMOTE : GitConst.BRANCH_LOCAL + err = ccall((:git_branch_lookup, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Cint), + ref_ptr_ptr, repo.ptr, branch_name, branch_type) + if err == Error.ENOTFOUND + return nothing + elseif err != Error.GIT_OK + if repo_ptr_ptr[] != C_NULL + finalize(GitReference(ref_ptr_ptr[])) + end + throw(Error.GitError(err)) + end + return GitReference(ref_ptr_ptr[]) end \ No newline at end of file diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index c1806df5ac1aa..163763b87d88c 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -91,7 +91,6 @@ function path(repo::GitRepo) (Ptr{Void},), repo.ptr)) end - function peel(obj::GitObject, obj_type::Cint) peeled_ptr_ptr = Ref{Ptr{Void}}(C_NULL) git_otype = getobjecttype(obj_type) @@ -145,8 +144,6 @@ function reset!(repo::GitRepo, obj::Nullable{GitAnyObject}, pathspecs::AbstractS repo.ptr, isnull(obj) ? C_NULL: Base.get(obj).ptr, Ref(sa)) - catch err - rethrow(err) finally finalize(sa) end @@ -160,8 +157,6 @@ function reset!(repo::GitRepo, obj::GitObject, mode::Cint; @check ccall((:git_reset, :libgit2), Cint, (Ptr{Void}, Ptr{Void}, Cint, Ptr{CheckoutOptionsStruct}, Ptr{SignatureStruct}, Ptr{UInt8}), repo.ptr, obj.ptr, mode, Ref(checkout_opts), sig.ptr, msg) - catch err - rethrow(err) finally finalize(sig) end diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index b35695b9a0728..3c642a669fe63 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -236,21 +236,12 @@ end function with_libgit2(f::Function, obj) try f(obj) - catch err - rethrow(err) finally finalize(obj) end end -function with{T}(f::Function, ::Type{T}, args...; warn_on_exception::Bool=true) - obj = T(args...) - try - with_libgit2(f, obj) - catch err - rethrow(err) - end -end +with{T}(f::Function, ::Type{T}, args...) = with_libgit2(f, T(args...)) function with_warn{T}(f::Function, ::Type{T}, args...) obj = T(args...) From 30688dc82adc2c9810bcfbb9ed2712f0edea09ec Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sat, 16 May 2015 21:41:15 -0400 Subject: [PATCH 0309/1938] transitioned `checkout`, `free`, `update`, `register` added merge functionality --- base/pkg/dir.jl | 4 +- base/pkg/entry.jl | 82 +++++++++++++++++++--------------- base/pkg/libgit2.jl | 34 ++++++-------- base/pkg/libgit2/const.jl | 27 ++++++----- base/pkg/libgit2/merge.jl | 69 ++++++++++++++++++++++++++++ base/pkg/libgit2/oid.jl | 5 +-- base/pkg/libgit2/reference.jl | 35 +++++++++++++-- base/pkg/libgit2/repository.jl | 10 ++--- base/pkg/libgit2/types.jl | 28 ++++++++++-- test/libgit2.jl | 10 ++++- 10 files changed, 218 insertions(+), 86 deletions(-) create mode 100644 base/pkg/libgit2/merge.jl diff --git a/base/pkg/dir.jl b/base/pkg/dir.jl index 2d00570b6b9f3..cea1f69146cd7 100644 --- a/base/pkg/dir.jl +++ b/base/pkg/dir.jl @@ -3,7 +3,7 @@ module Dir import ..Pkg: DEFAULT_META, META_BRANCH -import ..LibGit2, ..LibGit2.with_libgit2 +import ..LibGit2, ..LibGit2.with const DIR_NAME = ".julia" @@ -46,7 +46,7 @@ function init(meta::AbstractString=DEFAULT_META, branch::AbstractString=META_BRA temp_dir = mktempdir(dir) Base.cd(temp_dir) do info("Cloning METADATA from $meta") - with_libgit2(LibGit2.clone(meta, "METADATA", branch = branch)) do metadata_repo + with(LibGit2.clone(meta, "METADATA", branch = branch)) do metadata_repo LibGit2.set_remote_url(metadata_repo, meta) end touch("REQUIRE") diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 1877ab108e1bb..6dd6f088a4df8 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -212,23 +212,22 @@ function clone(url_or_pkg::AbstractString) clone(url,pkg) end -function _checkout(pkg::AbstractString, what::AbstractString, merge::Bool=false, pull::Bool=false, branch::Bool=false) - Git.transact(dir=pkg) do #TODO: finish restore() - Git.dirty(dir=pkg) && error("$pkg is dirty, bailing") - branch ? Git.run(`checkout -q -B $what -t origin/$what`, dir=pkg) : Git.run(`checkout -q $what`, dir=pkg) - merge && Git.run(`merge -q --ff-only $what`, dir=pkg) - if pull - info("Pulling $pkg latest $what...") - Git.run(`pull -q --ff-only`, dir=pkg) - end - resolve() - end -end - function checkout(pkg::AbstractString, branch::AbstractString, merge::Bool, pull::Bool) ispath(pkg,".git") || error("$pkg is not a git repo") info("Checking out $pkg $branch...") - _checkout(pkg,branch,merge,pull,true) + LibGit2.with(LibGit2.GitRepo, pkg) do r + LibGit2.transact(r) do repo + LibGit2.isdirty(repo) && error("$pkg is dirty, bailing") + LibGit2.branch!(repo, branch, track=LibGit2.GitConst.REMOTE_ORIGIN) + merge && LibGit2.merge!(repo) # merge changes + if pull + info("Pulling $pkg latest $branch...") + LibGit2.fetch(repo) + LibGit2.merge!(repo) + end + resolve() + end + end end function free(pkg::AbstractString) @@ -245,7 +244,11 @@ function free(pkg::AbstractString) for ver in vers sha1 = avail[ver].sha1 LibGit2.iscommit(sha1, repo) || continue - return _checkout(pkg, sha1) + return LibGit2.transact(repo) do r + LibGit2.isdirty(repo) && error("$pkg is dirty, bailing") + LibGit2.checkout!(repo, sha1) + resolve() + end end isempty(Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail])) && continue error("can't find any registered versions of $pkg to checkout") @@ -306,18 +309,22 @@ end function update(branch::AbstractString) info("Updating METADATA...") cd("METADATA") do - if Git.branch() != branch - Git.dirty() && error("METADATA is dirty and not on $branch, bailing") - Git.attached() || error("METADATA is detached not on $branch, bailing") - Git.run(`fetch -q --all`) - Git.run(`checkout -q HEAD^0`) - Git.run(`branch -f $branch refs/remotes/origin/$branch`) - Git.run(`checkout -q $branch`) + with(GitRepo, ".") do repo + with(LibGit2.head(repo)) do h + if LibGit2.branch(h) != branch + LibGit2.isdirty(repo) && error("METADATA is dirty and not on $branch, bailing") + LibGit2.isattached(repo) || error("METADATA is detached not on $branch, bailing") + LibGit2.fetch(repo) + LibGit2.checkout_head(repo) + LibGit2.branch!(repo, branch, track="refs/remotes/origin/$branch") + LibGit2.merge!(repo) + end + end end # TODO: handle merge conflicts - Base.withenv("GIT_MERGE_AUTOEDIT"=>"no") do - Git.run(`pull --rebase -q`, out=DevNull) - end + # Base.withenv("GIT_MERGE_AUTOEDIT"=>"no") do + # Git.run(`pull --rebase -q`, out=DevNull) + # end end avail = Read.available() # this has to happen before computing free/fixed @@ -332,17 +339,17 @@ function update(branch::AbstractString) fixed = Read.fixed(avail,instd) for (pkg,ver) in fixed ispath(pkg,".git") || continue - begin - if Git.attached(dir=pkg) && !Git.dirty(dir=pkg) + with(GitRepo, pkg) do repo + if LibGit2.isattached(repo) && !LibGit2.isdirty(repo) info("Updating $pkg...") @recover begin - Git.run(`fetch -q --all`, dir=pkg) - Git.success(`pull -q --ff-only`, dir=pkg) # suppress output + LibGit2.fetch(repo) + LibGit2.merge!(repo) end end - if haskey(avail,pkg) - Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail[pkg]]) - end + end + if haskey(avail,pkg) + Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail[pkg]]) end end info("Computing changes...") @@ -577,10 +584,13 @@ function register(pkg::AbstractString, url::AbstractString) end function register(pkg::AbstractString) - Git.success(`config remote.origin.url`, dir=pkg) || - error("$pkg: no URL configured") - url = Git.readchomp(`config remote.origin.url`, dir=pkg) - register(pkg,Git.normalize_url(url)) + url = LibGit2.with(LibGit2.GitRepo, pkg) do repo + LibGit2.with(LibGit2.GitConfig, repo) do cfg + LibGit2.get(cfg, "remote.origin.url", "") + end + end + isempty(url) || error("$pkg: no URL configured") + register(pkg, LibGit2.normalize_url(url)) end function isrewritable(v::VersionNumber) diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index d48eafd3fd1fe..2e0b53bd3d5dd 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -1,6 +1,6 @@ module LibGit2 -export with_libgit2, with, with_warn +export with, with_warn export GitRepo, GitConfig, GitIndex const GITHUB_REGEX = @@ -19,6 +19,7 @@ include("libgit2/walker.jl") include("libgit2/remote.jl") include("libgit2/strarray.jl") include("libgit2/index.jl") +include("libgit2/merge.jl") immutable State head::Oid @@ -39,14 +40,8 @@ end """Return HEAD Oid as string""" function head(pkg::AbstractString) - head_id ="" - repo = GitRepo(pkg) - try - head_id = string(head_oid(repo)) - catch err - warn(err) - finally - finalize(repo) + with(GitRepo, pkg) do repo + string(head_oid(repo)) end end @@ -96,7 +91,6 @@ function isdiff(repo::GitRepo, treeish::AbstractString, paths::AbstractString="" result = c > 0 finalize(diff) catch err - warn(err) result = true finally !emptypathspec && finalize(sa) @@ -228,7 +222,7 @@ function branch!(repo::GitRepo, branch_name::AbstractString, end end try - if !isempty(track) # setup tracking + if !isempty(track) # setup tracking #TODO: what is branch tracks other then "origin" remote with(GitConfig, repo) do cfg set!(cfg, "branch.$branch_name.remote", GitConst.REMOTE_ORIGIN) set!(cfg, "branch.$branch_name.merge", name(branch_ref)) @@ -252,15 +246,13 @@ function checkout!(repo::GitRepo, commit::AbstractString = ""; # grab head name head_name = GitConst.HEAD_FILE try - head_ref = head(repo) - head_name = shortname(head_ref) - # if it is HEAD use short OID instead - if head_name == GitConst.HEAD_FILE - head_name = string(Oid(head_ref)) + with(head(repo)) do head_ref + head_name = shortname(head_ref) + # if it is HEAD use short OID instead + if head_name == GitConst.HEAD_FILE + head_name = string(Oid(head_ref)) + end end - finalize(head_ref) - catch err - warn(err) end # search for commit to get a commit object @@ -322,7 +314,7 @@ function reset!(repo::GitRepo, committish::AbstractString, pathspecs::AbstractSt end """ git reset [--soft | --mixed | --hard] """ -function reset!(repo::GitRepo, commit::Oid, mode::Cint = GitConst.GIT_RESET_MIXED) +function reset!(repo::GitRepo, commit::Oid, mode::Cint = GitConst.RESET_MIXED) obj = get(GitAnyObject, repo, commit) obj == nothing && return @@ -378,7 +370,7 @@ function restore(s::State, repo::GitRepo) read_tree!(idx, s.index) # restore index end - reset!(repo, s.head, GitConst.GIT_RESET_SOFT) # restore head + reset!(repo, s.head, GitConst.RESET_SOFT) # restore head end function transact(f::Function, repo::GitRepo) diff --git a/base/pkg/libgit2/const.jl b/base/pkg/libgit2/const.jl index eae8ba50404ff..37cda27740a8c 100644 --- a/base/pkg/libgit2/const.jl +++ b/base/pkg/libgit2/const.jl @@ -172,6 +172,11 @@ module GitConst const MERGE_TREE_FIND_RENAMES = Cint(1) << Cint(0) + const MERGE_FILE_FAVOR_NORMAL = Cint(0) + const MERGE_FILE_FAVOR_OURS = Cint(1) + const MERGE_FILE_FAVOR_THEIRS = Cint(2) + const MERGE_FILE_FAVOR_UNION = Cint(3) + const MERGE_AUTOMERGE_NORMAL = Cint(0) const MERGE_AUTOMERGE_FAVOR_OURS = Cint(1) const MERGE_AUTOMERGE_FAVOR_THEIRS = Cint(2) @@ -180,15 +185,15 @@ module GitConst const MERGE_NO_FASTFORWARD = Cint(1) const MERGE_FASTFORWARD_ONLY = Cint(2) - const GIT_MERGE_ANALYSIS_NONE = 0, - const GIT_MERGE_ANALYSIS_NORMAL = (1 << 0) - const GIT_MERGE_ANALYSIS_UP_TO_DATE = (1 << 1) - const GIT_MERGE_ANALYSIS_FASTFORWARD = (1 << 2) - const GIT_MERGE_ANALYSIS_UNBORN = (1 << 3) + const MERGE_ANALYSIS_NONE = 0, + const MERGE_ANALYSIS_NORMAL = (1 << 0) + const MERGE_ANALYSIS_UP_TO_DATE = (1 << 1) + const MERGE_ANALYSIS_FASTFORWARD = (1 << 2) + const MERGE_ANALYSIS_UNBORN = (1 << 3) - const GIT_MERGE_PREFERENCE_NONE = 0 - const GIT_MERGE_PREFERENCE_NO_FASTFORWARD = (1 << 0) - const GIT_MERGE_PREFERENCE_FASTFORWARD_ONLY = (1 << 1) + const MERGE_PREFERENCE_NONE = 0 + const MERGE_PREFERENCE_NO_FASTFORWARD = (1 << 0) + const MERGE_PREFERENCE_FASTFORWARD_ONLY = (1 << 1) const DIRECTION_FETCH = Cint(0) const DIRECTION_PUSH = Cint(1) @@ -211,7 +216,7 @@ module GitConst const GIT_REPOSITORY_STATE_APPLY_MAILBOX = Cint(8) const GIT_REPOSITORY_STATE_APPLY_MAILBOX_OR_REBASE = Cint(9) - const GIT_RESET_SOFT = Cint(1) # Move the head to the given commit - const GIT_RESET_MIXED = Cint(2) # SOFT plus reset index to the commit - const GIT_RESET_HARD = Cint(3) # MIXED plus changes in working tree discarded + const RESET_SOFT = Cint(1) # Move the head to the given commit + const RESET_MIXED = Cint(2) # SOFT plus reset index to the commit + const RESET_HARD = Cint(3) # MIXED plus changes in working tree discarded end \ No newline at end of file diff --git a/base/pkg/libgit2/merge.jl b/base/pkg/libgit2/merge.jl new file mode 100644 index 0000000000000..d07d6da0be3ff --- /dev/null +++ b/base/pkg/libgit2/merge.jl @@ -0,0 +1,69 @@ +function GitAnnotated(repo::GitRepo, commit_id::Oid) + ann_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_annotated_commit_lookup, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}), + ann_ptr_ptr, repo.ptr, commit_id) + return GitAnnotated(ann_ptr_ptr[]) +end + +function GitAnnotated(repo::GitRepo, ref::GitReference) + ann_ref_ref = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_annotated_commit_from_ref, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}), + ann_ref_ref, repo.ptr, ref.ptr) + return GitAnnotated(ann_ref_ref[]) +end + +function merge_analysis(repo::GitRepo, ann::GitAnnotated) + analysis = Ref{Cint}(0) + preference = Ref{Cint}(0) + @check ccall((:git_merge_analysis, :libgit2), Cint, + (Ptr{Cint}, Ptr{Cint}, Ptr{Void}, Ptr{Ptr{Void}}, Csize_t), + analysis, preference, repo.ptr, Ref{Ptr{Void}}(ann.ptr), 1) + return analysis[], preference[] +end + +function commit(ann::GitAnnotated) + return Oid(ccall((:git_annotated_commit_id, :libgit2), Ptr{Oid}, (Ptr{Void},), ann.ptr)) +end + +""" Merge changes into current head """ +function merge!(repo::GitRepo; fast_forward::Bool=false) + # get head annotated upstream reference + with(head(repo)) do hr + with(upstream(hr)) do hur + with(GitAnnotated(repo, hur)) do hua + ma, mp = merge_analysis(repo, hua) + (ma & GitConst.MERGE_ANALYSIS_UP_TO_DATE == GitConst.MERGE_ANALYSIS_UP_TO_DATE) && return + if (ma & GitConst.MERGE_ANALYSIS_FASTFORWARD == GitConst.MERGE_ANALYSIS_FASTFORWARD) + # do fastforward: checkout tree and update branch references + # hur_oid = Oid(hur) + # with(get(GitCommit, repo, hur_oid)) do cmt + # checkout_tree(repo, cmt) + # end + # target!(hr, hur_oid, msg="pkg.libgit2.megre!: fastforward $(name(hur)) into $(name(hr))") + # head!(repo, hur, msg="--fastforward") + + hur_oid = Oid(hur) + target!(hr, hur_oid, msg="pkg.libgit2.megre!: fastforward $(name(hur)) into $(name(hr))") + reset!(repo, hur_oid, GitConst.RESET_HARD) + elseif (ma & GitConst.MERGE_ANALYSIS_NORMAL == GitConst.MERGE_ANALYSIS_NORMAL) + if fast_forward + warn("Fastforward merge is not possible. Abort merging.") + return + end + merge_opts = MergeOptionsStruct() + checkout_opts = CheckoutOptionsStruct(checkout_strategy = GitConst.CHECKOUT_SAFE) + @check ccall((:git_merge, :libgit2), Cint, + (Ptr{Void}, Ptr{Ptr{Void}}, Csize_t, Ptr{MergeOptionsStruct}, Ptr{CheckoutOptionsStruct}), + repo.ptr, Ref{Ptr{Void}}(hua.ptr), 1, Ref(merge_opts), Ref(checkout_opts)) + cleanup(repo) + info("Review and commit merged changes.") + else + warn("Unknown merge analysis result. Merging is not possible.") + end + end + end + end +end + diff --git a/base/pkg/libgit2/oid.jl b/base/pkg/libgit2/oid.jl index 60675082c6176..e0799b9579b2d 100644 --- a/base/pkg/libgit2/oid.jl +++ b/base/pkg/libgit2/oid.jl @@ -2,6 +2,7 @@ const OID_RAWSZ = 20 const OID_HEXSZ = OID_RAWSZ * 2 const OID_MINPREFIXLEN = 4 +#TODO: The Oid generated code should now use an immutable type that wraps an ntuple of length 20 # immutable Oid # id1::UInt8 # id2::UInt8 @@ -54,9 +55,7 @@ end function Oid(ref::GitReference) isempty(ref) && return Oid() - - typ = ccall((:git_reference_type, :libgit2), Cint, (Ptr{Void},), ref.ptr) - typ != 1 && return Oid() + reftype(ref) != GitConst.REF_OID && return Oid() oid_ptr = ccall((:git_reference_target, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) oid_ptr == C_NULL && return Oid() return Oid(oid_ptr) diff --git a/base/pkg/libgit2/reference.jl b/base/pkg/libgit2/reference.jl index 44c22b71b6ddf..77895a510f15e 100644 --- a/base/pkg/libgit2/reference.jl +++ b/base/pkg/libgit2/reference.jl @@ -20,10 +20,13 @@ function shortname(ref::GitReference) return bytestring(name_ptr) end +function reftype(ref::GitReference) + return ccall((:git_reference_type, :libgit2), Cint, (Ptr{Void},), ref.ptr) +end + function fullname(ref::GitReference) isempty(ref) && return "" - typ = ccall((:git_reference_type, :libgit2), Cint, (Ptr{Void},), ref.ptr) - typ == 1 && return "" + reftype(ref) == GitConst.REF_OID && return "" rname = ccall((:git_reference_symbolic_target, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) rname == C_NULL && return "" return bytestring(rname) @@ -120,4 +123,30 @@ function lookup_branch(repo::GitRepo, branch_name::AbstractString, remote::Bool= throw(Error.GitError(err)) end return GitReference(ref_ptr_ptr[]) -end \ No newline at end of file +end + +function upstream(ref::GitReference) + isempty(ref) && return nothing + ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_branch_upstream, :libgit2), Cint, + (Ref{Ptr{Void}}, Ptr{Void},), ref_ptr_ptr, ref.ptr) + return GitReference(ref_ptr_ptr[]) +end + +function owner(ref::GitReference) + repo_ptr = ccall((:git_reference_owner, :libgit2), Ptr{Void}, + (Ptr{Void},), ref.ptr) + return GitRepo(repo_ptr) +end + +function target!(ref::GitReference, new_oid::Oid; msg::AbstractString="") + ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + repo + with(default_signature(owner(ref))) do sig + @check ccall((:git_reference_set_target, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}, Ptr{SignatureStruct}, Ptr{UInt8}), + ref_ptr_ptr, ref.ptr, Ref(new_oid), sig.ptr, isempty(msg) ? Ptr{UInt8}(C_NULL) : msg) + end + return GitReference(ref_ptr_ptr[]) +end + diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index 163763b87d88c..cb0c988cc3370 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -11,7 +11,7 @@ function GitRepo(path::AbstractString) return GitRepo(repo_ptr_ptr[]) end -function close(r::GitRepo) +function cleanup(r::GitRepo) if r.ptr != C_NULL ccall((:git_repository__cleanup, :libgit2), Void, (Ptr{Void},), r.ptr) end @@ -111,8 +111,7 @@ function checkout_tree(repo::GitRepo, obj::GitObject; options::CheckoutOptionsStruct = CheckoutOptionsStruct()) @check ccall((:git_checkout_tree, :libgit2), Cint, (Ptr{Void}, Ptr{Void}, Ptr{CheckoutOptionsStruct}), - repo.ptr, obj.ptr, - options == CheckoutOptionsStruct() ? C_NULL : Ref(options)) + repo.ptr, obj.ptr, Ref(options)) end function checkout_index(repo::GitRepo, idx::Nullable{GitIndex} = Nullable{GitIndex}(); @@ -121,13 +120,13 @@ function checkout_index(repo::GitRepo, idx::Nullable{GitIndex} = Nullable{GitInd (Ptr{Void}, Ptr{Void}, Ptr{CheckoutOptionsStruct}), repo.ptr, isnull(idx) ? C_NULL : Base.get(idx).ptr, - options == CheckoutOptionsStruct() ? C_NULL : Ref(options)) + Ref(options)) end function checkout_head(repo::GitRepo; options::CheckoutOptionsStruct = CheckoutOptionsStruct()) @check ccall((:git_checkout_head, :libgit2), Cint, (Ptr{Void}, Ptr{CheckoutOptionsStruct}), - repo.ptr, options == CheckoutOptionsStruct() ? C_NULL : Ref(options)) + repo.ptr, Ref(options)) end function fetch(rmt::GitRemote) @@ -162,4 +161,3 @@ function reset!(repo::GitRepo, obj::GitObject, mode::Cint; end return end - diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index 3c642a669fe63..c08f3344201a1 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -176,6 +176,27 @@ DiffOptionsStruct(; flags::UInt32 = GitConst.DIFF_NORMAL, new_prefix ) +immutable MergeOptionsStruct + version::Cuint + flags::Cint + rename_threshold::Cuint + target_limit::Cuint + metric::Ptr{Void} + file_favor::Cint +end +MergeOptionsStruct(; flags::Cint = Cint(0), + rename_threshold::Cuint = Cuint(50), + target_limit::Cuint = Cuint(200), + metric::Ptr{Void} = Ptr{Void}(0), + file_favor::Cint = GitConst.MERGE_FILE_FAVOR_NORMAL +)=MergeOptionsStruct(one(Cuint), + flags, + rename_threshold, + target_limit, + metric, + file_favor + ) + # Abstract object types abstract AbstractGitObject Base.isempty(obj::AbstractGitObject) = (obj.ptr == C_NULL) @@ -197,6 +218,7 @@ for (typ, ref, sup, fnc) in ( (:GitDiff, :Void, :AbstractGitObject, :(:git_diff_free)), (:GitIndex, :Void, :AbstractGitObject, :(:git_index_free)), (:GitRepo, :Void, :AbstractGitObject, :(:git_repository_free)), + (:GitAnnotated, :Void, :AbstractGitObject, :(:git_annotated_commit_free)), (:GitSignature, :SignatureStruct, :AbstractGitObject, :(:git_signature_free)), (:GitAnyObject, :Void, :GitObject, nothing), (:GitCommit, :Void, :GitObject, nothing), @@ -233,7 +255,7 @@ end """ Resource management helper function """ -function with_libgit2(f::Function, obj) +function with(f::Function, obj) try f(obj) finally @@ -241,12 +263,12 @@ function with_libgit2(f::Function, obj) end end -with{T}(f::Function, ::Type{T}, args...) = with_libgit2(f, T(args...)) +with{T}(f::Function, ::Type{T}, args...) = with(f, T(args...)) function with_warn{T}(f::Function, ::Type{T}, args...) obj = T(args...) try - with_libgit2(f, obj) + with(f, obj) catch err warn("$(string(T)) thrown exception: $err") end diff --git a/test/libgit2.jl b/test/libgit2.jl index 90ef1353c2237..d819a64088df6 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -18,6 +18,14 @@ end @test check_version() +#TODO: tests need 'user.name' & 'user.email' in config ??? +Pkg.LibGit2.with(Pkg.LibGit2.GitConfig) do cfg + git_user = Pkg.LibGit2.get(cfg, "user.name", "") + isempty(git_user) && Pkg.LibGit2.set(cfg, "user.email", "Test User") + git_user_email = Pkg.LibGit2.get(cfg, "user.email", "") + isempty(git_user) && Pkg.LibGit2.set(cfg, "user.email", "Test@User.com") +end + function temp_dir(fn::Function, remove_tmp_dir::Bool=true) tmpdir = joinpath(tempdir(),randstring()) @test !isdir(tmpdir) @@ -150,7 +158,7 @@ temp_dir() do dir_cache cp(joinpath(path, "REQUIRE"), joinpath(path, "CCC")) cp(joinpath(path, "REQUIRE"), joinpath(path, "AAA")) Pkg.LibGit2.add!(repo, "AAA") - @test_throws ErrorException Pkg.LibGit2.transact(repo, dir=path) do repo + @test_throws ErrorException Pkg.LibGit2.transact(repo) do repo mv(joinpath(path, "REQUIRE"), joinpath(path, "BBB")) Pkg.LibGit2.add!(repo, "BBB") oid = Pkg.LibGit2.commit(repo, "test commit") From 331f8c275ec02c908ece81c8fbac1e7d66e7d24e Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sun, 17 May 2015 12:55:05 -0400 Subject: [PATCH 0310/1938] switched to https --- base/pkg.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/pkg.jl b/base/pkg.jl index c2c8a529d4a11..443741f98266e 100644 --- a/base/pkg.jl +++ b/base/pkg.jl @@ -7,7 +7,7 @@ export dir, init, rm, add, available, installed, status, clone, checkout, update, resolve, register, tag, publish, generate, test, build, free, pin -const DEFAULT_META = "git://github.com/JuliaLang/METADATA.jl" +const DEFAULT_META = "https://github.com/JuliaLang/METADATA.jl" const META_BRANCH = "metadata-v2" for file in split("git libgit2 dir github types reqs cache read query resolve write generate entry") From dcac546269192077e319d4d567d14cea563b731f Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sun, 17 May 2015 12:57:29 -0400 Subject: [PATCH 0311/1938] added `gitdir` (location of repo '.git') and `path` (repo location) --- base/pkg/generate.jl | 2 +- base/pkg/libgit2/repository.jl | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index a93985b84c4ad..f25e7491a7ca2 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -86,7 +86,7 @@ function package( elseif LibGit2.isdirty(repo) LibGit2.remove!(repo, files...) info("Regenerated files left unstaged, use `git add -p` to select") - open(io->print(io,msg), joinpath(LibGit2.gitdir(pkg, repo),"MERGE_MSG"), "w") + open(io->print(io,msg), joinpath(LibGit2.gitdir(repo),"MERGE_MSG"), "w") else info("Regenerated files are unchanged") end diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index cb0c988cc3370..9ca2e815f4650 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -86,11 +86,16 @@ function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::AbstractString) return get(T, r, Oid(oid), length(oid) != OID_HEXSZ) end -function path(repo::GitRepo) +function gitdir(repo::GitRepo) return bytestring(ccall((:git_repository_path, :libgit2), Ptr{UInt8}, (Ptr{Void},), repo.ptr)) end +function path(repo::GitRepo) + rpath = gitdir(repo) + return isbare(repo) ? rpath : splitdir(rpath[1:end-1])[1]*"/" # remove '.git' part +end + function peel(obj::GitObject, obj_type::Cint) peeled_ptr_ptr = Ref{Ptr{Void}}(C_NULL) git_otype = getobjecttype(obj_type) @@ -160,4 +165,4 @@ function reset!(repo::GitRepo, obj::GitObject, mode::Cint; finalize(sig) end return -end +end \ No newline at end of file From 52d72cb973f4d77f07bdc6c644208f705a742129 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sun, 17 May 2015 13:32:49 -0400 Subject: [PATCH 0312/1938] import Base for `finalize`s better warning information added `libgit2` tags --- base/pkg/libgit2.jl | 24 +++++++++++------------- base/pkg/libgit2/strarray.jl | 6 ------ base/pkg/libgit2/tag.jl | 27 +++++++++++++++++++++++++++ base/pkg/libgit2/types.jl | 10 ++++++++-- 4 files changed, 46 insertions(+), 21 deletions(-) create mode 100644 base/pkg/libgit2/tag.jl diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 2e0b53bd3d5dd..35d1a30eb2ed6 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -20,6 +20,7 @@ include("libgit2/remote.jl") include("libgit2/strarray.jl") include("libgit2/index.jl") include("libgit2/merge.jl") +include("libgit2/tag.jl") immutable State head::Oid @@ -32,12 +33,6 @@ function normalize_url(url::AbstractString) m == nothing ? url : "git://github.com/$(m.captures[1]).git" end -function gitdir(d, repo::GitRepo) - g = joinpath(d,".git") - isdir(g) && return g - path(repo) -end - """Return HEAD Oid as string""" function head(pkg::AbstractString) with(GitRepo, pkg) do repo @@ -109,7 +104,7 @@ function merge_base(one::AbstractString, two::AbstractString, repo::GitRepo) moid_ptr, repo.ptr, oid1_ptr, oid2_ptr) moid_ptr[] catch e - warn("merge_base: ", e.msg) + #warn("Pkg:",path(repo),"=>",e.msg) Oid() end return moid @@ -172,7 +167,7 @@ function fetch(repo::GitRepo, remote::AbstractString="origin"; try fetch(rmt) catch err - warn("'fetch' thrown exception: $err") + warn("fetch: $err") finally finalize(rmt) end @@ -223,9 +218,13 @@ function branch!(repo::GitRepo, branch_name::AbstractString, end try if !isempty(track) # setup tracking #TODO: what is branch tracks other then "origin" remote - with(GitConfig, repo) do cfg - set!(cfg, "branch.$branch_name.remote", GitConst.REMOTE_ORIGIN) - set!(cfg, "branch.$branch_name.merge", name(branch_ref)) + try + with(GitConfig, repo) do cfg + set!(cfg, "branch.$branch_name.remote", GitConst.REMOTE_ORIGIN) + set!(cfg, "branch.$branch_name.merge", name(branch_ref)) + end + catch + warn("Please provide remote tracking for branch '$branch_name' in '$(path(repo))'") end end @@ -337,9 +336,8 @@ function snapshot(repo::GitRepo) head = Oid(repo, GitConst.HEAD_FILE) index = with(GitIndex, repo) do idx; write_tree!(idx) end work = try - dir = splitdir(path(repo)[1:end-1])[1] # remove `/.git` part with(GitIndex, repo) do idx - content = readdir(dir) + content = readdir(path(repo)) if length(content) > 1 files = [utf8(bytestring(c))::UTF8String for c in content] push!(files, utf8(".")) diff --git a/base/pkg/libgit2/strarray.jl b/base/pkg/libgit2/strarray.jl index b604f2d9b2951..b43d4320effd9 100644 --- a/base/pkg/libgit2/strarray.jl +++ b/base/pkg/libgit2/strarray.jl @@ -20,10 +20,4 @@ function Base.convert(::Type{Vector{AbstractString}}, sa::StrArrayStruct) arr[i] = bytestring(unsafe_load(sa.strings, i)) end return arr -end - -function finalize(sa::StrArrayStruct) - sa_ptr = Ref(sa) - ccall((:git_strarray_free, :libgit2), Void, (Ptr{StrArrayStruct},), sa_ptr) - return sa_ptr[] end \ No newline at end of file diff --git a/base/pkg/libgit2/tag.jl b/base/pkg/libgit2/tag.jl new file mode 100644 index 0000000000000..1d70aaf710f7f --- /dev/null +++ b/base/pkg/libgit2/tag.jl @@ -0,0 +1,27 @@ +function tags(repo::GitRepo) + with(StrArrayStruct()) do sa + sa_ref = Ref(sa) + @check ccall((:git_tag_list, :libgit2), Cint, + (Ptr{StrArrayStruct}, Ptr{Void}), sa_ref, repo.ptr) + convert(Vector{AbstractString}, sa_ref[]) + end +end + +function tag_delete(repo::GitRepo, tag::AbstractString) + @check ccall((:git_tag_delete, :libgit2), Cint, + (Ptr{Void}, Ptr{UInt8}, ), repo.ptr, tag) +end + +function tag_create(repo::GitRepo, tag::AbstractString, commit::AbstractString; + msg::AbstractString = "", + force::Bool = false) + oid_ptr = Ref(Oid()) + with(get(GitCommit, repo, commit)) do commit_obj + with(default_signature(repo)) do sig + @check ccall((:git_tag_create, :libgit2), Cint, + (Ptr{Oid}, Ptr{Void}, Ptr{UInt8}, Ptr{Void}, Ptr{SignatureStruct}, Ptr{UInt8}, Cint), + oid_ptr, repo.ptr, tag, commit_obj.ptr, sig.ptr, msg, Cint(force)) + end + end + return oid_ptr[] +end diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index c08f3344201a1..51d69848407d7 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -18,6 +18,12 @@ immutable StrArrayStruct count::Csize_t end StrArrayStruct() = StrArrayStruct(Ptr{UInt8}(0), zero(Csize_t)) +function Base.finalize(sa::StrArrayStruct) + sa_ptr = Ref(sa) + ccall((:git_strarray_free, :libgit2), Void, (Ptr{StrArrayStruct},), sa_ptr) + return sa_ptr[] +end + immutable CheckoutOptionsStruct version::Cuint @@ -202,7 +208,7 @@ abstract AbstractGitObject Base.isempty(obj::AbstractGitObject) = (obj.ptr == C_NULL) abstract GitObject <: AbstractGitObject -function finalize(obj::GitObject) +function Base.finalize(obj::GitObject) if obj.ptr != C_NULL ccall((:git_object_free, :libgit2), Void, (Ptr{Void},), obj.ptr) obj.ptr = C_NULL @@ -235,7 +241,7 @@ for (typ, ref, sup, fnc) in ( end if fnc != nothing - @eval function finalize(obj::$typ) + @eval function Base.finalize(obj::$typ) if obj.ptr != C_NULL ccall(($fnc, :libgit2), Void, (Ptr{$ref},), obj.ptr) obj.ptr = C_NULL From 3e957e81576f98e76d9dd4e441181cf92b834eec Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sun, 17 May 2015 19:33:43 -0400 Subject: [PATCH 0313/1938] transitioned `tag` & `register` code cleanup & refactoring by default https url generated added blob type --- base/pkg/cache.jl | 4 +- base/pkg/entry.jl | 276 +++++++++++++++++++------------------- base/pkg/generate.jl | 4 +- base/pkg/libgit2.jl | 35 +++-- base/pkg/libgit2/blob.jl | 3 + base/pkg/libgit2/index.jl | 16 ++- base/pkg/libgit2/oid.jl | 21 --- base/pkg/libgit2/tag.jl | 2 +- base/pkg/libgit2/types.jl | 66 +++++++++ base/pkg/read.jl | 9 +- base/pkg/reqs.jl | 11 +- base/pkg/write.jl | 6 +- test/libgit2.jl | 9 ++ 13 files changed, 283 insertions(+), 179 deletions(-) create mode 100644 base/pkg/libgit2/blob.jl diff --git a/base/pkg/cache.jl b/base/pkg/cache.jl index 08d95bcf0247b..106c5a0a3bea5 100644 --- a/base/pkg/cache.jl +++ b/base/pkg/cache.jl @@ -30,6 +30,8 @@ end function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) isdir(".cache") || mkcachedir() + #TODO: force switch to https + #url = LibGit2.normalize_url(url) cache = path(pkg) repo = if isdir(cache) LibGit2.GitRepo(cache) # open repo, free it at the end @@ -55,7 +57,7 @@ function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) end filter(sha1->!LibGit2.iscommit(sha1, repo), sha1s) finally - LibGit2.finalize(repo) # closing repo opened/created above + finalize(repo) # closing repo opened/created above end end prefetch(pkg::AbstractString, url::AbstractString, sha1::AbstractString...) = prefetch(pkg, url, AbstractString[sha1...]) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 6dd6f088a4df8..37379baf1b60d 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -12,6 +12,7 @@ macro recover(ex) try $(esc(ex)) catch err show(err) + print('\n') end end end @@ -111,11 +112,11 @@ function installed(pkg::AbstractString) avail = Read.available(pkg) if Read.isinstalled(pkg) res = typemin(VersionNumber) - repo = LibGit2.GitRepo(pkg) + repo = GitRepo(pkg) try res = Read.installed_version(pkg, repo, avail) finally - LibGit2.finalize(repo) + finalize(repo) end return res end @@ -155,15 +156,15 @@ function status(io::IO, pkg::AbstractString, ver::VersionNumber, fix::Bool) fix || return println(io,ver) @printf io "%-19s" ver if ispath(pkg,".git") - prepo = LibGit2.GitRepo(pkg) + prepo = GitRepo(pkg) try - phead = LibGit2.head(prepo) - if LibGit2.isattached(prepo) - print(io, LibGit2.shortname(phead)) - else - print(io, string(LibGit2.Oid(phead))[1:8]) + with(LibGit2.head(prepo)) do phead + if LibGit2.isattached(prepo) + print(io, LibGit2.shortname(phead)) + else + print(io, string(LibGit2.Oid(phead))[1:8]) + end end - LibGit2.finalize(phead) attrs = AbstractString[] isfile("METADATA",pkg,"url") || push!(attrs,"unregistered") LibGit2.isdirty(prepo) && push!(attrs,"dirty") @@ -171,7 +172,7 @@ function status(io::IO, pkg::AbstractString, ver::VersionNumber, fix::Bool) catch print(io, "broken-repo (unregistered)") finally - LibGit2.finalize(prepo) + finalize(prepo) end else print(io, "non-repo (unregistered)") @@ -183,9 +184,9 @@ function clone(url::AbstractString, pkg::AbstractString) info("Cloning $pkg from $url") ispath(pkg) && error("$pkg already exists") try - repo = LibGit2.clone(url, pkg) - LibGit2.set_remote_url(repo, url) - LibGit2.finalize(repo) + LibGit2.with(LibGit2.clone(url, pkg)) do repo + LibGit2.set_remote_url(repo, url) + end catch Base.rm(pkg, recursive=true) rethrow() @@ -215,15 +216,15 @@ end function checkout(pkg::AbstractString, branch::AbstractString, merge::Bool, pull::Bool) ispath(pkg,".git") || error("$pkg is not a git repo") info("Checking out $pkg $branch...") - LibGit2.with(LibGit2.GitRepo, pkg) do r + with(GitRepo, pkg) do r LibGit2.transact(r) do repo LibGit2.isdirty(repo) && error("$pkg is dirty, bailing") LibGit2.branch!(repo, branch, track=LibGit2.GitConst.REMOTE_ORIGIN) - merge && LibGit2.merge!(repo) # merge changes + merge && LibGit2.merge!(repo, fast_forward=true) # merge changes if pull info("Pulling $pkg latest $branch...") LibGit2.fetch(repo) - LibGit2.merge!(repo) + LibGit2.merge!(repo, fast_forward=true) end resolve() end @@ -235,8 +236,7 @@ function free(pkg::AbstractString) Read.isinstalled(pkg) || error("$pkg cannot be freed – not an installed package") avail = Read.available(pkg) isempty(avail) && error("$pkg cannot be freed – not a registered package") - repo = LibGit2.GitRepo(pkg) - try + with(GitRepo, pkg) do repo LibGit2.isdirty(repo) && error("$pkg cannot be freed – repo is dirty") info("Freeing $pkg") vers = sort!(collect(keys(avail)), rev=true) @@ -253,34 +253,32 @@ function free(pkg::AbstractString) isempty(Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail])) && continue error("can't find any registered versions of $pkg to checkout") end - finally - LibGit2.finalize(repo) end end -function free(pkgs) - try - for pkg in pkgs - ispath(pkg,".git") || error("$pkg is not a git repo") - Read.isinstalled(pkg) || error("$pkg cannot be freed – not an installed package") - avail = Read.available(pkg) - isempty(avail) && error("$pkg cannot be freed – not a registered package") - Git.dirty(dir=pkg) && error("$pkg cannot be freed – repo is dirty") - info("Freeing $pkg") - vers = sort!(collect(keys(avail)), rev=true) - for ver in vers - sha1 = avail[ver].sha1 - Git.iscommit(sha1, dir=pkg) || continue - Git.run(`checkout -q $sha1`, dir=pkg) - break - end - isempty(Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail])) && continue - error("can't find any registered versions of $pkg to checkout") - end - finally - resolve() - end -end +# function free(pkgs) +# try +# for pkg in pkgs +# ispath(pkg,".git") || error("$pkg is not a git repo") +# Read.isinstalled(pkg) || error("$pkg cannot be freed – not an installed package") +# avail = Read.available(pkg) +# isempty(avail) && error("$pkg cannot be freed – not a registered package") +# Git.dirty(dir=pkg) && error("$pkg cannot be freed – repo is dirty") +# info("Freeing $pkg") +# vers = sort!(collect(keys(avail)), rev=true) +# for ver in vers +# sha1 = avail[ver].sha1 +# Git.iscommit(sha1, dir=pkg) || continue +# Git.run(`checkout -q $sha1`, dir=pkg) +# break +# end +# isempty(Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail])) && continue +# error("can't find any registered versions of $pkg to checkout") +# end +# finally +# resolve() +# end +# end function pin(pkg::AbstractString, head::AbstractString) ispath(pkg,".git") || error("$pkg is not a git repo") @@ -308,23 +306,20 @@ end function update(branch::AbstractString) info("Updating METADATA...") - cd("METADATA") do - with(GitRepo, ".") do repo - with(LibGit2.head(repo)) do h - if LibGit2.branch(h) != branch - LibGit2.isdirty(repo) && error("METADATA is dirty and not on $branch, bailing") - LibGit2.isattached(repo) || error("METADATA is detached not on $branch, bailing") - LibGit2.fetch(repo) - LibGit2.checkout_head(repo) - LibGit2.branch!(repo, branch, track="refs/remotes/origin/$branch") - LibGit2.merge!(repo) - end + with(GitRepo, "METADATA") do repo + with(LibGit2.head(repo)) do h + if LibGit2.branch(h) != branch + LibGit2.isdirty(repo) && error("METADATA is dirty and not on $branch, bailing") + LibGit2.isattached(repo) || error("METADATA is detached not on $branch, bailing") + LibGit2.fetch(repo) + LibGit2.checkout_head(repo) + LibGit2.branch!(repo, branch, track="refs/remotes/origin/$branch") + LibGit2.merge!(repo) end end # TODO: handle merge conflicts - # Base.withenv("GIT_MERGE_AUTOEDIT"=>"no") do - # Git.run(`pull --rebase -q`, out=DevNull) - # end + LibGit2.fetch(repo) + LibGit2.merge!(repo) end avail = Read.available() # this has to happen before computing free/fixed @@ -344,7 +339,7 @@ function update(branch::AbstractString) info("Updating $pkg...") @recover begin LibGit2.fetch(repo) - LibGit2.merge!(repo) + LibGit2.merge!(repo, fast_forward=true) end end end @@ -522,9 +517,9 @@ function resolve( build(map(x->x[1], filter(x -> x[2][2] !== nothing, changes))) end -function write_tag_metadata(pkg::AbstractString, ver::VersionNumber, commit::AbstractString, force::Bool=false) - cmd = Git.cmd(`cat-file blob $commit:REQUIRE`, dir=pkg) - reqs = success(cmd) ? Reqs.read(cmd) : Reqs.Line[] +function write_tag_metadata(repo::GitRepo, pkg::AbstractString, ver::VersionNumber, commit::AbstractString, force::Bool=false) + content = LibGit2.cat(repo, LibGit2.GitBlob, "$commit:REQUIRE") + reqs = content != nothing ? Reqs.read(split(content, '\n', keep=false)) : Reqs.Line[] cd("METADATA") do d = joinpath(pkg,"versions",string(ver)) mkpath(d) @@ -535,13 +530,13 @@ function write_tag_metadata(pkg::AbstractString, ver::VersionNumber, commit::Abs error("$pkg v$ver is already registered as $current, bailing") end open(io->println(io,commit), sha1file, "w") - Git.run(`add $sha1file`) + LibGit2.add!(repo, sha1file) reqsfile = joinpath(d,"requires") if isempty(reqs) - ispath(reqsfile) && Git.run(`rm -f -q $reqsfile`) + ispath(reqsfile) && LibGit2.remove!(repo, reqsfile) else Reqs.write(reqsfile,reqs) - Git.run(`add $reqsfile`) + LibGit2.add!(repo, reqsfile) end end return nothing @@ -550,44 +545,55 @@ end function register(pkg::AbstractString, url::AbstractString) ispath(pkg,".git") || error("$pkg is not a git repo") isfile("METADATA",pkg,"url") && error("$pkg already registered") - tags = split(Git.readall(`tag -l v*`, dir=pkg)) - filter!(tag->ismatch(Base.VERSION_REGEX,tag), tags) - versions = [ - convert(VersionNumber,tag) => - Git.readchomp(`rev-parse --verify $tag^{commit}`, dir=pkg) - for tag in tags - ] - Git.transact(dir="METADATA") do + LibGit2.transact(GitRepo("METADATA")) do repo + # Get versions from package repo + versions = with(GitRepo, pkg) do pkg_repo + tags = filter(t->startswith(t,"v"), LibGit2.tag_list(pkg_repo)) + filter!(tag->ismatch(Base.VERSION_REGEX,tag), tags) + [ + convert(VersionNumber,tag) => string(LibGit2.revparseid(pkg_repo, "$tag^{commit}")) + for tag in tags + ] + end + # Register package url in METADATA cd("METADATA") do info("Registering $pkg at $url") mkdir(pkg) path = joinpath(pkg,"url") open(io->println(io,url), path, "w") - Git.run(`add $path`) + LibGit2.add!(repo, path) end + # Register package version in METADATA vers = sort!(collect(keys(versions))) for ver in vers info("Tagging $pkg v$ver") - write_tag_metadata(pkg,ver,versions[ver]) + write_tag_metadata(repo, pkg,ver,versions[ver]) end - if Git.staged(dir="METADATA") + # Commit changes in METADATA + if LibGit2.isdirty(repo) info("Committing METADATA for $pkg") msg = "Register $pkg" if !isempty(versions) msg *= ": $(join(map(v->"v$v", vers),", "))" end - Git.run(`commit -q -m $msg -- $pkg`, dir="METADATA") + LibGit2.commit(repo, msg) else info("No METADATA changes to commit") end end + return end function register(pkg::AbstractString) - url = LibGit2.with(LibGit2.GitRepo, pkg) do repo - LibGit2.with(LibGit2.GitConfig, repo) do cfg - LibGit2.get(cfg, "remote.origin.url", "") + url = "" + try + url = with(GitRepo, pkg) do repo + with(GitConfig, repo) do cfg + LibGit2.get(cfg, "remote.origin.url", "") + end end + catch err + error("$pkg: $err") end isempty(url) || error("$pkg: no URL configured") register(pkg, LibGit2.normalize_url(url)) @@ -601,67 +607,67 @@ end nextbump(v::VersionNumber) = isrewritable(v) ? v : nextpatch(v) -function tag(pkg::AbstractString, ver::Union{Symbol,VersionNumber}, force::Bool=false, commit::AbstractString="HEAD") +function tag(pkg::AbstractString, ver::Union(Symbol,VersionNumber), force::Bool=false, commitish::AbstractString="HEAD") ispath(pkg,".git") || error("$pkg is not a git repo") - Git.dirty(dir=pkg) && - error("$pkg is dirty – commit or stash changes to tag") - Git.dirty(pkg, dir="METADATA") && - error("METADATA/$pkg is dirty – commit or stash changes to tag") - commit = Git.readchomp(`rev-parse $commit`, dir=pkg) - registered = isfile("METADATA",pkg,"url") - if !force - if registered - avail = Read.available(pkg) - existing = VersionNumber[keys(Read.available(pkg))...] - ancestors = filter(v->Git.is_ancestor_of(avail[v].sha1,commit,dir=pkg), existing) - else - tags = split(Git.readall(`tag -l v*`, dir=pkg)) - filter!(tag->ismatch(Base.VERSION_REGEX,tag), tags) - existing = VersionNumber[tags...] - filter!(tags) do tag - sha1 = Git.readchomp(`rev-parse --verify $tag^{commit}`, dir=pkg) - Git.is_ancestor_of(sha1,commit,dir=pkg) - end - ancestors = VersionNumber[tags...] - end - sort!(existing) - if isa(ver,Symbol) - prv = isempty(existing) ? v"0" : - isempty(ancestors) ? maximum(existing) : maximum(ancestors) - ver = (ver == :bump ) ? nextbump(prv) : - (ver == :patch) ? nextpatch(prv) : - (ver == :minor) ? nextminor(prv) : - (ver == :major) ? nextmajor(prv) : - error("invalid version selector: $ver") - end - isrewritable(ver) && filter!(v->v!=ver,existing) - check_new_version(existing,ver) - end - # TODO: check that SHA1 isn't the same as another version - info("Tagging $pkg v$ver") - opts = `` - if force || isrewritable(ver) - opts = `$opts --force` - end - if !isrewritable(ver) - opts = `$opts --annotate --message "$pkg v$ver [$(commit[1:10])]"` - end - Git.run(`tag $opts v$ver $commit`, dir=pkg, out=DevNull) - registered || return - try - Git.transact(dir="METADATA") do - write_tag_metadata(pkg,ver,commit,force) - if Git.staged(dir="METADATA") - info("Committing METADATA for $pkg") - Git.run(`commit -q -m "Tag $pkg v$ver" -- $pkg`, dir="METADATA") + with(GitRepo,"METADATA") do repo + LibGit2.isdirty(repo, pkg) && error("METADATA/$pkg is dirty – commit or stash changes to tag") + end + with(GitRepo,pkg) do repo + LibGit2.isdirty(repo) && error("$pkg is dirty – commit or stash changes to tag") + commit = string(LibGit2.revparseid(repo, commitish)) + registered = isfile("METADATA",pkg,"url") + + if !force + if registered + avail = Read.available(pkg) + existing = VersionNumber[keys(Read.available(pkg))...] + ancestors = filter(v->LibGit2.is_ancestor_of(avail[v].sha1, commit, repo), existing) else - info("No METADATA changes to commit") + tags = filter(t->startswith(t,"v"), Pkg.LibGit2.tag_list(repo)) + filter!(tag->ismatch(Base.VERSION_REGEX,tag), tags) + existing = VersionNumber[tags...] + filter!(tags) do tag + sha1 = LibGit2.revparseid(repo, "$tag^{commit}") + LibGit2.is_ancestor_of(sha1, commit, repo) + end + ancestors = VersionNumber[tags...] + end + sort!(existing) + if isa(ver,Symbol) + prv = isempty(existing) ? v"0" : + isempty(ancestors) ? maximum(existing) : maximum(ancestors) + ver = (ver == :bump ) ? nextbump(prv) : + (ver == :patch) ? nextpatch(prv) : + (ver == :minor) ? nextminor(prv) : + (ver == :major) ? nextmajor(prv) : + error("invalid version selector: $ver") + end + isrewritable(ver) && filter!(v->v!=ver,existing) + check_new_version(existing,ver) + end + # TODO: check that SHA1 isn't the same as another version + info("Tagging $pkg v$ver") + LibGit2.tag_create(repo, "v$ver", commit, + msg=(!isrewritable(ver) ? "$pkg v$ver [$(commit[1:10])]" : ""), + force=(force || isrewritable(ver)) ) + registered || return + try + LibGit2.transact(GitRepo("METADATA")) do repo + write_tag_metadata(repo, pkg, ver, commit, force) + if LibGit2.isdirty(repo) + info("Committing METADATA for $pkg") + LibGit2.commit(repo, "Tag $pkg v$ver") + #run(`commit -q -m "Tag $pkg v$ver" -- $pkg`, dir="METADATA") + else + info("No METADATA changes to commit") + end end + catch + LibGit2.tag_delete(repo, "v$ver") + rethrow() end - catch - Git.run(`tag -d v$ver`, dir=pkg) - rethrow() end + return end function check_metadata(pkgs::Set{ByteString} = Set{ByteString}()) diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index f25e7491a7ca2..343c25234903a 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -44,12 +44,12 @@ function package( isnew = !ispath(pkg) try repo = if isnew - url = isempty(user) ? "" : "git://github.com/$user/$pkg.jl.git" + url = isempty(user) ? "" : "https://github.com/$user/$pkg.jl.git" Generate.init(pkg,url,config=config) else repo = GitRepo(pkg) if LibGit2.isdirty(repo) - LibGit2.finalize(repo) + finalize(repo) error("$pkg is dirty – commit or stash your changes") end repo diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 35d1a30eb2ed6..549317f59284a 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -21,6 +21,7 @@ include("libgit2/strarray.jl") include("libgit2/index.jl") include("libgit2/merge.jl") include("libgit2/tag.jl") +include("libgit2/blob.jl") immutable State head::Oid @@ -30,7 +31,7 @@ end function normalize_url(url::AbstractString) m = match(GITHUB_REGEX,url) - m == nothing ? url : "git://github.com/$(m.captures[1]).git" + m == nothing ? url : "https://github.com/$(m.captures[1]).git" end """Return HEAD Oid as string""" @@ -61,10 +62,10 @@ function iscommit(id::AbstractString, repo::GitRepo) end """ git diff-index HEAD [-- ]""" -isdirty(repo::GitRepo, paths::AbstractString="") = isdiff(repo, GitConst.HEAD_FILE, paths) +isdirty(repo::GitRepo, paths::AbstractString=""; cached::Bool=false) = isdiff(repo, GitConst.HEAD_FILE, paths, cached=cached) """ git diff-index [-- ]""" -function isdiff(repo::GitRepo, treeish::AbstractString, paths::AbstractString="") +function isdiff(repo::GitRepo, treeish::AbstractString, paths::AbstractString=""; cached::Bool=false) tree_oid = revparseid(repo, "$treeish^{tree}") iszero(tree_oid) && return true emptypathspec = isempty(paths) @@ -77,9 +78,15 @@ function isdiff(repo::GitRepo, treeish::AbstractString, paths::AbstractString="" end try diff_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - @check ccall((:git_diff_tree_to_workdir_with_index, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{DiffOptionsStruct}), - diff_ptr_ptr, repo.ptr, tree.ptr, emptypathspec ? C_NULL : Ref(diff_opts)) + if cached + @check ccall((:git_diff_tree_to_index, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, Ptr{DiffOptionsStruct}), + diff_ptr_ptr, repo.ptr, tree.ptr, NULL, emptypathspec ? C_NULL : Ref(diff_opts)) + else + @check ccall((:git_diff_tree_to_workdir_with_index, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{DiffOptionsStruct}), + diff_ptr_ptr, repo.ptr, tree.ptr, emptypathspec ? C_NULL : Ref(diff_opts)) + end diff = GitDiff(diff_ptr_ptr[]) c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff.ptr) @@ -314,7 +321,6 @@ end """ git reset [--soft | --mixed | --hard] """ function reset!(repo::GitRepo, commit::Oid, mode::Cint = GitConst.RESET_MIXED) - obj = get(GitAnyObject, repo, commit) obj == nothing && return try @@ -324,11 +330,24 @@ function reset!(repo::GitRepo, commit::Oid, mode::Cint = GitConst.RESET_MIXED) end end +""" git cat-file """ +function cat{T<:GitObject}(repo::GitRepo, ::Type{T}, object::AbstractString) + obj_id = revparseid(repo, object) + iszero(obj_id) && return + + obj = get(T, repo, obj_id) + if isa(obj, GitBlob) + return bytestring(convert(Ptr{UInt8}, content(obj))) + else + return nothing + end +end + """ Returns all commit authors """ function authors(repo::GitRepo) athrs = map( (oid,repo)->author(get(GitCommit, repo, oid))::Signature, - repo) #, by = Pkg.LibGit2.GitConst.SORT_TIME) + repo) #, by = GitConst.SORT_TIME) return athrs end diff --git a/base/pkg/libgit2/blob.jl b/base/pkg/libgit2/blob.jl new file mode 100644 index 0000000000000..59440c560a2c7 --- /dev/null +++ b/base/pkg/libgit2/blob.jl @@ -0,0 +1,3 @@ +function content(blob::GitBlob) + return ccall((:git_blob_rawcontent, :libgit2), Ptr{Void}, (Ptr{Void},), blob.ptr) +end \ No newline at end of file diff --git a/base/pkg/libgit2/index.jl b/base/pkg/libgit2/index.jl index a1132991caee8..8aa25e82f6618 100644 --- a/base/pkg/libgit2/index.jl +++ b/base/pkg/libgit2/index.jl @@ -99,4 +99,18 @@ function read!(repo::GitRepo, force::Bool = false) with(GitIndex, repo) do idx read!(idx, force) end -end \ No newline at end of file +end + +function Base.count(idx::GitIndex) + return ccall((:git_index_entrycount, :libgit2), Csize_t, (Ptr{Void},), idx.ptr) +end + +function Base.getindex(idx::GitIndex, i::Csize_t) + # return ccall((:git_index_get_byindex, :libgit2), Ptr{Void}, + # (Ptr{Void}, Csize_t), idx.ptr, i) + ie_ptr = ccall((:git_index_get_byindex, :libgit2), Ptr{Void}, + (Ptr{Void}, Csize_t), idx.ptr, i) + ie_ptr == C_NULL && return nothing + return unsafe_load(convert(Ptr{IndexEntry}, ie_ptr), 1) +end +Base.getindex(idx::GitIndex, i::Int) = getindex(idx, Csize_t(i)) \ No newline at end of file diff --git a/base/pkg/libgit2/oid.jl b/base/pkg/libgit2/oid.jl index e0799b9579b2d..3c4b9a905e655 100644 --- a/base/pkg/libgit2/oid.jl +++ b/base/pkg/libgit2/oid.jl @@ -1,24 +1,3 @@ -const OID_RAWSZ = 20 -const OID_HEXSZ = OID_RAWSZ * 2 -const OID_MINPREFIXLEN = 4 - -#TODO: The Oid generated code should now use an immutable type that wraps an ntuple of length 20 -# immutable Oid -# id1::UInt8 -# id2::UInt8 -# ... -# id20::UInt8 -# end -@eval begin - $(Expr(:type, false, :Oid, - Expr(:block, - [Expr(:(::), symbol("id$i"), :UInt8) for i=1:OID_RAWSZ]...))) -end - -# default Oid constructor (all zeros) -@generated function Oid() - return Expr(:call, :Oid, [:(zero(UInt8)) for _=1:OID_RAWSZ]...) -end Oid(id::Oid) = id Oid(ptr::Ptr{Oid}) = unsafe_load(ptr)::Oid diff --git a/base/pkg/libgit2/tag.jl b/base/pkg/libgit2/tag.jl index 1d70aaf710f7f..00bfd9899088c 100644 --- a/base/pkg/libgit2/tag.jl +++ b/base/pkg/libgit2/tag.jl @@ -1,4 +1,4 @@ -function tags(repo::GitRepo) +function tag_list(repo::GitRepo) with(StrArrayStruct()) do sa sa_ref = Ref(sa) @check ccall((:git_tag_list, :libgit2), Cint, diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index 51d69848407d7..e24d6fa5c71b5 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -1,3 +1,25 @@ +const OID_RAWSZ = 20 +const OID_HEXSZ = OID_RAWSZ * 2 +const OID_MINPREFIXLEN = 4 + +#TODO: The Oid generated code should now use an immutable type that wraps an ntuple of length 20 +# immutable Oid +# id1::UInt8 +# id2::UInt8 +# ... +# id20::UInt8 +# end +@eval begin + $(Expr(:type, false, :Oid, + Expr(:block, + [Expr(:(::), symbol("id$i"), :UInt8) for i=1:OID_RAWSZ]...))) +end + +# default Oid constructor (all zeros) +@generated function Oid() + return Expr(:call, :Oid, [:(zero(UInt8)) for _=1:OID_RAWSZ]...) +end + immutable TimeStruct time::Int64 # time in seconds from epoch offset::Cint # timezone offset in minutes @@ -203,6 +225,45 @@ MergeOptionsStruct(; flags::Cint = Cint(0), file_favor ) +immutable IndexTime + seconds::Int64 + nanoseconds::Cuint + IndexTime() = new(zero(Int64), zero(Cuint)) +end + +immutable IndexEntry + ctime::IndexTime + mtime::IndexTime + + dev::Cuint + ino::Cuint + mode::Cuint + uid::Cuint + gid::Cuint + file_size::Int64 + + id::Oid + + flags::UInt16 + flags_extended::UInt16 + + path::Ptr{UInt8} +end +IndexEntry() = IndexEntry(IndexTime(), + IndexTime(), + Cuint(0), + Cuint(0), + Cuint(0), + Cuint(0), + Cuint(0), + Cuint(0), + Oid(), + UInt16(0), + UInt16(0), + Ptr{UInt8}(0)) +Base.show(io::IO, ie::IndexEntry) = print(io, "IndexEntry($(string(ie.id)))") + + # Abstract object types abstract AbstractGitObject Base.isempty(obj::AbstractGitObject) = (obj.ptr == C_NULL) @@ -228,6 +289,7 @@ for (typ, ref, sup, fnc) in ( (:GitSignature, :SignatureStruct, :AbstractGitObject, :(:git_signature_free)), (:GitAnyObject, :Void, :GitObject, nothing), (:GitCommit, :Void, :GitObject, nothing), + (:GitBlob, :Void, :GitObject, nothing), (:GitTree, :Void, :GitObject, nothing) ) @@ -286,6 +348,8 @@ function getobjecttype{T<:GitObject}(::Type{T}) GitConst.OBJ_COMMIT elseif T == GitTree GitConst.OBJ_TREE + elseif T == GitBlob + GitConst.OBJ_BLOB elseif T == GitAnyObject GitConst.OBJ_ANY else @@ -298,6 +362,8 @@ function getobjecttype(obj_type::Cint) GitCommit elseif obj_type == GitConst.OBJ_TREE GitTree + elseif obj_type == GitConst.OBJ_BLOB + GitBlob elseif obj_type == GitConst.OBJ_ANY GitAnyObject else diff --git a/base/pkg/read.jl b/base/pkg/read.jl index 74cc7c1854c90..fcc13a658f4d7 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -148,14 +148,11 @@ function requires_path(pkg::AbstractString, avail::Dict=available(pkg)) pkgreq = joinpath(pkg,"REQUIRE") ispath(pkg,".git") || return pkgreq repo = LibGit2.GitRepo(pkg) - head = "" - try + head = LibGit2.with(LibGit2.GitRepo, pkg) do repo LibGit2.isdirty(repo, "REQUIRE") && return pkgreq LibGit2.need_update(repo) LibGit2.iszero(LibGit2.revparseid(repo, "HEAD:REQUIRE")) && isfile(pkgreq) && return pkgreq - head = string(LibGit2.head_oid(repo)) - finally - LibGit2.finalize(repo) + string(LibGit2.head_oid(repo)) end for (ver,info) in avail if head == info.sha1 @@ -185,7 +182,7 @@ function installed(avail::Dict=available()) catch e pkgs[pkg] = (typemin(VersionNumber), true) finally - LibGit2.finalize(prepo) + finalize(prepo) end catch pkgs[pkg] = (typemin(VersionNumber), true) diff --git a/base/pkg/reqs.jl b/base/pkg/reqs.jl index 76226aaf698f6..aa6331e1ef5f9 100644 --- a/base/pkg/reqs.jl +++ b/base/pkg/reqs.jl @@ -54,7 +54,16 @@ hash(s::Line, h::UInt) = hash(s.content, h + (0x3f5a631add21cb1a % UInt)) # general machinery for parsing REQUIRE files -function read(readable::Union{IO,Base.AbstractCmd}) +function read{T<:AbstractString}(readable::Vector{T}) + lines = Line[] + for line in readable + line = chomp(line) + push!(lines, ismatch(r"^\s*(?:#|$)", line) ? Comment(line) : Requirement(line)) + end + return lines +end + +function read(readable::Union(IO,Base.AbstractCmd)) lines = Line[] for line in eachline(readable) line = chomp(line) diff --git a/base/pkg/write.jl b/base/pkg/write.jl index ea4e1cdde30f1..08e6953fc51e0 100644 --- a/base/pkg/write.jl +++ b/base/pkg/write.jl @@ -34,8 +34,8 @@ end function install(pkg::AbstractString, sha1::AbstractString) prefetch(pkg, sha1) repo = if isdir(".trash/$pkg") - mv(".trash/$pkg", "./$pkg") - LibGit2.GitRepo(pkg) + mv(".trash/$pkg", "./$pkg") #TODO check for newer version in cache before moving + GitRepo(pkg) else LibGit2.clone(Cache.path(pkg), pkg) end @@ -43,7 +43,7 @@ function install(pkg::AbstractString, sha1::AbstractString) fetch(repo, pkg, sha1) checkout(repo, pkg, sha1) finally - LibGit2.finalize(repo) + finalize(repo) end end diff --git a/test/libgit2.jl b/test/libgit2.jl index d819a64088df6..0e36d36113e2d 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -38,6 +38,7 @@ function temp_dir(fn::Function, remove_tmp_dir::Bool=true) end end +println("test 1") # clone bare temp_dir() do dir url = "https://github.com/JuliaLang/Example.jl" @@ -48,6 +49,7 @@ temp_dir() do dir @test isfile(joinpath(path, Pkg.LibGit2.GitConst.HEAD_FILE)) end +println("test 2") # clone temp_dir() do dir url = "https://github.com/JuliaLang/Example.jl" @@ -58,6 +60,7 @@ temp_dir() do dir @test isdir(joinpath(path, ".git")) end +println("test 3") # init temp_dir() do dir path = joinpath(dir, "Example") @@ -82,6 +85,7 @@ temp_dir() do dir Pkg.LibGit2.finalize(repo) end +println("test 4") # fetch temp_dir() do dir_cache # create cache @@ -91,6 +95,7 @@ temp_dir() do dir_cache Pkg.LibGit2.finalize(repo) + println("test 4.1") # fetch temp_dir() do dir # clone repo @@ -110,6 +115,7 @@ temp_dir() do dir_cache @test refs2 > refs1 end + println("test 4.2") # revwalk temp_dir() do dir path = joinpath(dir, "Example") @@ -125,6 +131,7 @@ temp_dir() do dir_cache end end + println("test 4.3") # signature temp_dir() do dir path = joinpath(dir, "Example") @@ -150,6 +157,7 @@ temp_dir() do dir_cache @test sig.time == sig2.time end + println("test 4.4") # transact temp_dir() do dir # clone repo @@ -172,6 +180,7 @@ temp_dir() do dir_cache end +println("test 5") # strarray begin p1 = "XXX" From 3ff4628ddfc5278d0e96e4a3fc63e411acab1207 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sun, 17 May 2015 23:58:01 -0400 Subject: [PATCH 0314/1938] fixed `get` with default for GitConfig fixed `libgit2` tests --- base/pkg/libgit2.jl | 2 +- base/pkg/libgit2/config.jl | 4 +++- base/pkg/libgit2/repository.jl | 19 +++++++++---------- test/libgit2.jl | 27 +++++++++++++++++++++------ 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 549317f59284a..79f290ab4e6a4 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -172,7 +172,7 @@ function fetch(repo::GitRepo, remote::AbstractString="origin"; end try - fetch(rmt) + fetch(repo, rmt, msg="from $(url(rmt))") catch err warn("fetch: $err") finally diff --git a/base/pkg/libgit2/config.jl b/base/pkg/libgit2/config.jl index 6fb84ffaa5804..a724e679e86ce 100644 --- a/base/pkg/libgit2/config.jl +++ b/base/pkg/libgit2/config.jl @@ -48,7 +48,9 @@ function get{T}(::Type{T}, c::GitConfig, name::AbstractString) end function get{T}(c::GitConfig, name::AbstractString, default::T) - return try get(T,c,name) catch default end + res = default + try res = get(T,c,name) end + return res end function set!{T}(c::GitConfig, name::AbstractString, value::T) diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index 9ca2e815f4650..431ae30349745 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -134,10 +134,13 @@ function checkout_head(repo::GitRepo; options::CheckoutOptionsStruct = CheckoutO repo.ptr, Ref(options)) end -function fetch(rmt::GitRemote) - @check ccall((:git_remote_fetch, :libgit2), Cint, - (Ptr{Void}, Ptr{Void}, Ptr{UInt8}), - rmt.ptr, C_NULL, C_NULL) +function fetch(repo::GitRepo, rmt::GitRemote; msg::AbstractString="") + msg = "pkg.libgit2.fetch: $msg" + with(default_signature(repo)) do sig + @check ccall((:git_remote_fetch, :libgit2), Cint, + (Ptr{Void}, Ptr{Void}, Ptr{SignatureStruct}, Ptr{UInt8}), + rmt.ptr, C_NULL, sig.ptr, msg) + end end function reset!(repo::GitRepo, obj::Nullable{GitAnyObject}, pathspecs::AbstractString...) @@ -155,14 +158,10 @@ end function reset!(repo::GitRepo, obj::GitObject, mode::Cint; checkout_opts::CheckoutOptionsStruct = CheckoutOptionsStruct()) - sig = default_signature(repo) - msg = "pkg.libgit2.reset: moving to $(string(Oid(obj)))" - try + with(default_signature(repo)) do sig + msg = "pkg.libgit2.reset: moving to $(string(Oid(obj)))" @check ccall((:git_reset, :libgit2), Cint, (Ptr{Void}, Ptr{Void}, Cint, Ptr{CheckoutOptionsStruct}, Ptr{SignatureStruct}, Ptr{UInt8}), repo.ptr, obj.ptr, mode, Ref(checkout_opts), sig.ptr, msg) - finally - finalize(sig) end - return end \ No newline at end of file diff --git a/test/libgit2.jl b/test/libgit2.jl index 0e36d36113e2d..26126d20a128d 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -2,7 +2,7 @@ # check that libgit2 has been installed correctly -const LIBGIT2_VER = v"0.22+" +const LIBGIT2_VER = v"0.22.2+" function check_version() major, minor, patch = Cint[0], Cint[0], Cint[0] @@ -18,12 +18,16 @@ end @test check_version() +function credentials!(cfg::Pkg.LibGit2.GitConfig, usr="Test User", usr_email="Test@User.com") + git_user = Pkg.LibGit2.get(cfg, "user.name", usr) + usr==git_user && Pkg.LibGit2.set!(cfg, "user.name", usr) + git_user_email = Pkg.LibGit2.get(cfg, "user.email", usr_email) + usr_email==git_user_email && Pkg.LibGit2.set!(cfg, "user.email", usr_email) +end + #TODO: tests need 'user.name' & 'user.email' in config ??? Pkg.LibGit2.with(Pkg.LibGit2.GitConfig) do cfg - git_user = Pkg.LibGit2.get(cfg, "user.name", "") - isempty(git_user) && Pkg.LibGit2.set(cfg, "user.email", "Test User") - git_user_email = Pkg.LibGit2.get(cfg, "user.email", "") - isempty(git_user) && Pkg.LibGit2.set(cfg, "user.email", "Test@User.com") + credentials!(cfg) end function temp_dir(fn::Function, remove_tmp_dir::Bool=true) @@ -92,6 +96,9 @@ temp_dir() do dir_cache url = "https://github.com/JuliaLang/Example.jl" path_cache = joinpath(dir_cache, "Example.Bare") repo = Pkg.LibGit2.clone(url, path_cache, bare = true, remote_cb = Pkg.LibGit2.mirror_cb) + Pkg.LibGit2.with(Pkg.LibGit2.GitConfig, repo) do cfg + credentials!(cfg) + end Pkg.LibGit2.finalize(repo) @@ -101,8 +108,10 @@ temp_dir() do dir_cache # clone repo path = joinpath(dir, "Example") repo = Pkg.LibGit2.clone(path_cache, path) + Pkg.LibGit2.with(Pkg.LibGit2.GitConfig, repo) do cfg + credentials!(cfg) + end - # fetch Pkg.LibGit2.fetch(repo) refs1 = parse(Int, readchomp(pipe(`find $(joinpath(path, ".git/refs"))`,`wc -l`))) @@ -120,6 +129,9 @@ temp_dir() do dir_cache temp_dir() do dir path = joinpath(dir, "Example") repo = Pkg.LibGit2.clone(path_cache, path) + Pkg.LibGit2.with(Pkg.LibGit2.GitConfig, repo) do cfg + credentials!(cfg) + end oids = Pkg.LibGit2.map((oid,repo)->string(oid), repo, by = Pkg.LibGit2.GitConst.SORT_TIME) @test length(oids) > 0 Pkg.LibGit2.finalize(repo) @@ -163,6 +175,9 @@ temp_dir() do dir_cache # clone repo path = joinpath(dir, "Example") repo = Pkg.LibGit2.clone(path_cache, path) + Pkg.LibGit2.with(Pkg.LibGit2.GitConfig, repo) do cfg + credentials!(cfg) + end cp(joinpath(path, "REQUIRE"), joinpath(path, "CCC")) cp(joinpath(path, "REQUIRE"), joinpath(path, "AAA")) Pkg.LibGit2.add!(repo, "AAA") From c9d04e2b3d92d5116559d058ee6c55d859045607 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Mon, 18 May 2015 01:48:22 -0400 Subject: [PATCH 0315/1938] fixed errors, set to run only pkg & libgit2 tests --- base/pkg/generate.jl | 4 ++-- test/libgit2.jl | 9 --------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index 343c25234903a..9f18a6eb6d15b 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -5,7 +5,7 @@ module Generate import ..LibGit2, ..Read importall ..LibGit2 -copyright_year() = Dates.year(Dates.today()) +copyright_year() = string(Dates.year(Dates.today())) copyright_name(repo::GitRepo) = with(GitConfig, repo) do cfg LibGit2.get(cfg, "user.name", "") @@ -150,7 +150,7 @@ function license(pkg::AbstractString, end print(io, LICENSES[license](pkg, string(years), authors)) end - isempty(file) || info("License file exists, leaving unmodified; use `force=true` to overwrite") + !isempty(file) || info("License file exists, leaving unmodified; use `force=true` to overwrite") file end diff --git a/test/libgit2.jl b/test/libgit2.jl index 26126d20a128d..2d2f83e44e525 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -42,7 +42,6 @@ function temp_dir(fn::Function, remove_tmp_dir::Bool=true) end end -println("test 1") # clone bare temp_dir() do dir url = "https://github.com/JuliaLang/Example.jl" @@ -53,7 +52,6 @@ temp_dir() do dir @test isfile(joinpath(path, Pkg.LibGit2.GitConst.HEAD_FILE)) end -println("test 2") # clone temp_dir() do dir url = "https://github.com/JuliaLang/Example.jl" @@ -64,7 +62,6 @@ temp_dir() do dir @test isdir(joinpath(path, ".git")) end -println("test 3") # init temp_dir() do dir path = joinpath(dir, "Example") @@ -89,7 +86,6 @@ temp_dir() do dir Pkg.LibGit2.finalize(repo) end -println("test 4") # fetch temp_dir() do dir_cache # create cache @@ -102,7 +98,6 @@ temp_dir() do dir_cache Pkg.LibGit2.finalize(repo) - println("test 4.1") # fetch temp_dir() do dir # clone repo @@ -124,7 +119,6 @@ temp_dir() do dir_cache @test refs2 > refs1 end - println("test 4.2") # revwalk temp_dir() do dir path = joinpath(dir, "Example") @@ -143,7 +137,6 @@ temp_dir() do dir_cache end end - println("test 4.3") # signature temp_dir() do dir path = joinpath(dir, "Example") @@ -169,7 +162,6 @@ temp_dir() do dir_cache @test sig.time == sig2.time end - println("test 4.4") # transact temp_dir() do dir # clone repo @@ -195,7 +187,6 @@ temp_dir() do dir_cache end -println("test 5") # strarray begin p1 = "XXX" From d409e13901dd5fd6ac3e4124364aaf7a10255c7b Mon Sep 17 00:00:00 2001 From: Art Wild Date: Tue, 19 May 2015 16:29:30 -0400 Subject: [PATCH 0316/1938] added PkgError exception --- base/pkg/cache.jl | 8 +--- base/pkg/dir.jl | 2 +- base/pkg/entry.jl | 94 +++++++++++++++++++------------------- base/pkg/generate.jl | 6 +-- base/pkg/github.jl | 24 +++++----- base/pkg/libgit2.jl | 2 +- base/pkg/libgit2/types.jl | 4 +- base/pkg/query.jl | 4 +- base/pkg/read.jl | 2 +- base/pkg/reqs.jl | 6 +-- base/pkg/resolve.jl | 2 +- base/pkg/resolve/maxsum.jl | 2 +- base/pkg/types.jl | 4 ++ base/pkg/write.jl | 6 +-- 14 files changed, 83 insertions(+), 83 deletions(-) diff --git a/base/pkg/cache.jl b/base/pkg/cache.jl index 106c5a0a3bea5..296868febf558 100644 --- a/base/pkg/cache.jl +++ b/base/pkg/cache.jl @@ -42,18 +42,14 @@ function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) LibGit2.clone(url, cache, bare = true, remote_cb = LibGit2.mirror_cb) catch err isdir(cache) && rm(cache, recursive=true) - error("Cannot clone $pkg from $url\n", err) + throw(PkgError("Cannot clone $pkg from $url\n", err)) end end try LibGit2.set_remote_url(repo, url) if !all(sha1->LibGit2.iscommit(sha1, repo), sha1s) info("Updating cache of $pkg...") - try - LibGit2.fetch(repo) - catch - error("couldn't update $cache using `git remote update`") - end + LibGit2.fetch(repo) end filter(sha1->!LibGit2.iscommit(sha1, repo), sha1s) finally diff --git a/base/pkg/dir.jl b/base/pkg/dir.jl index cea1f69146cd7..b511de1698f45 100644 --- a/base/pkg/dir.jl +++ b/base/pkg/dir.jl @@ -26,7 +26,7 @@ function cd(f::Function, args...; kws...) metadata_dir = joinpath(dir, "METADATA") if !isdir(metadata_dir) !haskey(ENV,"JULIA_PKGDIR") ? init() : - error("Package metadata directory $metadata_dir doesn't exist; run Pkg.init() to initialize it.") + throw(PkgError("Package metadata directory $metadata_dir doesn't exist; run Pkg.init() to initialize it.")) end Base.cd(()->f(args...; kws...), dir) end diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 37379baf1b60d..a6a49d6e01d0c 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -33,8 +33,8 @@ end function edit() editor = get(ENV,"VISUAL",get(ENV,"EDITOR",nothing)) - editor !== nothing || - error("set the EDITOR environment variable to an edit command") + editor != nothing || + throw(PkgError("set the EDITOR environment variable to an edit command")) editor = Base.shell_split(editor) reqs = Reqs.parse("REQUIRE") run(`$editor REQUIRE`) @@ -48,7 +48,7 @@ function add(pkg::AbstractString, vers::VersionSet) outdated = :maybe @sync begin @async if !edit(Reqs.add,pkg,vers) - ispath(pkg) || error("unknown package $pkg") + ispath(pkg) || throw(PkgError("unknown package $pkg")) info("Nothing to be done") end branch = Dir.getmetabranch() @@ -97,7 +97,7 @@ function available(pkg::AbstractString) if !isempty(avail) || Read.isinstalled(pkg) return sort!(collect(keys(avail))) end - error("$pkg is not a package (not registered or installed)") + throw(PkgError("$pkg is not a package (not registered or installed)")) end function installed() @@ -120,7 +120,7 @@ function installed(pkg::AbstractString) end return res end - isempty(avail) && error("$pkg is not a package (not registered or installed)") + isempty(avail) && throw(PkgError("$pkg is not a package (not registered or installed)")) return nothing # registered but not installed end @@ -182,7 +182,7 @@ end function clone(url::AbstractString, pkg::AbstractString) info("Cloning $pkg from $url") - ispath(pkg) && error("$pkg already exists") + ispath(pkg) && throw(PkgError("$pkg already exists")) try LibGit2.with(LibGit2.clone(url, pkg)) do repo LibGit2.set_remote_url(repo, url) @@ -207,18 +207,18 @@ function clone(url_or_pkg::AbstractString) else url = url_or_pkg m = match(r"(?:^|[/\\])(\w+?)(?:\.jl)?(?:\.git)?$", url) - m !== nothing || error("can't determine package name from URL: $url") + m != nothing || throw(PkgError("can't determine package name from URL: $url")) pkg = m.captures[1] end clone(url,pkg) end function checkout(pkg::AbstractString, branch::AbstractString, merge::Bool, pull::Bool) - ispath(pkg,".git") || error("$pkg is not a git repo") + ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) info("Checking out $pkg $branch...") with(GitRepo, pkg) do r LibGit2.transact(r) do repo - LibGit2.isdirty(repo) && error("$pkg is dirty, bailing") + LibGit2.isdirty(repo) && throw(PkgError("$pkg is dirty, bailing")) LibGit2.branch!(repo, branch, track=LibGit2.GitConst.REMOTE_ORIGIN) merge && LibGit2.merge!(repo, fast_forward=true) # merge changes if pull @@ -232,12 +232,12 @@ function checkout(pkg::AbstractString, branch::AbstractString, merge::Bool, pull end function free(pkg::AbstractString) - ispath(pkg,".git") || error("$pkg is not a git repo") - Read.isinstalled(pkg) || error("$pkg cannot be freed – not an installed package") + ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) + Read.isinstalled(pkg) || throw(PkgError("$pkg cannot be freed – not an installed package")) avail = Read.available(pkg) - isempty(avail) && error("$pkg cannot be freed – not a registered package") + isempty(avail) && throw(PkgError("$pkg cannot be freed – not a registered package")) with(GitRepo, pkg) do repo - LibGit2.isdirty(repo) && error("$pkg cannot be freed – repo is dirty") + LibGit2.isdirty(repo) && throw(PkgError("$pkg cannot be freed – repo is dirty")) info("Freeing $pkg") vers = sort!(collect(keys(avail)), rev=true) while true @@ -245,13 +245,13 @@ function free(pkg::AbstractString) sha1 = avail[ver].sha1 LibGit2.iscommit(sha1, repo) || continue return LibGit2.transact(repo) do r - LibGit2.isdirty(repo) && error("$pkg is dirty, bailing") + LibGit2.isdirty(repo) && throw(PkgError("$pkg is dirty, bailing")) LibGit2.checkout!(repo, sha1) resolve() end end isempty(Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail])) && continue - error("can't find any registered versions of $pkg to checkout") + throw(PkgError("can't find any registered versions of $pkg to checkout")) end end end @@ -281,7 +281,7 @@ end # end function pin(pkg::AbstractString, head::AbstractString) - ispath(pkg,".git") || error("$pkg is not a git repo") + ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) rslv = !isempty(head) # no need to resolve, branch will be from HEAD with(GitRepo, pkg) do repo if isempty(head) # get HEAD oid @@ -296,12 +296,12 @@ end pin(pkg::AbstractString) = pin(pkg, "") function pin(pkg::AbstractString, ver::VersionNumber) - ispath(pkg,".git") || error("$pkg is not a git repo") - Read.isinstalled(pkg) || error("$pkg cannot be pinned – not an installed package".tmp) + ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) + Read.isinstalled(pkg) || throw(PkgError("$pkg cannot be pinned – not an installed package".tmp)) avail = Read.available(pkg) - isempty(avail) && error("$pkg cannot be pinned – not a registered package".tmp) - haskey(avail,ver) || error("$pkg – $ver is not a registered version") - pin(pkg,avail[ver].sha1) + isempty(avail) && throw(PkgError("$pkg cannot be pinned – not a registered package".tmp)) + haskey(avail,ver) || throw(PkgError("$pkg – $ver is not a registered version")) + pin(pkg, avail[ver].sha1) end function update(branch::AbstractString) @@ -309,8 +309,8 @@ function update(branch::AbstractString) with(GitRepo, "METADATA") do repo with(LibGit2.head(repo)) do h if LibGit2.branch(h) != branch - LibGit2.isdirty(repo) && error("METADATA is dirty and not on $branch, bailing") - LibGit2.isattached(repo) || error("METADATA is detached not on $branch, bailing") + LibGit2.isdirty(repo) && throw(PkgError("METADATA is dirty and not on $branch, bailing")) + LibGit2.isattached(repo) || throw(PkgError("METADATA is detached not on $branch, bailing")) LibGit2.fetch(repo) LibGit2.checkout_head(repo) LibGit2.branch!(repo, branch, track="refs/remotes/origin/$branch") @@ -358,7 +358,7 @@ function pull_request(dir::AbstractString, commit::AbstractString="", url::Abstr Git.readchomp(`rev-parse --verify $commit`, dir=dir) isempty(url) && (url = Git.readchomp(`config remote.origin.url`, dir=dir)) m = match(Git.GITHUB_REGEX, url) - m === nothing && error("not a GitHub repo URL, can't make a pull request: $url") + m == nothing && throw(PkgError("not a GitHub repo URL, can't make a pull request: $url")) owner, repo = m.captures[2:3] user = GitHub.user() info("Forking $owner/$repo to $user") @@ -379,11 +379,11 @@ end function publish(branch::AbstractString) Git.branch(dir="METADATA") == branch || - error("METADATA must be on $branch to publish changes") + throw(PkgError("METADATA must be on $branch to publish changes")) Git.run(`fetch -q`, dir="METADATA") ahead_remote, ahead_local = map(x->parse(Int,x),split(Git.readchomp(`rev-list --count --left-right origin/$branch...$branch`, dir="METADATA"),'\t')) - ahead_remote > 0 && error("METADATA is behind origin/$branch – run `Pkg.update()` before publishing") - ahead_local == 0 && error("There are no METADATA changes to publish") + ahead_remote > 0 && throw(PkgError("METADATA is behind origin/$branch – run `Pkg.update()` before publishing")) + ahead_local == 0 && throw(PkgError("There are no METADATA changes to publish")) tags = Dict{ByteString,Vector{ASCIIString}}() Git.run(`update-index -q --really-refresh`, dir="METADATA") @@ -396,14 +396,14 @@ function publish(branch::AbstractString) sha1 = readchomp(joinpath("METADATA",path)) if Git.success(`cat-file -e origin/$branch:$path`, dir="METADATA") old = Git.readchomp(`cat-file blob origin/$branch:$path`, dir="METADATA") - old == sha1 || error("$pkg v$ver SHA1 changed in METADATA – refusing to publish") + old == sha1 || throw(PkgError("$pkg v$ver SHA1 changed in METADATA – refusing to publish")) end any(split(Git.readall(`tag --contains $sha1`, dir=pkg))) do tag ver == convert(VersionNumber,tag) || return false haskey(tags,pkg) || (tags[pkg] = ASCIIString[]) push!(tags[pkg], tag) return true - end || error("$pkg v$ver is incorrectly tagged – $sha1 expected") + end || throw(PkgError("$pkg v$ver is incorrectly tagged – $sha1 expected")) end isempty(tags) && info("No new package versions to publish") info("Validating METADATA") @@ -445,11 +445,11 @@ function resolve( for pkg in keys(reqs) if !haskey(deps,pkg) if "julia" in conflicts[pkg] - error("$pkg can't be installed because it has no versions that support ", VERSION, " of julia. " * - "You may need to update METADATA by running `Pkg.update()`") + throw(PkgError("$pkg can't be installed because it has no versions that support ", VERSION, " of julia. " * + "You may need to update METADATA by running `Pkg.update()`")) else - error("$pkg's requirements can't be satisfied because of the following fixed packages: ", - join(conflicts[pkg], ", ", " and ")) + throw(PkgError("$pkg's requirements can't be satisfied because of the following fixed packages: ", + join(conflicts[pkg], ", ", " and "))) end end end @@ -478,7 +478,7 @@ function resolve( for (pkg,ver,sha1) in missing msg *= " $pkg v$ver [$sha1[1:10]]\n" end - error(msg) + throw(PkgError(msg)) end # try applying changes, roll back everything if anything fails @@ -527,7 +527,7 @@ function write_tag_metadata(repo::GitRepo, pkg::AbstractString, ver::VersionNumb if !force && ispath(sha1file) current = readchomp(sha1file) current == commit || - error("$pkg v$ver is already registered as $current, bailing") + throw(PkgError("$pkg v$ver is already registered as $current, bailing")) end open(io->println(io,commit), sha1file, "w") LibGit2.add!(repo, sha1file) @@ -543,8 +543,8 @@ function write_tag_metadata(repo::GitRepo, pkg::AbstractString, ver::VersionNumb end function register(pkg::AbstractString, url::AbstractString) - ispath(pkg,".git") || error("$pkg is not a git repo") - isfile("METADATA",pkg,"url") && error("$pkg already registered") + ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) + isfile("METADATA",pkg,"url") && throw(PkgError("$pkg already registered")) LibGit2.transact(GitRepo("METADATA")) do repo # Get versions from package repo versions = with(GitRepo, pkg) do pkg_repo @@ -593,9 +593,9 @@ function register(pkg::AbstractString) end end catch err - error("$pkg: $err") + throw(PkgError("$pkg: $err")) end - isempty(url) || error("$pkg: no URL configured") + isempty(url) || throw(PkgError("$pkg: no URL configured")) register(pkg, LibGit2.normalize_url(url)) end @@ -608,12 +608,12 @@ end nextbump(v::VersionNumber) = isrewritable(v) ? v : nextpatch(v) function tag(pkg::AbstractString, ver::Union(Symbol,VersionNumber), force::Bool=false, commitish::AbstractString="HEAD") - ispath(pkg,".git") || error("$pkg is not a git repo") + ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) with(GitRepo,"METADATA") do repo - LibGit2.isdirty(repo, pkg) && error("METADATA/$pkg is dirty – commit or stash changes to tag") + LibGit2.isdirty(repo, pkg) && throw(PkgError("METADATA/$pkg is dirty – commit or stash changes to tag")) end with(GitRepo,pkg) do repo - LibGit2.isdirty(repo) && error("$pkg is dirty – commit or stash changes to tag") + LibGit2.isdirty(repo) && throw(PkgError("$pkg is dirty – commit or stash changes to tag")) commit = string(LibGit2.revparseid(repo, commitish)) registered = isfile("METADATA",pkg,"url") @@ -640,7 +640,7 @@ function tag(pkg::AbstractString, ver::Union(Symbol,VersionNumber), force::Bool= (ver == :patch) ? nextpatch(prv) : (ver == :minor) ? nextminor(prv) : (ver == :major) ? nextmajor(prv) : - error("invalid version selector: $ver") + throw(PkgError("invalid version selector: $ver")) end isrewritable(ver) && filter!(v->v!=ver,existing) check_new_version(existing,ver) @@ -675,7 +675,7 @@ function check_metadata(pkgs::Set{ByteString} = Set{ByteString}()) deps, conflicts = Query.dependencies(avail) for (dp,dv) in deps, (v,a) in dv, p in keys(a.requires) - haskey(deps, p) || error("package $dp v$v requires a non-registered package: $p") + haskey(deps, p) || throw(PkgError("package $dp v$v requires a non-registered package: $p")) end problematic = Resolve.sanity_check(deps, pkgs) @@ -684,7 +684,7 @@ function check_metadata(pkgs::Set{ByteString} = Set{ByteString}()) for (p, vn, rp) in problematic msg *= " $p v$vn – no valid versions exist for package $rp\n" end - error(msg) + throw(PkgError(msg)) end return end @@ -703,7 +703,7 @@ function build!(pkgs::Vector, errs::Dict, seen::Set=Set()) pkg == "julia" && continue pkg in seen && continue build!(Read.requires_list(pkg),errs,push!(seen,pkg)) - Read.isinstalled(pkg) || error("$pkg is not an installed package") + Read.isinstalled(pkg) || throw(PkgError("$pkg is not an installed package")) path = abspath(pkg,"deps","build.jl") isfile(path) || continue info("Building $pkg") @@ -802,7 +802,7 @@ function test(pkgs::Vector{AbstractString}; coverage::Bool=false) messages = AbstractString[] isempty(errs) || push!(messages, "$(join(errs,", "," and ")) had test errors") isempty(notests) || push!(messages, "$(join(notests,", "," and ")) did not provide a test/runtests.jl file") - error(join(messages, "and")) + throw(PkgError(join(messages, "and"))) end end diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index 9f18a6eb6d15b..a09967cb01641 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -50,7 +50,7 @@ function package( repo = GitRepo(pkg) if LibGit2.isdirty(repo) finalize(repo) - error("$pkg is dirty – commit or stash your changes") + throw(PkgError("$pkg is dirty – commit or stash your changes")) end repo end @@ -110,7 +110,7 @@ function init(pkg::AbstractString, url::AbstractString=""; config::Dict=Dict()) end LibGit2.commit(repo, "initial empty commit") catch err - error("Unable to initialize $pkg package: $err") + throw(PkgError("Unable to initialize $pkg package: $err")) end else repo = GitRepo(pkg) @@ -146,7 +146,7 @@ function license(pkg::AbstractString, file = genfile(pkg,"LICENSE.md",force) do io if !haskey(LICENSES,license) licenses = join(sort!(collect(keys(LICENSES)), by=lowercase), ", ") - error("$license is not a known license choice, choose one of: $licenses.") + throw(PkgError("$license is not a known license choice, choose one of: $licenses.")) end print(io, LICENSES[license](pkg, string(years), authors)) end diff --git a/base/pkg/github.jl b/base/pkg/github.jl index f827def0af915..7fe15174e7f8d 100644 --- a/base/pkg/github.jl +++ b/base/pkg/github.jl @@ -16,13 +16,13 @@ function user() LibGit2.get(cfg, "github.user", "") end if isempty(usr) - error(""" + throw(PkgError(""" no GitHub user name configured; please configure it with: git config --global github.user USERNAME where USERNAME is replaced with your GitHub user name. - """) + """)) end return usr end @@ -31,13 +31,13 @@ function json() isdefined(:JSON) || try eval(Main, :(import JSON)) catch err warn(err) - error("using the GitHub API requires having the JSON package installed ") + throw(PkgError("using the GitHub API requires having the JSON package installed ")) end Main.JSON end function curl(url::AbstractString, opts::Cmd=``) - success(`curl --version`) || error("using the GitHub API requires having `curl` installed") + success(`curl --version`) || throw(PkgError("using the GitHub API requires having `curl` installed")) out, proc = open(`curl -i -s -S $opts $url`,"r") head = readline(out) status = parse(Int,split(head,r"\s+";limit=3)[2]) @@ -50,7 +50,7 @@ function curl(url::AbstractString, opts::Cmd=``) end wait(proc); return status, header, readall(out) end - error("strangely formatted HTTP response") + throw(PkgError("strangely formatted HTTP response")) end curl(url::AbstractString, data::Void, opts::Cmd=``) = curl(url,opts) curl(url::AbstractString, data, opts::Cmd=``) = @@ -83,7 +83,7 @@ function token(user::AbstractString=user()) if tfa info("Retrieving existing GitHub token. (You may have to re-enter your password twice more.)") status, header, content = curl("https://api.github.com/authorizations",AUTH_DATA,`-u $user`) - status != 401 && error("$status: $(json().parse(content)["message"])") + status != 401 && throw(PkgError("$status: $(json().parse(content)["message"])")) print(STDERR, "New authentication code: ") code = readline(STDIN) |> chomp status, header, content = curl("https://api.github.com/authorizations",`-H "X-GitHub-OTP: $code" -u $user`) @@ -91,7 +91,7 @@ function token(user::AbstractString=user()) info("Retrieving existing GitHub token. (You may have to re-enter your password.)") status, header, content = curl("https://api.github.com/authorizations", `-u $user`) end - (status >= 400) && error("$status: $(json().parse(content)["message"])") + (status >= 400) && throw(PkgError("$status: $(json().parse(content)["message"])")) for entry in json().parse(content) if entry["note"] == AUTH_NOTE tok = entry["token"] @@ -99,10 +99,10 @@ function token(user::AbstractString=user()) end end else - error("GitHub returned validation error (422): $error_code: $(json().parse(content)["message"])") + throw(PkgError("GitHub returned validation error (422): $error_code: $(json().parse(content)["message"])")) end else - (status != 401 && status != 403) || error("$status: $(json().parse(content)["message"])") + (status != 401 && status != 403) || throw(PkgError("$status: $(json().parse(content)["message"])")) tok = json().parse(content)["token"] end @@ -131,11 +131,11 @@ end function pushable(owner::AbstractString, repo::AbstractString, user::AbstractString=user()) status, response = HEAD("repos/$owner/$repo") - status == 404 && error("repo $owner/$repo does not exist") + status == 404 && throw(PkgError("repo $owner/$repo does not exist")) status, response = GET("repos/$owner/$repo/collaborators/$user") status == 204 && return true status == 404 && return false - error("unexpected API status code: $status – $(response["message"])") + throw(PkgError("unexpected API status code: $status – $(response["message"])")) end function fork(owner::AbstractString, repo::AbstractString) @@ -144,7 +144,7 @@ function fork(owner::AbstractString, repo::AbstractString) delete_token() status, response = POST("repos/$owner/$repo/forks") end - status == 202 || error("forking $owner/$repo failed: $(response["message"])") + status == 202 || throw(PkgError("forking $owner/$repo failed: $(response["message"])")) return response end diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 79f290ab4e6a4..3b33225bbb871 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -403,7 +403,7 @@ end function __init__() err = ccall((:git_libgit2_init, :libgit2), Cint, ()) - err > 0 || error("error initializing LibGit2 module") + err > 0 || throw(PkgError("error initializing LibGit2 module")) atexit() do ccall((:git_libgit2_shutdown, :libgit2), Cint, ()) end diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index e24d6fa5c71b5..f768786f96bc8 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -353,7 +353,7 @@ function getobjecttype{T<:GitObject}(::Type{T}) elseif T == GitAnyObject GitConst.OBJ_ANY else - error("Type $T is not supported") + throw(GitError(Error.Object, Error.ENOTFOUND, "Type $T is not supported")) end end @@ -367,6 +367,6 @@ function getobjecttype(obj_type::Cint) elseif obj_type == GitConst.OBJ_ANY GitAnyObject else - error("Type $T is not supported") + throw(GitError(Error.Object, Error.ENOTFOUND, "Object type $obj_type is not supported")) end end \ No newline at end of file diff --git a/base/pkg/query.jl b/base/pkg/query.jl index 2a7245d524b84..a8e7618172e73 100644 --- a/base/pkg/query.jl +++ b/base/pkg/query.jl @@ -8,7 +8,7 @@ using ..Types function requirements(reqs::Dict, fix::Dict, avail::Dict) for (p1,f1) in fix for p2 in keys(f1.requires) - haskey(avail, p2) || haskey(fix, p2) || error("unknown package $p2 required by $p1") + haskey(avail, p2) || haskey(fix, p2) || throw(PkgError("unknown package $p2 required by $p1")) end satisfies(p1, f1.version, reqs) || warn("$p1 is fixed at $(f1.version) conflicting with top-level requirement: $(reqs[p1])") @@ -108,7 +108,7 @@ function check_requirements(reqs::Requires, deps::Dict{ByteString,Dict{VersionNu else err_msg *= " available versions are $(join(available_list, ", ", " and "))" end - error(err_msg) + throw(PkgError(err_msg)) end end end diff --git a/base/pkg/read.jl b/base/pkg/read.jl index fcc13a658f4d7..e259e73de7e42 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -56,7 +56,7 @@ isinstalled(pkg::AbstractString) = pkg != "METADATA" && pkg != "REQUIRE" && pkg[1] != '.' && isdir(pkg) function isfixed(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::Dict=available(pkg)) - isinstalled(pkg) || error("$pkg is not an installed package.") + isinstalled(pkg) || throw(PkgError("$pkg is not an installed package.")) isfile("METADATA", pkg, "url") || return true ispath(pkg, ".git") || return true diff --git a/base/pkg/reqs.jl b/base/pkg/reqs.jl index aa6331e1ef5f9..8caf72f9afb7d 100644 --- a/base/pkg/reqs.jl +++ b/base/pkg/reqs.jl @@ -24,12 +24,12 @@ immutable Requirement <: Line while !isempty(fields) && fields[1][1] == '@' push!(system,shift!(fields)[2:end]) end - isempty(fields) && error("invalid requires entry: $content") + isempty(fields) && throw(PkgError("invalid requires entry: $content")) package = shift!(fields) all(field->ismatch(Base.VERSION_REGEX, field), fields) || - error("invalid requires entry for $package: $content") + throw(PkgError("invalid requires entry for $package: $content")) versions = VersionNumber[fields...] - issorted(versions) || error("invalid requires entry for $package: $content") + issorted(versions) || throw(PkgError("invalid requires entry for $package: $content")) new(content, package, VersionSet(versions), system) end function Requirement(package::AbstractString, versions::VersionSet, system::Vector{AbstractString}=AbstractString[]) diff --git a/base/pkg/resolve.jl b/base/pkg/resolve.jl index 83a5b9822a562..ce7655c27437f 100644 --- a/base/pkg/resolve.jl +++ b/base/pkg/resolve.jl @@ -34,7 +34,7 @@ function resolve(reqs::Requires, deps::Dict{ByteString,Dict{VersionNumber,Availa msg *= "\n (you may try increasing the value of the" * "\n JULIA_PKGRESOLVE_ACCURACY environment variable)" end - error(msg) + throw(PkgError(msg)) end rethrow(err) end diff --git a/base/pkg/resolve/maxsum.jl b/base/pkg/resolve/maxsum.jl index 958f26d9d9be5..c98dc6ee49e8e 100644 --- a/base/pkg/resolve/maxsum.jl +++ b/base/pkg/resolve/maxsum.jl @@ -25,7 +25,7 @@ type MaxSumParams function MaxSumParams() accuracy = parse(Int,get(ENV, "JULIA_PKGRESOLVE_ACCURACY", "1")) if accuracy <= 0 - error("JULIA_PKGRESOLVE_ACCURACY must be >= 1") + error("JULIA_PKGRESOLVE_ACCURACY must be > 0") end nondec_iterations = accuracy * 20 dec_interval = accuracy * 10 diff --git a/base/pkg/types.jl b/base/pkg/types.jl index 8bfd7854cfa47..4995c90b90d14 100644 --- a/base/pkg/types.jl +++ b/base/pkg/types.jl @@ -91,4 +91,8 @@ show(io::IO, f::Fixed) = isempty(f.requires) ? # Free could include the same information too, it just isn't # required by anything that processes these things. +type PkgError <: Exception + msg::AbstractString +end + end # module diff --git a/base/pkg/write.jl b/base/pkg/write.jl index 08e6953fc51e0..ac15f7287058c 100644 --- a/base/pkg/write.jl +++ b/base/pkg/write.jl @@ -7,7 +7,7 @@ importall ..LibGit2 function prefetch(pkg::AbstractString, sha1::AbstractString) isempty(Cache.prefetch(pkg, Read.url(pkg), sha1)) && return - error("$pkg: couldn't find commit $(sha1[1:10])") + throw(PkgError("$pkg: couldn't find commit $(sha1[1:10])")) end function fetch(repo::GitRepo, pkg::AbstractString, sha1::AbstractString) @@ -20,9 +20,9 @@ function fetch(repo::GitRepo, pkg::AbstractString, sha1::AbstractString) end ? "fetch" : "prefetch" url = Read.issue_url(pkg) if isempty(url) - error("$pkg: $f failed to get commit $(sha1[1:10]), please file a bug report with the package author.") + throw(PkgError("$pkg: $f failed to get commit $(sha1[1:10]), please file a bug report with the package author.")) else - error("$pkg: $f failed to get commit $(sha1[1:10]), please file an issue at $url") + throw(PkgError("$pkg: $f failed to get commit $(sha1[1:10]), please file an issue at $url")) end end From 3335bf796b5350a4b5703871db8cea244a225b34 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Tue, 19 May 2015 16:32:24 -0400 Subject: [PATCH 0317/1938] added licence line --- base/pkg/libgit2.jl | 2 ++ base/pkg/libgit2/blob.jl | 2 ++ base/pkg/libgit2/commit.jl | 2 ++ base/pkg/libgit2/config.jl | 2 ++ base/pkg/libgit2/const.jl | 2 ++ base/pkg/libgit2/error.jl | 2 ++ base/pkg/libgit2/index.jl | 2 ++ base/pkg/libgit2/merge.jl | 2 ++ base/pkg/libgit2/oid.jl | 2 ++ base/pkg/libgit2/reference.jl | 2 ++ base/pkg/libgit2/remote.jl | 2 ++ base/pkg/libgit2/repository.jl | 2 ++ base/pkg/libgit2/signature.jl | 2 ++ base/pkg/libgit2/strarray.jl | 2 ++ base/pkg/libgit2/tag.jl | 2 ++ base/pkg/libgit2/types.jl | 2 ++ base/pkg/libgit2/walker.jl | 2 ++ 17 files changed, 34 insertions(+) diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 3b33225bbb871..315ac5fa90fe2 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + module LibGit2 export with, with_warn diff --git a/base/pkg/libgit2/blob.jl b/base/pkg/libgit2/blob.jl index 59440c560a2c7..502a15238dfbc 100644 --- a/base/pkg/libgit2/blob.jl +++ b/base/pkg/libgit2/blob.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + function content(blob::GitBlob) return ccall((:git_blob_rawcontent, :libgit2), Ptr{Void}, (Ptr{Void},), blob.ptr) end \ No newline at end of file diff --git a/base/pkg/libgit2/commit.jl b/base/pkg/libgit2/commit.jl index b2e80bbede555..e61699bad1c93 100644 --- a/base/pkg/libgit2/commit.jl +++ b/base/pkg/libgit2/commit.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + function message(c::GitCommit, raw::Bool=false) local msg_ptr::Ptr{UInt8} msg_ptr = raw? ccall((:git_commit_message_raw, :libgit2), Ptr{UInt8}, (Ptr{Void},), c.ptr) : diff --git a/base/pkg/libgit2/config.jl b/base/pkg/libgit2/config.jl index a724e679e86ce..336b9b623c7dd 100644 --- a/base/pkg/libgit2/config.jl +++ b/base/pkg/libgit2/config.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + function GitConfig(path::AbstractString) cfg_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_config_open_ondisk, :libgit2), Cint, diff --git a/base/pkg/libgit2/const.jl b/base/pkg/libgit2/const.jl index 37cda27740a8c..f76f560f5acc7 100644 --- a/base/pkg/libgit2/const.jl +++ b/base/pkg/libgit2/const.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + module GitConst const HEAD_FILE = "HEAD" diff --git a/base/pkg/libgit2/error.jl b/base/pkg/libgit2/error.jl index b77c9a1524271..9b177e49b27e7 100644 --- a/base/pkg/libgit2/error.jl +++ b/base/pkg/libgit2/error.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + module Error @enum(Code, GIT_OK = Cint(0), # no error diff --git a/base/pkg/libgit2/index.jl b/base/pkg/libgit2/index.jl index 8aa25e82f6618..168c9ff79c2f7 100644 --- a/base/pkg/libgit2/index.jl +++ b/base/pkg/libgit2/index.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + function GitIndex(repo::GitRepo) idx_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_repository_index, :libgit2), Cint, diff --git a/base/pkg/libgit2/merge.jl b/base/pkg/libgit2/merge.jl index d07d6da0be3ff..883d13cee1ecb 100644 --- a/base/pkg/libgit2/merge.jl +++ b/base/pkg/libgit2/merge.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + function GitAnnotated(repo::GitRepo, commit_id::Oid) ann_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_annotated_commit_lookup, :libgit2), Cint, diff --git a/base/pkg/libgit2/oid.jl b/base/pkg/libgit2/oid.jl index 3c4b9a905e655..1dc005845370f 100644 --- a/base/pkg/libgit2/oid.jl +++ b/base/pkg/libgit2/oid.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + Oid(id::Oid) = id Oid(ptr::Ptr{Oid}) = unsafe_load(ptr)::Oid diff --git a/base/pkg/libgit2/reference.jl b/base/pkg/libgit2/reference.jl index 77895a510f15e..44b33f4bd6c99 100644 --- a/base/pkg/libgit2/reference.jl +++ b/base/pkg/libgit2/reference.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + function GitReference(repo::GitRepo, ref_name::AbstractString) ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_reference_lookup, :libgit2), Cint, diff --git a/base/pkg/libgit2/remote.jl b/base/pkg/libgit2/remote.jl index a5bd95625d6d1..6a82056f7aabd 100644 --- a/base/pkg/libgit2/remote.jl +++ b/base/pkg/libgit2/remote.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + function GitRemote(repo::GitRepo, rmt_name::AbstractString, rmt_url::AbstractString) rmt_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_remote_create, :libgit2), Cint, diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index 431ae30349745..bbea90db1d3c9 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + function GitRepo(path::AbstractString) repo_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_repository_open, :libgit2), Cint, diff --git a/base/pkg/libgit2/signature.jl b/base/pkg/libgit2/signature.jl index 2f79117f95f28..82484d91eccc2 100644 --- a/base/pkg/libgit2/signature.jl +++ b/base/pkg/libgit2/signature.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + function Signature(ptr::Ptr{SignatureStruct}) sig = unsafe_load(ptr)::SignatureStruct name = bytestring(sig.name) diff --git a/base/pkg/libgit2/strarray.jl b/base/pkg/libgit2/strarray.jl index b43d4320effd9..b4a012dca216b 100644 --- a/base/pkg/libgit2/strarray.jl +++ b/base/pkg/libgit2/strarray.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + function StrArrayStruct{T<:AbstractString}(strs::T...) count = length(strs) strings = convert(Ptr{Ptr{Uint8}}, Libc.malloc(sizeof(Ptr{Uint8}) * count)) diff --git a/base/pkg/libgit2/tag.jl b/base/pkg/libgit2/tag.jl index 00bfd9899088c..ec5249e4fc14c 100644 --- a/base/pkg/libgit2/tag.jl +++ b/base/pkg/libgit2/tag.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + function tag_list(repo::GitRepo) with(StrArrayStruct()) do sa sa_ref = Ref(sa) diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index f768786f96bc8..892dac4aed62d 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + const OID_RAWSZ = 20 const OID_HEXSZ = OID_RAWSZ * 2 const OID_MINPREFIXLEN = 4 diff --git a/base/pkg/libgit2/walker.jl b/base/pkg/libgit2/walker.jl index c0715fe1b2e70..006fb1b98fef5 100644 --- a/base/pkg/libgit2/walker.jl +++ b/base/pkg/libgit2/walker.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + function GitRevWalker(r::GitRepo) w_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_revwalk_new, :libgit2), Cint, From b5efeef967b4c1339592f61ae4dfa0e1c60760c4 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Tue, 19 May 2015 19:22:57 -0400 Subject: [PATCH 0318/1938] fixed GitHub saved token retrieval & PkgError visibility --- base/pkg.jl | 4 ++++ base/pkg/cache.jl | 2 +- base/pkg/dir.jl | 2 +- base/pkg/github.jl | 7 +++++-- base/pkg/query.jl | 2 +- base/pkg/read.jl | 2 +- base/pkg/reqs.jl | 2 +- base/pkg/types.jl | 4 ---- 8 files changed, 14 insertions(+), 11 deletions(-) diff --git a/base/pkg.jl b/base/pkg.jl index 443741f98266e..ef5298e9a1524 100644 --- a/base/pkg.jl +++ b/base/pkg.jl @@ -10,6 +10,10 @@ export dir, init, rm, add, available, installed, status, clone, checkout, const DEFAULT_META = "https://github.com/JuliaLang/METADATA.jl" const META_BRANCH = "metadata-v2" +type PkgError <: Exception + msg::AbstractString +end + for file in split("git libgit2 dir github types reqs cache read query resolve write generate entry") include("pkg/$file.jl") end diff --git a/base/pkg/cache.jl b/base/pkg/cache.jl index 296868febf558..e5fcfafedc2ea 100644 --- a/base/pkg/cache.jl +++ b/base/pkg/cache.jl @@ -2,7 +2,7 @@ module Cache -import ..LibGit2, ..Dir +import ..LibGit2, ..Dir, Pkg.PkgError using ..Types path(pkg::AbstractString) = abspath(".cache", pkg) diff --git a/base/pkg/dir.jl b/base/pkg/dir.jl index b511de1698f45..ca08a0291a686 100644 --- a/base/pkg/dir.jl +++ b/base/pkg/dir.jl @@ -2,7 +2,7 @@ module Dir -import ..Pkg: DEFAULT_META, META_BRANCH +import ..Pkg: DEFAULT_META, META_BRANCH, PkgError import ..LibGit2, ..LibGit2.with const DIR_NAME = ".julia" diff --git a/base/pkg/github.jl b/base/pkg/github.jl index 7fe15174e7f8d..dbb2cb8be0dc8 100644 --- a/base/pkg/github.jl +++ b/base/pkg/github.jl @@ -2,7 +2,7 @@ module GitHub -import Main, ..LibGit2, ..Dir +import Main, ..LibGit2, ..Dir, ...Pkg.PkgError const AUTH_NOTE = "Julia Package Manager" const AUTH_DATA = Dict{Any,Any}( @@ -64,7 +64,10 @@ end function token(user::AbstractString=user()) tokfile = Dir.path(".github","token") - isfile(tokfile) && return strip(readchomp(tokfile)) + if isfile(tokfile) + tok = strip(readchomp(tokfile)) + !isempty(tok) && return tok + end status, header, content = curl("https://api.github.com/authorizations",AUTH_DATA,`-u $user`) tfa = false diff --git a/base/pkg/query.jl b/base/pkg/query.jl index a8e7618172e73..1cec1817c9696 100644 --- a/base/pkg/query.jl +++ b/base/pkg/query.jl @@ -2,7 +2,7 @@ module Query -import ...Pkg +import ...Pkg.PkgError using ..Types function requirements(reqs::Dict, fix::Dict, avail::Dict) diff --git a/base/pkg/read.jl b/base/pkg/read.jl index e259e73de7e42..f684693045fd7 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -2,7 +2,7 @@ module Read -import ..LibGit2, ..Cache, ..Reqs +import ..LibGit2, ..Cache, ..Reqs, ...Pkg.PkgError using ..Types readstrip(path...) = strip(readall(joinpath(path...))) diff --git a/base/pkg/reqs.jl b/base/pkg/reqs.jl index 8caf72f9afb7d..b3821b68af670 100644 --- a/base/pkg/reqs.jl +++ b/base/pkg/reqs.jl @@ -3,7 +3,7 @@ module Reqs import Base: ==, hash - +import Pkg.PkgError using ..Types # representing lines of REQUIRE files diff --git a/base/pkg/types.jl b/base/pkg/types.jl index 4995c90b90d14..8bfd7854cfa47 100644 --- a/base/pkg/types.jl +++ b/base/pkg/types.jl @@ -91,8 +91,4 @@ show(io::IO, f::Fixed) = isempty(f.requires) ? # Free could include the same information too, it just isn't # required by anything that processes these things. -type PkgError <: Exception - msg::AbstractString -end - end # module From efde7ed81ecad4290146de3d6ccd111a919d672b Mon Sep 17 00:00:00 2001 From: Art Wild Date: Tue, 19 May 2015 19:29:22 -0400 Subject: [PATCH 0319/1938] added `push`, transitioned `pull_request` --- base/pkg/entry.jl | 40 +++++++++++++++++++++------------- base/pkg/libgit2.jl | 11 ++++++++++ base/pkg/libgit2/remote.jl | 13 +++++++++++ base/pkg/libgit2/repository.jl | 5 +---- base/pkg/libgit2/types.jl | 6 +++++ 5 files changed, 56 insertions(+), 19 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index a6a49d6e01d0c..c500b5514ddc7 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -354,21 +354,31 @@ function update(branch::AbstractString) end function pull_request(dir::AbstractString, commit::AbstractString="", url::AbstractString="") - commit = isempty(commit) ? Git.head(dir=dir) : - Git.readchomp(`rev-parse --verify $commit`, dir=dir) - isempty(url) && (url = Git.readchomp(`config remote.origin.url`, dir=dir)) - m = match(Git.GITHUB_REGEX, url) - m == nothing && throw(PkgError("not a GitHub repo URL, can't make a pull request: $url")) - owner, repo = m.captures[2:3] - user = GitHub.user() - info("Forking $owner/$repo to $user") - response = GitHub.fork(owner,repo) - fork = response["ssh_url"] - branch = "pull-request/$(commit[1:8])" - info("Pushing changes as branch $branch") - Git.run(`push -q $fork $commit:refs/heads/$branch`, dir=dir) - pr_url = "$(response["html_url"])/compare/$branch" - @osx? run(`open $pr_url`) : info("To create a pull-request, open:\n\n $pr_url\n") + with(GitRepo, dir) do repo + if isempty(commit) + commit = string(LibGit2.head_oid(repo)) + else + !LibGit2.iscommit(commit, repo) && throw(PkgError("Cannot find pull commit: $commit")) + end + if isempty(url) + with(GitConfig, repo) do cfg + url = LibGit2.get(cfg, "remote.origin.url", "") + end + end + + m = match(LibGit2.GITHUB_REGEX, url) + m == nothing && throw(PkgError("not a GitHub repo URL, can't make a pull request: $url")) + owner, repo = m.captures[2:3] + user = GitHub.user() + info("Forking $owner/$repo to $user") + response = GitHub.fork(owner,repo) + fork = response["ssh_url"] + branch = "pull-request/$(commit[1:8])" + info("Pushing changes as branch $branch") + LibGit2.push(repo, fork, "$commit:refs/heads/$branch") + pr_url = "$(response["html_url"])/compare/$branch" + @osx? run(`open $pr_url`) : info("To create a pull-request, open:\n\n $pr_url\n") + end end function submit(pkg::AbstractString, commit::AbstractString="") diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 315ac5fa90fe2..761edee4fc0e5 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -2,6 +2,8 @@ module LibGit2 +import ...Pkg.PkgError + export with, with_warn export GitRepo, GitConfig, GitIndex @@ -345,6 +347,15 @@ function cat{T<:GitObject}(repo::GitRepo, ::Type{T}, object::AbstractString) end end +""" git push []""" +function push(repo::GitRepo, url::AbstractString, refspecs::AbstractString="") + with(GitRemoteAnon(repo, url, refspecs)) do rmt + with(default_signature(repo)) do sig + push(rmt, sig) + end + end +end + """ Returns all commit authors """ function authors(repo::GitRepo) athrs = map( diff --git a/base/pkg/libgit2/remote.jl b/base/pkg/libgit2/remote.jl index 6a82056f7aabd..194b9a93743b6 100644 --- a/base/pkg/libgit2/remote.jl +++ b/base/pkg/libgit2/remote.jl @@ -33,3 +33,16 @@ function url(rmt::GitRemote) url_ptr == C_NULL && return "" return bytestring(url_ptr) end + +function push(rmt::GitRemote, sig::GitSignature, refspecs::AbstractString=""; + msg::AbstractString="") + push_opts = PushOptionsStruct() + !isempty(refspecs) && (sa = StrArrayStruct(pathspecs...)) + try + @check ccall((:git_remote_push, :libgit2), Cint, + (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{PushOptionsStruct}, Ptr{Void}, Ptr{UInt8}), + rmt.ptr, !isempty(refspecs) ? Ref(sa) : C_NULL, Ref(push_opts), sig.ptr, isempty(msg) ? C_NULL : msg) + finally + !isempty(refspecs) && finalize(sa) + end +end diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index bbea90db1d3c9..2a4383a81fbd3 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -146,15 +146,12 @@ function fetch(repo::GitRepo, rmt::GitRemote; msg::AbstractString="") end function reset!(repo::GitRepo, obj::Nullable{GitAnyObject}, pathspecs::AbstractString...) - sa = StrArrayStruct(pathspecs...) - try + with(StrArrayStruct(pathspecs...)) do sa @check ccall((:git_reset_default, :libgit2), Cint, (Ptr{Void}, Ptr{Void}, Ptr{StrArrayStruct}), repo.ptr, isnull(obj) ? C_NULL: Base.get(obj).ptr, Ref(sa)) - finally - finalize(sa) end end diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index 892dac4aed62d..a07ea16c75f94 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -227,6 +227,12 @@ MergeOptionsStruct(; flags::Cint = Cint(0), file_favor ) +immutable PushOptionsStruct + version::Cuint + parallelism::Cint +end +PushOptionsStruct() = PushOptionsStruct(one(Cuint),one(Cuint)) + immutable IndexTime seconds::Int64 nanoseconds::Cuint From f1b5694d1ab58d4e6c872c2a02a42ad55d886092 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Tue, 19 May 2015 21:03:40 -0400 Subject: [PATCH 0320/1938] fixed import error --- base/pkg/cache.jl | 2 +- base/pkg/entry.jl | 1 + base/pkg/generate.jl | 2 +- base/pkg/reqs.jl | 4 ++-- base/pkg/resolve.jl | 2 +- base/pkg/write.jl | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/base/pkg/cache.jl b/base/pkg/cache.jl index e5fcfafedc2ea..0f1e216ee1267 100644 --- a/base/pkg/cache.jl +++ b/base/pkg/cache.jl @@ -2,7 +2,7 @@ module Cache -import ..LibGit2, ..Dir, Pkg.PkgError +import ..LibGit2, ..Dir, ...Pkg.PkgError using ..Types path(pkg::AbstractString) = abspath(".cache", pkg) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index c500b5514ddc7..123e93619e1e0 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -5,6 +5,7 @@ module Entry import Base: thispatch, nextpatch, nextminor, nextmajor, check_new_version import ..LibGit2, ..Git, ..Reqs, ..Read, ..Query, ..Resolve, ..Cache, ..Write, ..GitHub, ..Dir importall ..LibGit2 +import ...Pkg.PkgError using ..Types macro recover(ex) diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index a09967cb01641..4529808d56bde 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -2,7 +2,7 @@ module Generate -import ..LibGit2, ..Read +import ..LibGit2, ..Read, ...Pkg.PkgError importall ..LibGit2 copyright_year() = string(Dates.year(Dates.today())) diff --git a/base/pkg/reqs.jl b/base/pkg/reqs.jl index b3821b68af670..6b86e26a77bcd 100644 --- a/base/pkg/reqs.jl +++ b/base/pkg/reqs.jl @@ -2,8 +2,8 @@ module Reqs -import Base: ==, hash -import Pkg.PkgError +import Base: == +import ...Pkg.PkgError using ..Types # representing lines of REQUIRE files diff --git a/base/pkg/resolve.jl b/base/pkg/resolve.jl index ce7655c27437f..f3a7a59f8f29c 100644 --- a/base/pkg/resolve.jl +++ b/base/pkg/resolve.jl @@ -6,7 +6,7 @@ include("resolve/versionweight.jl") include("resolve/interface.jl") include("resolve/maxsum.jl") -using ..Types, ..Query, .PkgToMaxSumInterface, .MaxSum +using ..Types, ..Query, .PkgToMaxSumInterface, .MaxSum, ...Pkg.PkgError export resolve, sanity_check diff --git a/base/pkg/write.jl b/base/pkg/write.jl index ac15f7287058c..fde650cffcaeb 100644 --- a/base/pkg/write.jl +++ b/base/pkg/write.jl @@ -2,7 +2,7 @@ module Write -import ..LibGit2, ..Cache, ..Read +import ..LibGit2, ..Cache, ..Read, ...Pkg.PkgError importall ..LibGit2 function prefetch(pkg::AbstractString, sha1::AbstractString) From c1298e8b1ec7c6b08cc3f270944b561e16e0cf04 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Wed, 20 May 2015 00:09:14 -0400 Subject: [PATCH 0321/1938] added `revcount` for counting differences --- base/pkg/entry.jl | 14 +++++++------ base/pkg/libgit2.jl | 13 ++++++++++++ base/pkg/libgit2/walker.jl | 43 ++++++++++++++++++++++++++++++++++---- 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 123e93619e1e0..4a39cfd48d3e1 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -389,13 +389,15 @@ function submit(pkg::AbstractString, commit::AbstractString="") end function publish(branch::AbstractString) - Git.branch(dir="METADATA") == branch || - throw(PkgError("METADATA must be on $branch to publish changes")) - Git.run(`fetch -q`, dir="METADATA") - ahead_remote, ahead_local = map(x->parse(Int,x),split(Git.readchomp(`rev-list --count --left-right origin/$branch...$branch`, dir="METADATA"),'\t')) - ahead_remote > 0 && throw(PkgError("METADATA is behind origin/$branch – run `Pkg.update()` before publishing")) - ahead_local == 0 && throw(PkgError("There are no METADATA changes to publish")) + with(GitRepo, "METADATA") do repo + LibGit2.branch(repo) == branch || + throw(PkgError("METADATA must be on $branch to publish changes")) + LibGit2.fetch(repo) + ahead_remote, ahead_local = LibGit2.revcount(repo, "origin/$branch", branch) + ahead_remote > 0 && throw(PkgError("METADATA is behind origin/$branch – run `Pkg.update()` before publishing")) + ahead_local == 0 && throw(PkgError("There are no METADATA changes to publish")) + end tags = Dict{ByteString,Vector{ASCIIString}}() Git.run(`update-index -q --really-refresh`, dir="METADATA") cmd = `diff --name-only --diff-filter=AMR origin/$branch HEAD --` diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 761edee4fc0e5..dd60708faab18 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -356,6 +356,19 @@ function push(repo::GitRepo, url::AbstractString, refspecs::AbstractString="") end end +""" git rev-list --count """ +function revcount(repo::GitRepo, fst::AbstractString, snd::AbstractString) + fst_id = revparseid(repo, fst) + snd_id = revparseid(repo, snd) + base_id = merge_base(string(fst_id), string(snd_id), repo) + fc = count((i,r)->i!=base_id, repo, oid=fst_id, + by=Pkg.LibGit2.GitConst.SORT_TOPOLOGICAL) + sc = count((i,r)->i!=base_id, repo, oid=snd_id, + by=Pkg.LibGit2.GitConst.SORT_TOPOLOGICAL) + return (fc-1, sc-1) +end + + """ Returns all commit authors """ function authors(repo::GitRepo) athrs = map( diff --git a/base/pkg/libgit2/walker.jl b/base/pkg/libgit2/walker.jl index 006fb1b98fef5..68afae59ad6fe 100644 --- a/base/pkg/libgit2/walker.jl +++ b/base/pkg/libgit2/walker.jl @@ -35,21 +35,32 @@ function Base.push!(w::GitRevWalker, cid::Oid) return w end +function Base.push!(w::GitRevWalker, range::AbstractString) + @check ccall((:git_revwalk_push_range, :libgit2), Cint, (Ptr{Void}, Ptr{UInt8}), w.ptr, range) + return w +end + function Base.sort!(w::GitRevWalker; by::Cint = GitConst.SORT_NONE, rev::Bool=false) rev && (by |= GitConst.SORT_REVERSE) ccall((:git_revwalk_sorting, :libgit2), Void, (Ptr{Void}, Cint), w.ptr, by) return w end -function Base.map(f::Function, repo::GitRepo; oid::Oid=Oid(), by::Cint = GitConst.SORT_NONE, rev::Bool=false) +function Base.map(f::Function, repo::GitRepo; + oid::Oid=Oid(), + range::AbstractString="", + by::Cint = GitConst.SORT_NONE, + rev::Bool=false) walker = GitRevWalker(repo) res = nothing try sort!(walker, by=by, rev=rev) - if iszero(oid) - push_head!(walker) - else + if !iszero(oid) push!(walker, oid) + elseif !isempty(range) + push!(walker, range) + else + push_head!(walker) end s = start(walker) @@ -65,4 +76,28 @@ function Base.map(f::Function, repo::GitRepo; oid::Oid=Oid(), by::Cint = GitCons finalize(walker) end return res +end + +function Base.count(f::Function, repo::GitRepo; + oid::Oid=Oid(), + by::Cint = GitConst.SORT_NONE, + rev::Bool=false) + c = 0 + with(GitRevWalker(repo)) do walker + sort!(walker, by=by, rev=rev) + if !iszero(oid) + push!(walker, oid) + else + push_head!(walker) + end + s = start(walker) + + val = true + while !done(walker, s) && val + val = f(s[1], repo) + _, s = next(walker, s) + c += 1 + end + end + return c end \ No newline at end of file From 6943209b2e4e408c0247a95fdc66e5c032a92354 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Wed, 20 May 2015 00:29:01 -0400 Subject: [PATCH 0322/1938] fixed exceptions in tests --- test/resolve.jl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/resolve.jl b/test/resolve.jl index e17209d8836a4..94c99b691fb7c 100644 --- a/test/resolve.jl +++ b/test/resolve.jl @@ -4,6 +4,7 @@ using Base.Pkg.Types using Base.Pkg.Query using Base.Pkg.Resolve using Base.Pkg.Resolve.VersionWeights +using Base.Pkg.PkgError # Check that VersionWeight keeps the same ordering as VersionNumber @@ -209,7 +210,7 @@ reqs_data = Any[ ["A", v"1", v"2"], ["C", v"2"] ] -@test_throws ErrorException resolve_tst(deps_data, reqs_data) +@test_throws PkgError resolve_tst(deps_data, reqs_data) ## DEPENDENCY SCHEME 4: TWO PACKAGES, DAG, WITH TRIVIAL INCONSISTENCY @@ -256,7 +257,7 @@ want = resolve_tst(deps_data, reqs_data) reqs_data = Any[ ["A", v"2"] ] -@test_throws ErrorException resolve_tst(deps_data, reqs_data) +@test_throws PkgError resolve_tst(deps_data, reqs_data) ## DEPENDENCY SCHEME 6: TWO PACKAGES, CYCLIC, TOTALLY INCONSISTENT @@ -274,13 +275,13 @@ deps_data = Any[ reqs_data = Any[ ["A"] ] -@test_throws ErrorException resolve_tst(deps_data, reqs_data) +@test_throws PkgError resolve_tst(deps_data, reqs_data) # require B (impossible) reqs_data = Any[ ["B"] ] -@test_throws ErrorException resolve_tst(deps_data, reqs_data) +@test_throws PkgError resolve_tst(deps_data, reqs_data) ## DEPENDENCY SCHEME 7: THREE PACKAGES, CYCLIC, WITH INCONSISTENCY @@ -314,7 +315,7 @@ want = resolve_tst(deps_data, reqs_data) reqs_data = Any[ ["C", v"1", v"2"] ] -@test_throws ErrorException resolve_tst(deps_data, reqs_data) +@test_throws PkgError resolve_tst(deps_data, reqs_data) ## DEPENDENCY SCHEME 8: THREE PACKAGES, CYCLIC, TOTALLY INCONSISTENT @@ -335,19 +336,19 @@ deps_data = Any[ reqs_data = Any[ ["A"] ] -@test_throws ErrorException resolve_tst(deps_data, reqs_data) +@test_throws PkgError resolve_tst(deps_data, reqs_data) # require B (impossible) reqs_data = Any[ ["B"] ] -@test_throws ErrorException resolve_tst(deps_data, reqs_data) +@test_throws PkgError resolve_tst(deps_data, reqs_data) # require C (impossible) reqs_data = Any[ ["C"] ] -@test_throws ErrorException resolve_tst(deps_data, reqs_data) +@test_throws PkgError resolve_tst(deps_data, reqs_data) ## DEPENDENCY SCHEME 9: SIX PACKAGES, DAG deps_data = Any[ From 86127988916e82f96b99b87fc4d344dad654c64b Mon Sep 17 00:00:00 2001 From: Art Wild Date: Thu, 21 May 2015 19:39:42 -0400 Subject: [PATCH 0323/1938] speedup `prefetch` call --- base/pkg/cache.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/base/pkg/cache.jl b/base/pkg/cache.jl index 0f1e216ee1267..13ff6c6b926db 100644 --- a/base/pkg/cache.jl +++ b/base/pkg/cache.jl @@ -47,11 +47,13 @@ function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) end try LibGit2.set_remote_url(repo, url) - if !all(sha1->LibGit2.iscommit(sha1, repo), sha1s) + in_cache = map(sha1->LibGit2.iscommit(sha1, repo), sha1s) + if !all(in_cache) info("Updating cache of $pkg...") LibGit2.fetch(repo) + in_cache = map(sha1->LibGit2.iscommit(sha1, repo), sha1s) end - filter(sha1->!LibGit2.iscommit(sha1, repo), sha1s) + sha1s[!in_cache] finally finalize(repo) # closing repo opened/created above end From 9016b119e9d0ba6259f5aae0359fde19b7f81321 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Thu, 21 May 2015 19:45:16 -0400 Subject: [PATCH 0324/1938] refactored GitConfig methods --- base/pkg/generate.jl | 9 ++----- base/pkg/github.jl | 4 +-- base/pkg/libgit2/config.jl | 55 +++++++++++++++++++++++++------------- 3 files changed, 40 insertions(+), 28 deletions(-) diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index 4529808d56bde..c218cef49526d 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -6,13 +6,8 @@ import ..LibGit2, ..Read, ...Pkg.PkgError importall ..LibGit2 copyright_year() = string(Dates.year(Dates.today())) -copyright_name(repo::GitRepo) = - with(GitConfig, repo) do cfg - LibGit2.get(cfg, "user.name", "") - end -github_user() = with(GitConfig) do cfg - LibGit2.get(cfg, "github.user", "") - end +copyright_name(repo::GitRepo) = getconfig(repo, "user.name", "") +github_user() = getconfig("user.name", "github.user", "") function git_contributors(repo::GitRepo, n::Int=typemax(Int)) contrib = Dict() diff --git a/base/pkg/github.jl b/base/pkg/github.jl index dbb2cb8be0dc8..f7a081cb03b47 100644 --- a/base/pkg/github.jl +++ b/base/pkg/github.jl @@ -12,9 +12,7 @@ const AUTH_DATA = Dict{Any,Any}( ) function user() - usr = LibGit2.with(LibGit2.GitConfig) do cfg - LibGit2.get(cfg, "github.user", "") - end + usr = LibGit2.getconfig("github.user", "") if isempty(usr) throw(PkgError(""" no GitHub user name configured; please configure it with: diff --git a/base/pkg/libgit2/config.jl b/base/pkg/libgit2/config.jl index 336b9b623c7dd..d6ab81e75da51 100644 --- a/base/pkg/libgit2/config.jl +++ b/base/pkg/libgit2/config.jl @@ -55,24 +55,43 @@ function get{T}(c::GitConfig, name::AbstractString, default::T) return res end -function set!{T}(c::GitConfig, name::AbstractString, value::T) - err = if T<:AbstractString - @check ccall((:git_config_set_string, :libgit2), Cint, - (Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}), c.ptr, name, value) - elseif is(T, Bool) - bval = Int32(value) - @check ccall((:git_config_set_bool, :libgit2), Cint, - (Ptr{Void}, Ptr{UInt8}, Cint), c.ptr, name, bval) - elseif is(T, Int32) - @check ccall((:git_config_set_int32, :libgit2), Cint, - (Ptr{Void}, Ptr{UInt8}, Cint), c.ptr, name, value) - elseif is(T, Int64) - @check ccall((:git_config_set_int64, :libgit2), Cint, - (Ptr{Void}, Ptr{UInt8}, Cintmax_t), c.ptr, name, value) - else - warn("Type $T is not supported") - -1 +function getconfig{T}(r::GitRepo, name::AbstractString, default::T) + with(GitConfig, r) do cfg + get(cfg, name, default) end - return err end +function getconfig{T}(rname::AbstractString, name::AbstractString, default::T) + with(GitRepo, rname) do r + with(GitConfig, r) do cfg + get(cfg, name, default) + end + end +end + +function getconfig{T}(name::AbstractString, default) + with(GitConfig) do cfg + get(cfg, name, default) + end +end + +function set!{T <: AbstractString}(c::GitConfig, name::AbstractString, value::T) + @check ccall((:git_config_set_string, :libgit2), Cint, + (Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}), c.ptr, name, value) +end + +function set!(c::GitConfig, name::AbstractString, value::Bool) + bval = Int32(value) + @check ccall((:git_config_set_bool, :libgit2), Cint, + (Ptr{Void}, Ptr{UInt8}, Cint), c.ptr, name, bval) +end + +function set!(c::GitConfig, name::AbstractString, value::Int32) + @check ccall((:git_config_set_int32, :libgit2), Cint, + (Ptr{Void}, Ptr{UInt8}, Cint), c.ptr, name, value) +end + +function set!(c::GitConfig, name::AbstractString, value::Int64) + @check ccall((:git_config_set_int64, :libgit2), Cint, + (Ptr{Void}, Ptr{UInt8}, Cintmax_t), c.ptr, name, value) +end From 21fcc2c715c832287c30333152051aa45106a5f9 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Thu, 21 May 2015 19:45:46 -0400 Subject: [PATCH 0325/1938] removed prefix from `finalize` calls --- test/libgit2.jl | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/libgit2.jl b/test/libgit2.jl index 2d2f83e44e525..e837075e219a7 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -2,7 +2,7 @@ # check that libgit2 has been installed correctly -const LIBGIT2_VER = v"0.22.2+" +const LIBGIT2_VER = v"0.22.1+" function check_version() major, minor, patch = Cint[0], Cint[0], Cint[0] @@ -47,7 +47,7 @@ temp_dir() do dir url = "https://github.com/JuliaLang/Example.jl" path = joinpath(dir, "Example.Bare") repo = Pkg.LibGit2.clone(url, path, bare = true, remote_cb = Pkg.LibGit2.mirror_cb) - Pkg.LibGit2.finalize(repo) + finalize(repo) @test isdir(path) @test isfile(joinpath(path, Pkg.LibGit2.GitConst.HEAD_FILE)) end @@ -57,7 +57,7 @@ temp_dir() do dir url = "https://github.com/JuliaLang/Example.jl" path = joinpath(dir, "Example") repo = Pkg.LibGit2.clone(url, path) - Pkg.LibGit2.finalize(repo) + finalize(repo) @test isdir(path) @test isdir(joinpath(path, ".git")) end @@ -74,7 +74,7 @@ temp_dir() do dir url = "https://github.com/JuliaLang/julia.git" remote = Pkg.LibGit2.GitRemote(repo, branch, url) Pkg.LibGit2.save(remote) - Pkg.LibGit2.finalize(remote) + finalize(remote) config = joinpath(path, ".git", "config") lines = split(open(readall, config, "r"), "\n") @@ -83,7 +83,7 @@ temp_dir() do dir remote = Pkg.LibGit2.get(Pkg.LibGit2.GitRemote, repo, branch) @test Pkg.LibGit2.url(remote) == url - Pkg.LibGit2.finalize(repo) + finalize(repo) end # fetch @@ -95,7 +95,7 @@ temp_dir() do dir_cache Pkg.LibGit2.with(Pkg.LibGit2.GitConfig, repo) do cfg credentials!(cfg) end - Pkg.LibGit2.finalize(repo) + finalize(repo) # fetch @@ -113,7 +113,7 @@ temp_dir() do dir_cache Pkg.LibGit2.fetch(repo, path_cache, refspecs = "+refs/*:refs/remotes/cache/*") refs2 = parse(Int, readchomp(pipe(`find $(joinpath(path, ".git/refs"))`,`wc -l`))) - Pkg.LibGit2.finalize(repo) + finalize(repo) @test refs1 > 0 @test refs2 > 0 @test refs2 > refs1 @@ -128,7 +128,7 @@ temp_dir() do dir_cache end oids = Pkg.LibGit2.map((oid,repo)->string(oid), repo, by = Pkg.LibGit2.GitConst.SORT_TIME) @test length(oids) > 0 - Pkg.LibGit2.finalize(repo) + finalize(repo) Pkg.LibGit2.with(Pkg.LibGit2.GitRepo, path) do repo oid = Pkg.LibGit2.Oid(oids[end]) @@ -151,12 +151,12 @@ temp_dir() do dir_cache @test isa(cmtr, Pkg.LibGit2.Signature) @test length(cmtr.email) > 0 @test length(cmsg) > 0 - Pkg.LibGit2.finalize(repo) + finalize(repo) sig = Pkg.LibGit2.Signature("AAA", "AAA@BBB.COM", round(time(), 0), 0) git_sig = convert(Pkg.LibGit2.GitSignature, sig) sig2 = Pkg.LibGit2.Signature(git_sig) - Pkg.LibGit2.finalize(git_sig) + finalize(git_sig) @test sig.name == sig2.name @test sig.email == sig2.email @test sig.time == sig2.time @@ -196,7 +196,7 @@ begin arr = convert(Vector{AbstractString}, sa1) @test arr[1] == p1 finally - Pkg.LibGit2.finalize(sa1) + finalize(sa1) end sa2 = Pkg.LibGit2.StrArrayStruct(p1, p2) @@ -205,7 +205,7 @@ begin @test arr[1] == p1 @test arr[2] == p2 finally - Pkg.LibGit2.finalize(sa2) + finalize(sa2) end end From 2d456ab609756059ccd7e70a53ef8d009e13dadd Mon Sep 17 00:00:00 2001 From: Art Wild Date: Thu, 21 May 2015 19:48:27 -0400 Subject: [PATCH 0326/1938] newline at end of file --- base/pkg/libgit2.jl | 2 +- base/pkg/libgit2/commit.jl | 2 +- base/pkg/libgit2/error.jl | 2 +- base/pkg/libgit2/index.jl | 2 +- base/pkg/libgit2/merge.jl | 1 - base/pkg/libgit2/reference.jl | 1 - base/pkg/libgit2/repository.jl | 2 +- base/pkg/libgit2/signature.jl | 2 +- base/pkg/libgit2/strarray.jl | 2 +- base/pkg/libgit2/types.jl | 2 +- base/pkg/libgit2/walker.jl | 2 +- 11 files changed, 9 insertions(+), 11 deletions(-) diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index dd60708faab18..ad7d7866ed483 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -435,4 +435,4 @@ function __init__() end end -end # module \ No newline at end of file +end # module diff --git a/base/pkg/libgit2/commit.jl b/base/pkg/libgit2/commit.jl index e61699bad1c93..1a740b1670ac0 100644 --- a/base/pkg/libgit2/commit.jl +++ b/base/pkg/libgit2/commit.jl @@ -86,4 +86,4 @@ function commit(repo::GitRepo, msg::AbstractString; finalize(comm_sig) end return commit_id -end \ No newline at end of file +end diff --git a/base/pkg/libgit2/error.jl b/base/pkg/libgit2/error.jl index 9b177e49b27e7..cb772c0a99f0b 100644 --- a/base/pkg/libgit2/error.jl +++ b/base/pkg/libgit2/error.jl @@ -100,4 +100,4 @@ macro check(git_func) end err end -end \ No newline at end of file +end diff --git a/base/pkg/libgit2/index.jl b/base/pkg/libgit2/index.jl index 168c9ff79c2f7..e9250b425d522 100644 --- a/base/pkg/libgit2/index.jl +++ b/base/pkg/libgit2/index.jl @@ -115,4 +115,4 @@ function Base.getindex(idx::GitIndex, i::Csize_t) ie_ptr == C_NULL && return nothing return unsafe_load(convert(Ptr{IndexEntry}, ie_ptr), 1) end -Base.getindex(idx::GitIndex, i::Int) = getindex(idx, Csize_t(i)) \ No newline at end of file +Base.getindex(idx::GitIndex, i::Int) = getindex(idx, Csize_t(i)) diff --git a/base/pkg/libgit2/merge.jl b/base/pkg/libgit2/merge.jl index 883d13cee1ecb..3186bc35c8ba1 100644 --- a/base/pkg/libgit2/merge.jl +++ b/base/pkg/libgit2/merge.jl @@ -68,4 +68,3 @@ function merge!(repo::GitRepo; fast_forward::Bool=false) end end end - diff --git a/base/pkg/libgit2/reference.jl b/base/pkg/libgit2/reference.jl index 44b33f4bd6c99..1ae4dc874bf35 100644 --- a/base/pkg/libgit2/reference.jl +++ b/base/pkg/libgit2/reference.jl @@ -151,4 +151,3 @@ function target!(ref::GitReference, new_oid::Oid; msg::AbstractString="") end return GitReference(ref_ptr_ptr[]) end - diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index 2a4383a81fbd3..07d13b8cc0b19 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -163,4 +163,4 @@ function reset!(repo::GitRepo, obj::GitObject, mode::Cint; (Ptr{Void}, Ptr{Void}, Cint, Ptr{CheckoutOptionsStruct}, Ptr{SignatureStruct}, Ptr{UInt8}), repo.ptr, obj.ptr, mode, Ref(checkout_opts), sig.ptr, msg) end -end \ No newline at end of file +end diff --git a/base/pkg/libgit2/signature.jl b/base/pkg/libgit2/signature.jl index 82484d91eccc2..de57d7640d3f4 100644 --- a/base/pkg/libgit2/signature.jl +++ b/base/pkg/libgit2/signature.jl @@ -41,4 +41,4 @@ function default_signature(repo::GitRepo) @check ccall((:git_signature_default, :libgit2), Cint, (Ptr{Ptr{SignatureStruct}}, Ptr{Void}), sig_ptr_ptr, repo.ptr) return GitSignature(sig_ptr_ptr[]) -end \ No newline at end of file +end diff --git a/base/pkg/libgit2/strarray.jl b/base/pkg/libgit2/strarray.jl index b4a012dca216b..22f4b108672d1 100644 --- a/base/pkg/libgit2/strarray.jl +++ b/base/pkg/libgit2/strarray.jl @@ -22,4 +22,4 @@ function Base.convert(::Type{Vector{AbstractString}}, sa::StrArrayStruct) arr[i] = bytestring(unsafe_load(sa.strings, i)) end return arr -end \ No newline at end of file +end diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index a07ea16c75f94..8a3227a68d492 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -377,4 +377,4 @@ function getobjecttype(obj_type::Cint) else throw(GitError(Error.Object, Error.ENOTFOUND, "Object type $obj_type is not supported")) end -end \ No newline at end of file +end diff --git a/base/pkg/libgit2/walker.jl b/base/pkg/libgit2/walker.jl index 68afae59ad6fe..327a68e704a7e 100644 --- a/base/pkg/libgit2/walker.jl +++ b/base/pkg/libgit2/walker.jl @@ -100,4 +100,4 @@ function Base.count(f::Function, repo::GitRepo; end end return c -end \ No newline at end of file +end From 327e989045041c467222146c947e36f1a9c9ef31 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sat, 23 May 2015 04:50:36 -0400 Subject: [PATCH 0327/1938] No more `Git`! updated `merge`, `fetch`, `rebase`, `push`, `diff` & many small fixes general cleanup and refactoring --- base/pkg/cache.jl | 2 +- base/pkg/entry.jl | 93 ++++++------ base/pkg/generate.jl | 4 +- base/pkg/libgit2.jl | 175 +++++++++++++++++----- base/pkg/libgit2/blob.jl | 10 +- base/pkg/libgit2/config.jl | 4 +- base/pkg/libgit2/const.jl | 260 +++++++++++++++------------------ base/pkg/libgit2/diff.jl | 44 ++++++ base/pkg/libgit2/index.jl | 2 +- base/pkg/libgit2/merge.jl | 66 +++------ base/pkg/libgit2/oid.jl | 2 +- base/pkg/libgit2/rebase.jl | 68 +++++++++ base/pkg/libgit2/remote.jl | 37 +++-- base/pkg/libgit2/repository.jl | 17 +-- base/pkg/libgit2/tag.jl | 9 ++ base/pkg/libgit2/types.jl | 63 ++++++-- base/pkg/write.jl | 2 +- test/libgit2.jl | 2 +- 18 files changed, 545 insertions(+), 315 deletions(-) create mode 100644 base/pkg/libgit2/diff.jl create mode 100644 base/pkg/libgit2/rebase.jl diff --git a/base/pkg/cache.jl b/base/pkg/cache.jl index 13ff6c6b926db..47fc05f5d8a03 100644 --- a/base/pkg/cache.jl +++ b/base/pkg/cache.jl @@ -42,7 +42,7 @@ function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) LibGit2.clone(url, cache, bare = true, remote_cb = LibGit2.mirror_cb) catch err isdir(cache) && rm(cache, recursive=true) - throw(PkgError("Cannot clone $pkg from $url\n", err)) + throw(PkgError("Cannot clone $pkg from $url. $(err.msg)")) end end try diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 4a39cfd48d3e1..03b4dd3cf782a 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -318,14 +318,17 @@ function update(branch::AbstractString) LibGit2.merge!(repo) end end - # TODO: handle merge conflicts LibGit2.fetch(repo) - LibGit2.merge!(repo) + LibGit2.merge!(repo, fast_forward=true) || LibGit2.rebase!(repo, "origin/$branch") end avail = Read.available() # this has to happen before computing free/fixed for pkg in filter!(Read.isinstalled,collect(keys(avail))) - Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail[pkg]]) + try + Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail[pkg]]) + catch err + warn("Package $pkg: unable to update cache\n$(err.msg)") + end end instd = Read.installed(avail) free = Read.free(instd) @@ -345,7 +348,11 @@ function update(branch::AbstractString) end end if haskey(avail,pkg) - Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail[pkg]]) + try + Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail[pkg]]) + catch err + warn("Package $pkg: unable to update cache\n$(err.msg)") + end end end info("Computing changes...") @@ -362,9 +369,7 @@ function pull_request(dir::AbstractString, commit::AbstractString="", url::Abstr !LibGit2.iscommit(commit, repo) && throw(PkgError("Cannot find pull commit: $commit")) end if isempty(url) - with(GitConfig, repo) do cfg - url = LibGit2.get(cfg, "remote.origin.url", "") - end + url = LibGit2.getconfig(repo, "remote.origin.url", "") end m = match(LibGit2.GITHUB_REGEX, url) @@ -376,9 +381,9 @@ function pull_request(dir::AbstractString, commit::AbstractString="", url::Abstr fork = response["ssh_url"] branch = "pull-request/$(commit[1:8])" info("Pushing changes as branch $branch") - LibGit2.push(repo, fork, "$commit:refs/heads/$branch") + LibGit2.push(repo, url=fork, refspecs=["$commit:refs/heads/$branch"]) pr_url = "$(response["html_url"])/compare/$branch" - @osx? run(`open $pr_url`) : info("To create a pull-request, open:\n\n $pr_url\n") + info("To create a pull-request, open:\n\n $pr_url\n") end end @@ -397,32 +402,34 @@ function publish(branch::AbstractString) ahead_remote, ahead_local = LibGit2.revcount(repo, "origin/$branch", branch) ahead_remote > 0 && throw(PkgError("METADATA is behind origin/$branch – run `Pkg.update()` before publishing")) ahead_local == 0 && throw(PkgError("There are no METADATA changes to publish")) - end - tags = Dict{ByteString,Vector{ASCIIString}}() - Git.run(`update-index -q --really-refresh`, dir="METADATA") - cmd = `diff --name-only --diff-filter=AMR origin/$branch HEAD --` - for line in eachline(Git.cmd(cmd, dir="METADATA")) - path = chomp(line) - m = match(r"^(.+?)/versions/([^/]+)/sha1$", path) - m !== nothing && ismatch(Base.VERSION_REGEX, m.captures[2]) || continue - pkg, ver = m.captures; ver = convert(VersionNumber,ver) - sha1 = readchomp(joinpath("METADATA",path)) - if Git.success(`cat-file -e origin/$branch:$path`, dir="METADATA") - old = Git.readchomp(`cat-file blob origin/$branch:$path`, dir="METADATA") - old == sha1 || throw(PkgError("$pkg v$ver SHA1 changed in METADATA – refusing to publish")) - end - any(split(Git.readall(`tag --contains $sha1`, dir=pkg))) do tag - ver == convert(VersionNumber,tag) || return false - haskey(tags,pkg) || (tags[pkg] = ASCIIString[]) - push!(tags[pkg], tag) - return true - end || throw(PkgError("$pkg v$ver is incorrectly tagged – $sha1 expected")) - end - isempty(tags) && info("No new package versions to publish") - info("Validating METADATA") - check_metadata(Set(keys(tags))) - @sync for pkg in sort!(collect(keys(tags))) - @async begin + + tags = Dict{ByteString,Vector{ASCIIString}}() + + # get changed files + for path in LibGit2.diff_files(repo, "origin/metadata-v2", LibGit2.GitConst.HEAD_FILE) + println(path) + m = match(r"^(.+?)/versions/([^/]+)/sha1$", path) + m != nothing && ismatch(Base.VERSION_REGEX, m.captures[2]) || continue + pkg, ver = m.captures; ver = convert(VersionNumber,ver) + sha1 = readchomp(joinpath("METADATA",path)) + old = LibGit2.cat(repo, LibGit2.GitBlob, "origin/$branch:$path") + old != nothing && old != sha1 && throw(PkgError("$pkg v$ver SHA1 changed in METADATA – refusing to publish")) + with(GitRepo, pkg) do pkg_repo + tag_name = "v$ver" + tag_commit = LibGit2.revparseid(pkg_repo, "$(tag_name)^{commit}") + LibGit2.iszero(tag_commit) || string(tag_commit) == sha1 || return false + haskey(tags,pkg) || (tags[pkg] = ASCIIString[]) + push!(tags[pkg], tag_name) + return true + end || throw(PkgError("$pkg v$ver is incorrectly tagged – $sha1 expected")) + end + isempty(tags) && info("No new package versions to publish") + info("Validating METADATA") + check_metadata(Set(keys(tags))) + end + + for pkg in sort!(collect(keys(tags))) + with(GitRepo, pkg) do pkg_repo forced = ASCIIString[] unforced = ASCIIString[] for tag in tags[pkg] @@ -431,13 +438,13 @@ function publish(branch::AbstractString) end if !isempty(forced) info("Pushing $pkg temporary tags: ", join(forced,", ")) - refspecs = ["refs/tags/$tag:refs/tags/$tag" for tag in forced] - Git.run(`push -q --force origin $refspecs`, dir=pkg) + LibGit2.push(pkg_repo, remote="origin", force=true, + refspecs=["refs/tags/$tag:refs/tags/$tag" for tag in forced]) end if !isempty(unforced) info("Pushing $pkg permanent tags: ", join(unforced,", ")) - refspecs = ["refs/tags/$tag:refs/tags/$tag" for tag in unforced] - Git.run(`push -q origin $refspecs`, dir=pkg) + LibGit2.push(pkg_repo, remote="origin", + refspecs=["refs/tags/$tag:refs/tags/$tag" for tag in unforced]) end end end @@ -600,15 +607,11 @@ end function register(pkg::AbstractString) url = "" try - url = with(GitRepo, pkg) do repo - with(GitConfig, repo) do cfg - LibGit2.get(cfg, "remote.origin.url", "") - end - end + url = LibGit2.getconfig(pkg, "remote.origin.url", "") catch err throw(PkgError("$pkg: $err")) end - isempty(url) || throw(PkgError("$pkg: no URL configured")) + !isempty(url) || throw(PkgError("$pkg: no URL configured")) register(pkg, LibGit2.normalize_url(url)) end diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index c218cef49526d..c1668b5208d66 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -6,8 +6,8 @@ import ..LibGit2, ..Read, ...Pkg.PkgError importall ..LibGit2 copyright_year() = string(Dates.year(Dates.today())) -copyright_name(repo::GitRepo) = getconfig(repo, "user.name", "") -github_user() = getconfig("user.name", "github.user", "") +copyright_name(repo::GitRepo) = LibGit2.getconfig(repo, "user.name", "") +github_user() = LibGit2.getconfig("github.user", "") function git_contributors(repo::GitRepo, n::Int=typemax(Int)) contrib = Dict() diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index ad7d7866ed483..5d180a87c570c 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -26,6 +26,8 @@ include("libgit2/index.jl") include("libgit2/merge.jl") include("libgit2/tag.jl") include("libgit2/blob.jl") +include("libgit2/diff.jl") +include("libgit2/rebase.jl") immutable State head::Oid @@ -72,39 +74,45 @@ isdirty(repo::GitRepo, paths::AbstractString=""; cached::Bool=false) = isdiff(re function isdiff(repo::GitRepo, treeish::AbstractString, paths::AbstractString=""; cached::Bool=false) tree_oid = revparseid(repo, "$treeish^{tree}") iszero(tree_oid) && return true - emptypathspec = isempty(paths) - result = false tree = get(GitTree, repo, tree_oid) - if !emptypathspec - sa = StrArrayStruct(paths) - diff_opts = DiffOptionsStruct(pathspec = sa) - end try - diff_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - if cached - @check ccall((:git_diff_tree_to_index, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, Ptr{DiffOptionsStruct}), - diff_ptr_ptr, repo.ptr, tree.ptr, NULL, emptypathspec ? C_NULL : Ref(diff_opts)) - else - @check ccall((:git_diff_tree_to_workdir_with_index, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{DiffOptionsStruct}), - diff_ptr_ptr, repo.ptr, tree.ptr, emptypathspec ? C_NULL : Ref(diff_opts)) - end - diff = GitDiff(diff_ptr_ptr[]) - - c = ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff.ptr) - result = c > 0 + diff = diff_tree(repo, tree, paths) + result = count(diff) > 0 finalize(diff) catch err result = true finally - !emptypathspec && finalize(sa) finalize(tree) end return result end +""" git diff --name-only --diff-filter= """ +function diff_files(repo::GitRepo, branch1::AbstractString, branch2::AbstractString; + filter::Set{Cint}=Set([GitConst.DELTA_ADDED, GitConst.DELTA_MODIFIED, GitConst.DELTA_DELETED])) + b1_id = revparseid(repo, branch1*"^{tree}") + b2_id = revparseid(repo, branch2*"^{tree}") + tree1 = get(GitTree, repo, b1_id) + tree2 = get(GitTree, repo, b2_id) + files = AbstractString[] + try + diff = diff_tree(repo, tree1, tree2) + for i in 1:count(diff) + delta = diff[i] + delta == nothing && break + if delta.status in filter + push!(files, bytestring(delta.new_file.path)) + end + end + finalize(diff) + finally + finalize(tree1) + finalize(tree2) + end + return files +end + function merge_base(one::AbstractString, two::AbstractString, repo::GitRepo) oid1_ptr = Ref(Oid(one)) oid2_ptr = Ref(Oid(two)) @@ -166,17 +174,20 @@ function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, name::Ptr{ end const mirror_cb = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}, Ptr{Void})) -""" git fetch []""" -function fetch(repo::GitRepo, remote::AbstractString="origin"; - refspecs::AbstractString="") - rmt = if isempty(refspecs) +""" git fetch [|] []""" +function fetch{T<:AbstractString}(repo::GitRepo; + remote::AbstractString="origin", + remoteurl::AbstractString="", + refspecs::Vector{T}=AbstractString[]) + rmt = if isempty(remoteurl) get(GitRemote, repo, remote) else - GitRemoteAnon(repo, remote, refspecs) + GitRemoteAnon(repo, remoteurl) end - try - fetch(repo, rmt, msg="from $(url(rmt))") + with(default_signature(repo)) do sig + fetch(rmt, sig, refspecs, msg="from $(url(rmt))") + end catch err warn("fetch: $err") finally @@ -184,6 +195,28 @@ function fetch(repo::GitRepo, remote::AbstractString="origin"; end end +""" git push [|] []""" +function push{T<:AbstractString}(repo::GitRepo; + refspecs::Vector{T}=AbstractString[], + remote::AbstractString="origin", + remoteurl::AbstractString="", + force::Bool=false) + rmt = if isempty(remoteurl) + get(GitRemote, repo, remote) + else + GitRemoteAnon(repo, remoteurl) + end + try + with(default_signature(repo)) do sig + push(rmt, sig, refspecs, force=force, msg="to $(url(rmt))") + end + catch err + warn("push: $err") + finally + finalize(rmt) + end +end + """ git branch """ function branch(repo::GitRepo) head_ref = head(repo) @@ -337,7 +370,7 @@ end """ git cat-file """ function cat{T<:GitObject}(repo::GitRepo, ::Type{T}, object::AbstractString) obj_id = revparseid(repo, object) - iszero(obj_id) && return + iszero(obj_id) && return nothing obj = get(T, repo, obj_id) if isa(obj, GitBlob) @@ -347,15 +380,6 @@ function cat{T<:GitObject}(repo::GitRepo, ::Type{T}, object::AbstractString) end end -""" git push []""" -function push(repo::GitRepo, url::AbstractString, refspecs::AbstractString="") - with(GitRemoteAnon(repo, url, refspecs)) do rmt - with(default_signature(repo)) do sig - push(rmt, sig) - end - end -end - """ git rev-list --count """ function revcount(repo::GitRepo, fst::AbstractString, snd::AbstractString) fst_id = revparseid(repo, fst) @@ -368,6 +392,81 @@ function revcount(repo::GitRepo, fst::AbstractString, snd::AbstractString) return (fc-1, sc-1) end +""" git merge [--ff-only] [] """ +function merge!(repo::GitRepo, committish::AbstractString=""; fast_forward::Bool=false) + # get head annotated upstream reference + ret = with(head(repo)) do head_ref + brn_ref = upstream(head_ref) + upst_ann = isempty(committish) ? GitAnnotated(repo, brn_ref) : GitAnnotated(repo, committish) + try + ma, mp = merge_analysis(repo, upst_ann) + (ma & GitConst.MERGE_ANALYSIS_UP_TO_DATE == GitConst.MERGE_ANALYSIS_UP_TO_DATE) && return true + if (ma & GitConst.MERGE_ANALYSIS_FASTFORWARD == GitConst.MERGE_ANALYSIS_FASTFORWARD) + # do fastforward: checkout tree and update branch references + # hur_oid = Oid(hur) + # with(get(GitCommit, repo, hur_oid)) do cmt + # checkout_tree(repo, cmt) + # end + # target!(hr, hur_oid, msg="pkg.libgit2.megre!: fastforward $(name(hur)) into $(name(hr))") + # head!(repo, hur, msg="--fastforward") + + brn_ref_oid = Oid(brn_ref) + target!(head_ref, brn_ref_oid, msg="pkg.libgit2.megre!: fastforward $(name(brn_ref)) into $(name(head_ref))") + reset!(repo, brn_ref_oid, GitConst.RESET_HARD) + elseif (ma & GitConst.MERGE_ANALYSIS_NORMAL == GitConst.MERGE_ANALYSIS_NORMAL) + fast_forward && return false # do not do merge + merge(repo, hua) + cleanup(repo) + info("Review and commit merged changes.") + else + warn("Unknown merge analysis result. Merging is not possible.") + return false + end + return true + finally + finalize(upst_ann) + finalize(brn_ref) + end + end + return ret +end + +""" git rebase --merge [--onto ] [] """ +function rebase!(repo::GitRepo, upstream::AbstractString="", newbase::AbstractString="") + with(head(repo)) do head_ref + head_ann = GitAnnotated(repo, head_ref) + upst_ann = if isempty(upstream) + with(upstream(head_ref)) do brn_ref + GitAnnotated(repo, brn_ref) + end + else + GitAnnotated(repo, upstream) + end + try + sig = default_signature(repo) + try + rbs = GitRebase(repo, head_ann, upst_ann, sig=Nullable(sig)) + try + while (rbs_op = next(rbs)) != nothing + commit(rbs, sig) + end + finish(rbs, sig) + catch err + abort(rbs, sig) + finally + finalize(rbs) + end + finally + #!isnull(onto_ann) && finalize(get(onto_ann)) + finalize(sig) + end + finally + finalize(upst_ann) + finalize(head_ann) + end + end +end + """ Returns all commit authors """ function authors(repo::GitRepo) diff --git a/base/pkg/libgit2/blob.jl b/base/pkg/libgit2/blob.jl index 502a15238dfbc..3c10dfd6fd706 100644 --- a/base/pkg/libgit2/blob.jl +++ b/base/pkg/libgit2/blob.jl @@ -2,4 +2,12 @@ function content(blob::GitBlob) return ccall((:git_blob_rawcontent, :libgit2), Ptr{Void}, (Ptr{Void},), blob.ptr) -end \ No newline at end of file +end + +function isbinary(blob::GitBlob) + return ccall((:git_blob_is_binary, :libgit2), CInt, (Ptr{Void},), blob.ptr) == 1 +end + +function Base.length(blob::GitBlob) + return ccall((:git_blob_rawsize, :libgit2), Coff_t, (Ptr{Void},), blob.ptr) +end diff --git a/base/pkg/libgit2/config.jl b/base/pkg/libgit2/config.jl index d6ab81e75da51..70170de01c128 100644 --- a/base/pkg/libgit2/config.jl +++ b/base/pkg/libgit2/config.jl @@ -26,7 +26,7 @@ end function get{T}(::Type{T}, c::GitConfig, name::AbstractString) if T<:AbstractString str_ptr = Ref{Ptr{UInt8}}(C_NULL) - @check ccall((:git_config_get_string, :libgit2), Cint, + @check ccall((:git_config_get_string, :libgit2), Cint, #TODO: git_config_get_string_buf (Ptr{Ptr{UInt8}}, Ptr{Void}, Ptr{UInt8}), str_ptr, c.ptr, name) return bytestring(str_ptr[]) elseif is(T, Bool) @@ -69,7 +69,7 @@ function getconfig{T}(rname::AbstractString, name::AbstractString, default::T) end end -function getconfig{T}(name::AbstractString, default) +function getconfig{T}(name::AbstractString, default::T) with(GitConfig) do cfg get(cfg, name, default) end diff --git a/base/pkg/libgit2/const.jl b/base/pkg/libgit2/const.jl index f76f560f5acc7..44bb89b4d46e8 100644 --- a/base/pkg/libgit2/const.jl +++ b/base/pkg/libgit2/const.jl @@ -5,6 +5,7 @@ module GitConst const HEAD_FILE = "HEAD" const REMOTE_ORIGIN = "origin" + # objs const OBJ_ANY = Cint(-2) const OBJ_BAD = Cint(-1) const OBJ_COMMIT = Cint(1) @@ -12,19 +13,23 @@ module GitConst const OBJ_BLOB = Cint(3) const OBJ_TAG = Cint(4) - const SORT_NONE = Cint(0) - const SORT_TOPOLOGICAL = Cint(1) << Cint(0) - const SORT_TIME = Cint(1) << Cint(1) - const SORT_REVERSE = Cint(1) << Cint(2) + #revwalk + const SORT_NONE = Cint(0) + const SORT_TOPOLOGICAL = Cint(1 << 0) + const SORT_TIME = Cint(1 << 1) + const SORT_REVERSE = Cint(1 << 2) - const REF_INVALID = Cint(0) - const REF_OID = Cint(1) + # refs + const REF_INVALID = Cint(0) + const REF_OID = Cint(1) const REF_SYMBOLIC = Cint(2) - const REF_LISTALL = REF_OID | REF_SYMBOLIC + const REF_LISTALL = REF_OID | REF_SYMBOLIC - const BRANCH_LOCAL = Cint(1) + # branch + const BRANCH_LOCAL = Cint(1) const BRANCH_REMOTE = Cint(2) + # file const FILEMODE_NEW = Cint(00000) const FILEMODE_TREE = Cint(16384) const FILEMODE_BLOB = Cint(33188) @@ -32,91 +37,73 @@ module GitConst const FILEMODE_LINK = Cint(40960) const FILEMODE_COMMIT = Cint(57344) + # checkout const CHECKOUT_NONE = Cuint(0) - const CHECKOUT_SAFE = Cuint(1) << Cint(0) - const CHECKOUT_SAFE_CREATE = Cuint(1) << Cint(1) - const CHECKOUT_FORCE = Cuint(1) << Cint(2) - const CHECKOUT_ALLOW_CONFLICTS = Cuint(1) << Cint(4) - const CHECKOUT_REMOVE_UNTRACKED = Cuint(1) << Cint(5) - const CHECKOUT_REMOVE_IGNORED = Cuint(1) << Cint(6) - const CHECKOUT_UPDATE_ONLY = Cuint(1) << Cint(7) - const CHECKOUT_DONT_UPDATE_INDEX = Cuint(1) << Cint(8) - const CHECKOUT_NO_REFRESH = Cuint(1) << Cint(9) - const CHECKOUT_SKIP_UNMERGED = Cuint(1) << Cint(10) - const CHECKOUT_USE_OURS = Cuint(1) << Cint(11) - const CHECKOUT_USE_THEIRS = Cuint(1) << Cint(12) - const CHECKOUT_DISABLE_PATHSPEC_MATCH = Cuint(1) << Cint(13) - const CHECKOUT_SKIP_LOCKED_DIRECTORIES = Cuint(1) << Cint(18) - const CHECKOUT_DONT_OVERWRITE_IGNORED = Cuint(1) << Cint(19) - - const CHECKOUT_UPDATE_SUBMODULES = Cuint(1) << Cint(16) - const CHECKOUT_UPDATE_SUBMODULES_IF_CHANGED = Cuint(1) << Cint(17) + const CHECKOUT_SAFE = Cuint(1 << 0) + const CHECKOUT_SAFE_CREATE = Cuint(1 << 1) + const CHECKOUT_FORCE = Cuint(1 << 2) + const CHECKOUT_ALLOW_CONFLICTS = Cuint(1 << 4) + const CHECKOUT_REMOVE_UNTRACKED = Cuint(1 << 5) + const CHECKOUT_REMOVE_IGNORED = Cuint(1 << 6) + const CHECKOUT_UPDATE_ONLY = Cuint(1 << 7) + const CHECKOUT_DONT_UPDATE_INDEX = Cuint(1 << 8) + const CHECKOUT_NO_REFRESH = Cuint(1 << 9) + const CHECKOUT_SKIP_UNMERGED = Cuint(1 << 10) + const CHECKOUT_USE_OURS = Cuint(1 << 11) + const CHECKOUT_USE_THEIRS = Cuint(1 << 12) + const CHECKOUT_DISABLE_PATHSPEC_MATCH = Cuint(1 << 13) + const CHECKOUT_SKIP_LOCKED_DIRECTORIES = Cuint(1 << 18) + const CHECKOUT_DONT_OVERWRITE_IGNORED = Cuint(1 << 19) + + const CHECKOUT_UPDATE_SUBMODULES = Cuint(1 << 16) + const CHECKOUT_UPDATE_SUBMODULES_IF_CHANGED = Cuint(1 << 17) const CHECKOUT_NOTIFY_NONE = Cuint(0) - const CHECKOUT_NOTIFY_CONFLICT = Cuint(1) << Cint(0) - const CHECKOUT_NOTIFY_DIRTY = Cuint(1) << Cint(1) - const CHECKOUT_NOTIFY_UPDATED = Cuint(1) << Cint(2) - const CHECKOUT_NOTIFY_UNTRACKED = Cuint(1) << Cint(3) - const CHECKOUT_NOTIFY_IGNORED = Cuint(1) << Cint(4) + const CHECKOUT_NOTIFY_CONFLICT = Cuint(1 << 0) + const CHECKOUT_NOTIFY_DIRTY = Cuint(1 << 1) + const CHECKOUT_NOTIFY_UPDATED = Cuint(1 << 2) + const CHECKOUT_NOTIFY_UNTRACKED = Cuint(1 << 3) + const CHECKOUT_NOTIFY_IGNORED = Cuint(1 << 4) const CHECKOUT_NOTIFY_ALL = 0x0FFFF - const SUBMODULE_UPDATE_RESET = Cint(-1) - const SUBMODULE_UPDATE_CHECKOUT = Cint(1) - const SUBMODULE_UPDATE_REBASE = Cint(2) - const SUBMODULE_UPDATE_MERGE = Cint(3) - const SUBMODULE_UPDATE_NONE = Cint(4) - const SUBMODULE_UPDATE_DEFAULT = Cint(0) - - # git_submodule_ignore_t - const SUBMODULE_IGNORE_RESET = Cint(-1) - const SUBMODULE_IGNORE_NONE = Cint(1) - const SUBMODULE_IGNORE_UNTRACKED = Cint(2) - const SUBMODULE_IGNORE_DIRTY = Cint(3) - const SUBMODULE_IGNORE_ALL = Cint(4) - const SUBMODULE_IGNORE_DEFAULT = Cint(0) - - const TREEWALK_PRE = Cint(0) - const TREEWALK_POST = Cint(1) - - const GIT_PATH_MAX = Cint(4096) - + # diff const DIFF_OPTIONS_VERSION = Cuint(1) - const DIFF_NORMAL = Cuint(0) - const DIFF_REVERSE = Cuint(1) << Cint(0) - const DIFF_INCLUDE_IGNORED = Cuint(1) << Cint(1) - const DIFF_RECURSE_IGNORED_DIRS = Cuint(1) << Cint(2) - const DIFF_INCLUDE_UNTRACKED = Cuint(1) << Cint(3) - const DIFF_RECURSE_UNTRACKED_DIRS = Cuint(1) << Cint(4) - const DIFF_INCLUDE_UNMODIFIED = Cuint(1) << Cint(5) - const DIFF_INCLUDE_TYPECHANGE = Cuint(1) << Cint(6) - const DIFF_INCLUDE_TYPECHANGE_TREES = Cuint(1) << Cint(7) - const DIFF_IGNORE_FILEMODE = Cuint(1) << Cint(8) - const DIFF_IGNORE_SUBMODULES = Cuint(1) << Cint(9) - const DIFF_IGNORE_CASE = Cuint(1) << Cint(10) - const DIFF_DISABLE_PATHSPEC_MATCH = Cuint(1) << Cint(12) - const DIFF_SKIP_BINARY_CHECK = Cuint(1) << Cint(13) - const DIFF_ENABLE_FAST_UNTRACKED_DIRS = Cuint(1) << Cint(14) - - const DIFF_FORCE_TEXT = Cuint(1) << Cint(20) - const DIFF_FORCE_BINARY = Cuint(1) << Cint(21) - const DIFF_IGNORE_WHITESPACE = Cuint(1) << Cint(22) - const DIFF_IGNORE_WHITESPACE_CHANGE = Cuint(1) << Cint(23) - const DIFF_IGNORE_WHITESPACE_EOL = Cuint(1) << Cint(24) - const DIFF_SHOW_UNTRACKED_CONTENT = Cuint(1) << Cint(25) - const DIFF_SHOW_UNMODIFIED = Cuint(1) << Cint(26) - const DIFF_PATIENCE = Cuint(1) << Cint(28) - const DIFF_MINIMAL = Cuint(1) << Cint(29) - - const DIFF_FLAG_BINARY = Cuint(1) << Cint(0) - const DIFF_FLAG_NOT_BINARY = Cuint(1) << Cint(1) - const DIFF_FLAG_VALID_OID = Cuint(1) << Cint(2) - - const DIFF_FORMAT_PATCH = Cuint(1) + const DIFF_NORMAL = Cuint(0) + const DIFF_REVERSE = Cuint(1 << 0) + const DIFF_INCLUDE_IGNORED = Cuint(1 << 1) + const DIFF_RECURSE_IGNORED_DIRS = Cuint(1 << 2) + const DIFF_INCLUDE_UNTRACKED = Cuint(1 << 3) + const DIFF_RECURSE_UNTRACKED_DIRS = Cuint(1 << 4) + const DIFF_INCLUDE_UNMODIFIED = Cuint(1 << 5) + const DIFF_INCLUDE_TYPECHANGE = Cuint(1 << 6) + const DIFF_INCLUDE_TYPECHANGE_TREES = Cuint(1 << 7) + const DIFF_IGNORE_FILEMODE = Cuint(1 << 8) + const DIFF_IGNORE_SUBMODULES = Cuint(1 << 9) + const DIFF_IGNORE_CASE = Cuint(1 << 10) + const DIFF_DISABLE_PATHSPEC_MATCH = Cuint(1 << 12) + const DIFF_SKIP_BINARY_CHECK = Cuint(1 << 13) + const DIFF_ENABLE_FAST_UNTRACKED_DIRS = Cuint(1 << 14) + + const DIFF_FORCE_TEXT = Cuint(1 << 20) + const DIFF_FORCE_BINARY = Cuint(1 << 21) + const DIFF_IGNORE_WHITESPACE = Cuint(1 << 22) + const DIFF_IGNORE_WHITESPACE_CHANGE = Cuint(1 << 23) + const DIFF_IGNORE_WHITESPACE_EOL = Cuint(1 << 24) + const DIFF_SHOW_UNTRACKED_CONTENT = Cuint(1 << 25) + const DIFF_SHOW_UNMODIFIED = Cuint(1 << 26) + const DIFF_PATIENCE = Cuint(1 << 28) + const DIFF_MINIMAL = Cuint(1 << 29) + + const DIFF_FLAG_BINARY = Cuint(1 << 0) + const DIFF_FLAG_NOT_BINARY = Cuint(1 << 1) + const DIFF_FLAG_VALID_OID = Cuint(1 << 2) + + const DIFF_FORMAT_PATCH = Cuint(1) const DIFF_FORMAT_PATCH_HEADER = Cuint(2) - const DIFF_FORMAT_RAW = Cuint(3) - const DIFF_FORMAT_NAME_ONLY = Cuint(4) - const DIFF_FORMAT_NAME_STATUS = Cuint(5) + const DIFF_FORMAT_RAW = Cuint(3) + const DIFF_FORMAT_NAME_ONLY = Cuint(4) + const DIFF_FORMAT_NAME_STATUS = Cuint(5) const DELTA_UNMODIFIED = Cint(0) const DELTA_ADDED = Cint(1) @@ -128,18 +115,6 @@ module GitConst const DELTA_UNTRACKED = Cint(7) const DELTA_TYPECHANGE = Cint(8) - const DIFF_LINE_CONTEXT = Cchar(' ') - const DIFF_LINE_ADDITION = Cchar('+') - const DIFF_LINE_DELETION = Cchar('-') - - const DIFF_LINE_CONTEXT_EOFNL = Cchar('=') - const DIFF_LINE_ADD_EOFNL = Cchar('>') - const DIFF_LINE_DEL_EOFNL = Cchar('<') - - const DIFF_LINE_FILE_HDR = Cchar('F') - const DIFF_LINE_HUNK_HDR = Cchar('H') - const DIFF_LINE_BINARY = Cchar('B') - # index const IDXENTRY_NAMEMASK = (0x0fff) const IDXENTRY_STAGEMASK = (0x3000) @@ -147,78 +122,73 @@ module GitConst const IDXENTRY_VALID = (0x8000) const IDXENTRY_STAGESHIFT = Cint(12) - const IDXENTRY_UPDATE = Cint(1) << Cint(0) - const IDXENTRY_REMOVE = Cint(1) << Cint(1) - const IDXENTRY_UPTODATE = Cint(1) << Cint(2) - const IDXENTRY_ADDED = Cint(1) << Cint(3) + const IDXENTRY_UPDATE = Cint(1 << 0) + const IDXENTRY_REMOVE = Cint(1 << 1) + const IDXENTRY_UPTODATE = Cint(1 << 2) + const IDXENTRY_ADDED = Cint(1 << 3) - const IDXENTRY_HASHED = Cint(1) << Cint(4) - const IDXENTRY_UNHASHED = Cint(1) << Cint(5) - const IDXENTRY_WT_REMOVE = Cint(1) << Cint(6) - const IDXENTRY_CONFLICTED = Cint(1) << Cint(7) + const IDXENTRY_HASHED = Cint(1 << 4) + const IDXENTRY_UNHASHED = Cint(1 << 5) + const IDXENTRY_WT_REMOVE = Cint(1 << 6) + const IDXENTRY_CONFLICTED = Cint(1 << 7) - const IDXENTRY_UNPACKED = Cint(1) << Cint(8) - const IDXENTRY_NEW_SKIP_WORKTREE = Cint(1) << Cint(9) + const IDXENTRY_UNPACKED = Cint(1 << 8) + const IDXENTRY_NEW_SKIP_WORKTREE = Cint(1 << 9) const INDEXCAP_IGNORE_CASE = Cuint(1) const INDEXCAP_NO_FILEMODE = Cuint(2) const INDEXCAP_NO_SYMLINKS = Cuint(4) const INDEXCAP_FROM_OWNER = ~(Cuint(0)) - const INDEX_ADD_DEFAULT = Cuint(0) - const INDEX_ADD_FORCE = Cuint(1) << Cint(0) - const INDEX_ADD_DISABLE_PATHSPEC_MATCH = Cuint(1) << Cint(1) - const INDEX_ADD_CHECK_PATHSPEC = Cuint(1) << Cint(2) + const INDEX_ADD_DEFAULT = Cuint(0) + const INDEX_ADD_FORCE = Cuint(1 << 0) + const INDEX_ADD_DISABLE_PATHSPEC_MATCH = Cuint(1 << 1) + const INDEX_ADD_CHECK_PATHSPEC = Cuint(1 << 2) const INDEX_STAGE_ANY = Cint(-1) - const MERGE_TREE_FIND_RENAMES = Cint(1) << Cint(0) + # merge + const MERGE_TREE_FIND_RENAMES = Cint(1 << 0) const MERGE_FILE_FAVOR_NORMAL = Cint(0) const MERGE_FILE_FAVOR_OURS = Cint(1) const MERGE_FILE_FAVOR_THEIRS = Cint(2) const MERGE_FILE_FAVOR_UNION = Cint(3) - const MERGE_AUTOMERGE_NORMAL = Cint(0) - const MERGE_AUTOMERGE_FAVOR_OURS = Cint(1) + const MERGE_AUTOMERGE_NORMAL = Cint(0) + const MERGE_AUTOMERGE_FAVOR_OURS = Cint(1) const MERGE_AUTOMERGE_FAVOR_THEIRS = Cint(2) const MERGE_AUTOMERGE_FAVOR_UNION = Cint(3) - const MERGE_NO_FASTFORWARD = Cint(1) + const MERGE_NO_FASTFORWARD = Cint(1) const MERGE_FASTFORWARD_ONLY = Cint(2) - const MERGE_ANALYSIS_NONE = 0, - const MERGE_ANALYSIS_NORMAL = (1 << 0) - const MERGE_ANALYSIS_UP_TO_DATE = (1 << 1) - const MERGE_ANALYSIS_FASTFORWARD = (1 << 2) - const MERGE_ANALYSIS_UNBORN = (1 << 3) - - const MERGE_PREFERENCE_NONE = 0 - const MERGE_PREFERENCE_NO_FASTFORWARD = (1 << 0) - const MERGE_PREFERENCE_FASTFORWARD_ONLY = (1 << 1) - - const DIRECTION_FETCH = Cint(0) - const DIRECTION_PUSH = Cint(1) - - const BLAME_NORMAL = Cint(0) - - const CREDTYPE_USERPASS_PLAINTEXT = Cuint(1) << Cint(0) - const CREDTYPE_SSH_KEY = Cuint(1) << Cint(1) - const CREDTYPE_SSH_CUSTOM = Cuint(1) << Cint(2) - const CREDTYPE_DEFAULT = Cuint(1) << Cint(3) - - const GIT_REPOSITORY_STATE_NONE = Cint(0) - const GIT_REPOSITORY_STATE_MERGE = Cint(1) - const GIT_REPOSITORY_STATE_REVERT = Cint(2) - const GIT_REPOSITORY_STATE_CHERRY_PICK = Cint(3) - const GIT_REPOSITORY_STATE_BISECT = Cint(4) - const GIT_REPOSITORY_STATE_REBASE = Cint(5) - const GIT_REPOSITORY_STATE_REBASE_INTERACTIVE = Cint(6) - const GIT_REPOSITORY_STATE_REBASE_MERGE = Cint(7) - const GIT_REPOSITORY_STATE_APPLY_MAILBOX = Cint(8) - const GIT_REPOSITORY_STATE_APPLY_MAILBOX_OR_REBASE = Cint(9) + const MERGE_ANALYSIS_NONE = Cint(0) + const MERGE_ANALYSIS_NORMAL = Cint(1 << 0) + const MERGE_ANALYSIS_UP_TO_DATE = Cint(1 << 1) + const MERGE_ANALYSIS_FASTFORWARD = Cint(1 << 2) + const MERGE_ANALYSIS_UNBORN = Cint(1 << 3) + const MERGE_PREFERENCE_NONE = Cint(0) + const MERGE_PREFERENCE_NO_FASTFORWARD = Cint(1 << 0) + const MERGE_PREFERENCE_FASTFORWARD_ONLY = Cint(1 << 1) + + # reset const RESET_SOFT = Cint(1) # Move the head to the given commit const RESET_MIXED = Cint(2) # SOFT plus reset index to the commit const RESET_HARD = Cint(3) # MIXED plus changes in working tree discarded + + #rebase + const REBASE_OPERATION_PICK = Cint(0) + const REBASE_OPERATION_REWORD = Cint(1) + const REBASE_OPERATION_EDIT = Cint(2) + const REBASE_OPERATION_SQUASH = Cint(3) + const REBASE_OPERATION_FIXUP = Cint(4) + const REBASE_OPERATION_EXEC = Cint(5) + + # credentials + const CREDTYPE_USERPASS_PLAINTEXT = Cuint(1 << 0) + const CREDTYPE_SSH_KEY = Cuint(1 << 1) + const CREDTYPE_SSH_CUSTOM = Cuint(1 << 2) + const CREDTYPE_DEFAULT = Cuint(1 << 3) end \ No newline at end of file diff --git a/base/pkg/libgit2/diff.jl b/base/pkg/libgit2/diff.jl new file mode 100644 index 0000000000000..221098567d49c --- /dev/null +++ b/base/pkg/libgit2/diff.jl @@ -0,0 +1,44 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + +function diff_tree(repo::GitRepo, tree::GitTree, pathspecs::AbstractString=""; cached::Bool=false) + emptypathspec = isempty(pathspecs) + diff_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + if !emptypathspec + sa = StrArrayStruct(paths) + diff_opts = DiffOptionsStruct(pathspec = sa) + end + try + if cached + @check ccall((:git_diff_tree_to_index, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, Ptr{DiffOptionsStruct}), + diff_ptr_ptr, repo.ptr, tree.ptr, C_NULL, emptypathspec ? C_NULL : Ref(diff_opts)) + else + @check ccall((:git_diff_tree_to_workdir_with_index, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{DiffOptionsStruct}), + diff_ptr_ptr, repo.ptr, tree.ptr, emptypathspec ? C_NULL : Ref(diff_opts)) + end + finally + !emptypathspec && finalize(sa) + end + return GitDiff(diff_ptr_ptr[]) +end + +function diff_tree(repo::GitRepo, oldtree::GitTree, newtree::GitTree) + diff_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_diff_tree_to_tree, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, Ptr{DiffOptionsStruct}), + diff_ptr_ptr, repo.ptr, oldtree.ptr, newtree.ptr, C_NULL) + return GitDiff(diff_ptr_ptr[]) +end + +function Base.count(diff::GitDiff) + return ccall((:git_diff_num_deltas, :libgit2), Cint, (Ptr{Void},), diff.ptr) +end + +function Base.getindex(diff::GitDiff, i::Csize_t) + delta_ptr = ccall((:git_diff_get_delta, :libgit2), Ptr{Void}, + (Ptr{Void}, Csize_t), diff.ptr, i-1) + delta_ptr == C_NULL && return nothing + return unsafe_load(convert(Ptr{DiffDelta}, delta_ptr), 1) +end +Base.getindex(diff::GitDiff, i::Int) = getindex(diff, Csize_t(i)) diff --git a/base/pkg/libgit2/index.jl b/base/pkg/libgit2/index.jl index e9250b425d522..3bd1f21c86325 100644 --- a/base/pkg/libgit2/index.jl +++ b/base/pkg/libgit2/index.jl @@ -111,7 +111,7 @@ function Base.getindex(idx::GitIndex, i::Csize_t) # return ccall((:git_index_get_byindex, :libgit2), Ptr{Void}, # (Ptr{Void}, Csize_t), idx.ptr, i) ie_ptr = ccall((:git_index_get_byindex, :libgit2), Ptr{Void}, - (Ptr{Void}, Csize_t), idx.ptr, i) + (Ptr{Void}, Csize_t), idx.ptr, i-1) ie_ptr == C_NULL && return nothing return unsafe_load(convert(Ptr{IndexEntry}, ie_ptr), 1) end diff --git a/base/pkg/libgit2/merge.jl b/base/pkg/libgit2/merge.jl index 3186bc35c8ba1..35e09d0997e98 100644 --- a/base/pkg/libgit2/merge.jl +++ b/base/pkg/libgit2/merge.jl @@ -4,7 +4,7 @@ function GitAnnotated(repo::GitRepo, commit_id::Oid) ann_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_annotated_commit_lookup, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}), - ann_ptr_ptr, repo.ptr, commit_id) + ann_ptr_ptr, repo.ptr, Ref(commit_id)) return GitAnnotated(ann_ptr_ptr[]) end @@ -16,6 +16,21 @@ function GitAnnotated(repo::GitRepo, ref::GitReference) return GitAnnotated(ann_ref_ref[]) end +function GitAnnotated(repo::GitRepo, comittish::AbstractString) + obj = revparse(repo, comittish) + try + cmt = peel(obj, GitConst.OBJ_COMMIT) + cmt == nothing && return nothing + return GitAnnotated(repo, Oid(cmt)) + finally + finalize(obj) + end +end + +function commit(ann::GitAnnotated) + return Oid(ccall((:git_annotated_commit_id, :libgit2), Ptr{Oid}, (Ptr{Void},), ann.ptr)) +end + function merge_analysis(repo::GitRepo, ann::GitAnnotated) analysis = Ref{Cint}(0) preference = Ref{Cint}(0) @@ -25,46 +40,13 @@ function merge_analysis(repo::GitRepo, ann::GitAnnotated) return analysis[], preference[] end -function commit(ann::GitAnnotated) - return Oid(ccall((:git_annotated_commit_id, :libgit2), Ptr{Oid}, (Ptr{Void},), ann.ptr)) -end - """ Merge changes into current head """ -function merge!(repo::GitRepo; fast_forward::Bool=false) - # get head annotated upstream reference - with(head(repo)) do hr - with(upstream(hr)) do hur - with(GitAnnotated(repo, hur)) do hua - ma, mp = merge_analysis(repo, hua) - (ma & GitConst.MERGE_ANALYSIS_UP_TO_DATE == GitConst.MERGE_ANALYSIS_UP_TO_DATE) && return - if (ma & GitConst.MERGE_ANALYSIS_FASTFORWARD == GitConst.MERGE_ANALYSIS_FASTFORWARD) - # do fastforward: checkout tree and update branch references - # hur_oid = Oid(hur) - # with(get(GitCommit, repo, hur_oid)) do cmt - # checkout_tree(repo, cmt) - # end - # target!(hr, hur_oid, msg="pkg.libgit2.megre!: fastforward $(name(hur)) into $(name(hr))") - # head!(repo, hur, msg="--fastforward") - - hur_oid = Oid(hur) - target!(hr, hur_oid, msg="pkg.libgit2.megre!: fastforward $(name(hur)) into $(name(hr))") - reset!(repo, hur_oid, GitConst.RESET_HARD) - elseif (ma & GitConst.MERGE_ANALYSIS_NORMAL == GitConst.MERGE_ANALYSIS_NORMAL) - if fast_forward - warn("Fastforward merge is not possible. Abort merging.") - return - end - merge_opts = MergeOptionsStruct() - checkout_opts = CheckoutOptionsStruct(checkout_strategy = GitConst.CHECKOUT_SAFE) - @check ccall((:git_merge, :libgit2), Cint, - (Ptr{Void}, Ptr{Ptr{Void}}, Csize_t, Ptr{MergeOptionsStruct}, Ptr{CheckoutOptionsStruct}), - repo.ptr, Ref{Ptr{Void}}(hua.ptr), 1, Ref(merge_opts), Ref(checkout_opts)) - cleanup(repo) - info("Review and commit merged changes.") - else - warn("Unknown merge analysis result. Merging is not possible.") - end - end - end - end +function merge(repo::GitRepo, their_head::GitAnnotated; + merge_opts = MergeOptionsStruct(), + checkout_opts = CheckoutOptionsStruct(checkout_strategy = GitConst.CHECKOUT_SAFE)) + return @check ccall((:git_merge, :libgit2), Cint, + (Ptr{Void}, Ptr{Ptr{Void}}, Csize_t, + Ptr{MergeOptionsStruct}, Ptr{CheckoutOptionsStruct}), + repo.ptr, Ref{Ptr{Void}}(their_head.ptr), 1, + Ref(merge_opts), Ref(checkout_opts)) end diff --git a/base/pkg/libgit2/oid.jl b/base/pkg/libgit2/oid.jl index 1dc005845370f..c1f7a91667333 100644 --- a/base/pkg/libgit2/oid.jl +++ b/base/pkg/libgit2/oid.jl @@ -57,7 +57,7 @@ function Oid(obj::Ptr{Void}) return Oid(oid_ptr) end -function Oid(obj::GitObject) +function Oid{T<:GitObject}(obj::T) obj == nothing && return Oid() return Oid(obj.ptr) end diff --git a/base/pkg/libgit2/rebase.jl b/base/pkg/libgit2/rebase.jl new file mode 100644 index 0000000000000..9d44cd71466d5 --- /dev/null +++ b/base/pkg/libgit2/rebase.jl @@ -0,0 +1,68 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + +function GitRebase(repo::GitRepo, branch::GitAnnotated, upstream::GitAnnotated; + onto::Nullable{GitAnnotated}=Nullable{GitAnnotated}(), + sig::Nullable{GitSignature}=Nullable{GitSignature}(), + opts::RebaseOptionsStruct = RebaseOptionsStruct()) + rebase_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + sig_obj = isnull(sig) ? default_signature(repo) : Base.get(sig) + try + @check ccall((:git_rebase_init, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, + Ptr{Void}, Ptr{SignatureStruct}, Ptr{RebaseOptionsStruct}), + rebase_ptr_ptr, repo.ptr, branch.ptr, upstream.ptr, + isnull(onto) ? C_NULL : Base.get(onto).ptr, sig_obj.ptr, Ref(opts)) + finally + isnull(sig) && finalize(sig_obj) + end + return GitRebase(rebase_ptr_ptr[]) +end + +function Base.count(rb::GitRebase) + return ccall((:git_rebase_operation_entrycount, :libgit2), Csize_t, (Ptr{Void},), rb.ptr) +end + +function current(rb::GitRebase) + return ccall((:git_rebase_operation_current, :libgit2), Csize_t, (Ptr{Void},), rb.ptr) +end + +function Base.getindex(rb::GitRebase, i::Csize_t) + rb_op_ptr = ccall((:git_rebase_operation_byindex, :libgit2), Ptr{Void}, + (Ptr{Void}, Csize_t), rb.ptr, i-1) + rb_op_ptr == C_NULL && return nothing + return unsafe_load(convert(Ptr{RebaseOperation}, rb_op_ptr), 1) +end +Base.getindex(rb::GitRebase, i::Int) = getindex(rb, Csize_t(i)) + +function Base.next(rb::GitRebase; opts::CheckoutOptionsStruct = CheckoutOptionsStruct()) + rb_op_ptr_ptr = Ref{Ptr{RebaseOperation}}(C_NULL) + try + @check ccall((:git_rebase_next, :libgit2), Cint, + (Ptr{Ptr{RebaseOperation}}, Ptr{Void}, Ptr{CheckoutOptionsStruct}), + rb_op_ptr_ptr, rb.ptr, Ref(opts)) + catch err + err.code == Error.ITEROVER && return nothing + rethrow(err) + end + return unsafe_load(convert(Ptr{RebaseOperation}, rb_op_ptr_ptr[]), 1) +end + +function commit(rb::GitRebase, sig::GitSignature; + opts::CheckoutOptionsStruct = CheckoutOptionsStruct()) + oid_ptr = Ref(Oid()) + @check ccall((:git_rebase_commit, :libgit2), Cint, + (Ptr{Oid}, Ptr{Void}, Ptr{SignatureStruct}, Ptr{SignatureStruct}, Ptr{UInt8}, Ptr{UInt8}), + oid_ptr, rb.ptr, C_NULL, sig.ptr, C_NULL, C_NULL) + return oid_ptr[] +end + +function abort(rb::GitRebase, sig::GitSignature) + return ccall((:git_rebase_abort, :libgit2), Csize_t, + (Ptr{Void}, Ptr{SignatureStruct}), rb.ptr, sig.ptr) +end + +function finish(rb::GitRebase, sig::GitSignature; opts::RebaseOptionsStruct = RebaseOptionsStruct()) + return ccall((:git_rebase_finish, :libgit2), Csize_t, + (Ptr{Void}, Ptr{SignatureStruct}, Ptr{RebaseOptionsStruct}), + rb.ptr, sig.ptr, Ref(opts)) +end diff --git a/base/pkg/libgit2/remote.jl b/base/pkg/libgit2/remote.jl index 194b9a93743b6..fea24c4c4ba04 100644 --- a/base/pkg/libgit2/remote.jl +++ b/base/pkg/libgit2/remote.jl @@ -3,23 +3,23 @@ function GitRemote(repo::GitRepo, rmt_name::AbstractString, rmt_url::AbstractString) rmt_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_remote_create, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}), + (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring), rmt_ptr_ptr, repo.ptr, rmt_name, rmt_url) return GitRemote(rmt_ptr_ptr[]) end -function GitRemoteAnon(repo::GitRepo, url::AbstractString, refspec::AbstractString) +function GitRemoteAnon(repo::GitRepo, url::AbstractString) rmt_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_remote_create_anonymous, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}), - rmt_ptr_ptr, repo.ptr, url, refspec) + (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Ptr{UInt8}), + rmt_ptr_ptr, repo.ptr, url, C_NULL) return GitRemote(rmt_ptr_ptr[]) end function get(::Type{GitRemote}, repo::GitRepo, rmt_name::AbstractString) rmt_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_remote_lookup, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), + (Ptr{Ptr{Void}}, Ptr{Void}, Cstring), rmt_ptr_ptr, repo.ptr, rmt_name) return GitRemote(rmt_ptr_ptr[]) end @@ -34,15 +34,32 @@ function url(rmt::GitRemote) return bytestring(url_ptr) end -function push(rmt::GitRemote, sig::GitSignature, refspecs::AbstractString=""; +function fetch{T<:AbstractString}(rmt::GitRemote, sig::GitSignature, refspecs::Vector{T}; + msg::AbstractString="") + msg = "pkg.libgit2.fetch: $msg" + no_refs = (length(refspecs) == 0) + !no_refs && (sa = StrArrayStruct(refspecs...)) + try + @check ccall((:git_remote_fetch, :libgit2), Cint, + (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{SignatureStruct}, Cstring), + rmt.ptr, no_refs ? C_NULL : Ref(sa), sig.ptr, msg) + finally + !no_refs && finalize(sa) + end +end + +function push{T<:AbstractString}(rmt::GitRemote, sig::GitSignature, refspecs::Vector{T}; + force::Bool=false, msg::AbstractString="") + msg = "pkg.libgit2.push: $msg" push_opts = PushOptionsStruct() - !isempty(refspecs) && (sa = StrArrayStruct(pathspecs...)) + no_refs = (length(refspecs) == 0) + !no_refs && (sa = StrArrayStruct(refspecs...)) try @check ccall((:git_remote_push, :libgit2), Cint, - (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{PushOptionsStruct}, Ptr{Void}, Ptr{UInt8}), - rmt.ptr, !isempty(refspecs) ? Ref(sa) : C_NULL, Ref(push_opts), sig.ptr, isempty(msg) ? C_NULL : msg) + (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{PushOptionsStruct}, Ptr{Void}, Cstring), + rmt.ptr, no_refs ? C_NULL : Ref(sa), Ref(push_opts), sig.ptr, msg) finally - !isempty(refspecs) && finalize(sa) + !no_refs && finalize(sa) end end diff --git a/base/pkg/libgit2/repository.jl b/base/pkg/libgit2/repository.jl index 07d13b8cc0b19..d0f606cebdcbf 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/pkg/libgit2/repository.jl @@ -59,15 +59,15 @@ function revparseid(repo::GitRepo, objname::AbstractString) return oid end -function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::Oid, prefix::Bool=false) +function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::Oid, oid_size::Int=OID_HEXSZ) id_ptr = Ref(oid) obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) git_otype = getobjecttype(T) - err = if prefix + err = if oid_size != OID_HEXSZ ccall((:git_object_lookup_prefix, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}, Csize_t, Cint), - obj_ptr_ptr, r.ptr, id_ptr, len, git_otype) #TODO: length + obj_ptr_ptr, r.ptr, id_ptr, Csize_t(oid_size), git_otype) else ccall((:git_object_lookup, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}, Cint), @@ -85,7 +85,7 @@ function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::Oid, prefix::Bool=false end function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::AbstractString) - return get(T, r, Oid(oid), length(oid) != OID_HEXSZ) + return get(T, r, Oid(oid), length(oid)) end function gitdir(repo::GitRepo) @@ -136,15 +136,6 @@ function checkout_head(repo::GitRepo; options::CheckoutOptionsStruct = CheckoutO repo.ptr, Ref(options)) end -function fetch(repo::GitRepo, rmt::GitRemote; msg::AbstractString="") - msg = "pkg.libgit2.fetch: $msg" - with(default_signature(repo)) do sig - @check ccall((:git_remote_fetch, :libgit2), Cint, - (Ptr{Void}, Ptr{Void}, Ptr{SignatureStruct}, Ptr{UInt8}), - rmt.ptr, C_NULL, sig.ptr, msg) - end -end - function reset!(repo::GitRepo, obj::Nullable{GitAnyObject}, pathspecs::AbstractString...) with(StrArrayStruct(pathspecs...)) do sa @check ccall((:git_reset_default, :libgit2), Cint, diff --git a/base/pkg/libgit2/tag.jl b/base/pkg/libgit2/tag.jl index ec5249e4fc14c..aa47cdf9653fc 100644 --- a/base/pkg/libgit2/tag.jl +++ b/base/pkg/libgit2/tag.jl @@ -27,3 +27,12 @@ function tag_create(repo::GitRepo, tag::AbstractString, commit::AbstractString; end return oid_ptr[] end + +function name(tag::GitTag) + return bytestring(ccall((:git_tag_name, :libgit2), Cstring, (Ptr{Void}, ), tag.ptr)) +end + +function target(tag::GitTag) + oid_ptr = Ref(ccall((:git_tag_target_id, :libgit2), Ptr{Oid}, (Ptr{Void}, ), tag.ptr)) + return oid_ptr[] +end diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index 8a3227a68d492..2058292bfb233 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -206,6 +206,25 @@ DiffOptionsStruct(; flags::UInt32 = GitConst.DIFF_NORMAL, new_prefix ) +immutable DiffFile + id::Oid + path::Cstring + size::Coff_t + flags::Cuint + mode::UInt16 +end +DiffFile()=DiffFile(Oid(),convert(Cstring, Ptr{UInt8}(C_NULL)),Coff_t(0),Cuint(0),UInt16(0)) + +immutable DiffDelta + status::Cint + flags::Cuint + similarity::UInt16 + nfiles::UInt16 + old_file::DiffFile + new_file::DiffFile +end +DiffDelta()=DiffDelta(Cint(0),Cuint(0),UInt16(0),UInt16(0),DiffFile(),DiffFile()) + immutable MergeOptionsStruct version::Cuint flags::Cint @@ -215,16 +234,16 @@ immutable MergeOptionsStruct file_favor::Cint end MergeOptionsStruct(; flags::Cint = Cint(0), - rename_threshold::Cuint = Cuint(50), - target_limit::Cuint = Cuint(200), - metric::Ptr{Void} = Ptr{Void}(0), - file_favor::Cint = GitConst.MERGE_FILE_FAVOR_NORMAL + rename_threshold::Cuint = Cuint(50), + target_limit::Cuint = Cuint(200), + metric::Ptr{Void} = Ptr{Void}(0), + file_favor::Cint = GitConst.MERGE_FILE_FAVOR_NORMAL )=MergeOptionsStruct(one(Cuint), - flags, - rename_threshold, - target_limit, - metric, - file_favor + flags, + rename_threshold, + target_limit, + metric, + file_favor ) immutable PushOptionsStruct @@ -248,7 +267,7 @@ immutable IndexEntry mode::Cuint uid::Cuint gid::Cuint - file_size::Int64 + file_size::Coff_t id::Oid @@ -264,13 +283,27 @@ IndexEntry() = IndexEntry(IndexTime(), Cuint(0), Cuint(0), Cuint(0), - Cuint(0), + Coff_t(0), Oid(), UInt16(0), UInt16(0), Ptr{UInt8}(0)) Base.show(io::IO, ie::IndexEntry) = print(io, "IndexEntry($(string(ie.id)))") +immutable RebaseOptionsStruct + version::Cuint + quiet::Cint + rewrite_notes_ref::Cstring +end +RebaseOptionsStruct()=RebaseOptionsStruct(one(Cuint), Cint(1), convert(Cstring, Ptr{UInt8}(C_NULL))) + +immutable RebaseOperation + optype::Cint + id::Oid + exec::Cstring +end +RebaseOperation()=RebaseOperation(Cint(0),Oid(),convert(Cstring, Ptr{UInt8}(C_NULL))) +Base.show(io::IO, rbo::RebaseOperation) = print(io, "RebaseOperation($(string(rbo.id)))") # Abstract object types abstract AbstractGitObject @@ -294,11 +327,13 @@ for (typ, ref, sup, fnc) in ( (:GitIndex, :Void, :AbstractGitObject, :(:git_index_free)), (:GitRepo, :Void, :AbstractGitObject, :(:git_repository_free)), (:GitAnnotated, :Void, :AbstractGitObject, :(:git_annotated_commit_free)), + (:GitRebase, :Void, :AbstractGitObject, :(:git_rebase_free)), (:GitSignature, :SignatureStruct, :AbstractGitObject, :(:git_signature_free)), (:GitAnyObject, :Void, :GitObject, nothing), (:GitCommit, :Void, :GitObject, nothing), (:GitBlob, :Void, :GitObject, nothing), - (:GitTree, :Void, :GitObject, nothing) + (:GitTree, :Void, :GitObject, nothing), + (:GitTag, :Void, :GitObject, nothing) ) @eval type $typ <: $sup @@ -358,6 +393,8 @@ function getobjecttype{T<:GitObject}(::Type{T}) GitConst.OBJ_TREE elseif T == GitBlob GitConst.OBJ_BLOB + elseif T == GitTag + GitConst.OBJ_TAG elseif T == GitAnyObject GitConst.OBJ_ANY else @@ -372,6 +409,8 @@ function getobjecttype(obj_type::Cint) GitTree elseif obj_type == GitConst.OBJ_BLOB GitBlob + elseif obj_type == GitConst.OBJ_TAG + GitTag elseif obj_type == GitConst.OBJ_ANY GitAnyObject else diff --git a/base/pkg/write.jl b/base/pkg/write.jl index fde650cffcaeb..f8d3d13c2cd0a 100644 --- a/base/pkg/write.jl +++ b/base/pkg/write.jl @@ -12,7 +12,7 @@ end function fetch(repo::GitRepo, pkg::AbstractString, sha1::AbstractString) cache = Cache.path(pkg) - LibGit2.fetch(repo, cache, refspecs = "+refs/*:refs/remotes/cache/*") + LibGit2.fetch(repo, remoteurl=cache, refspecs=["+refs/*:refs/remotes/cache/*"]) LibGit2.need_update(repo) LibGit2.iscommit(sha1, repo) && return f = with(GitRepo, cache) do repo diff --git a/test/libgit2.jl b/test/libgit2.jl index e837075e219a7..dddb4429c5391 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -110,7 +110,7 @@ temp_dir() do dir_cache Pkg.LibGit2.fetch(repo) refs1 = parse(Int, readchomp(pipe(`find $(joinpath(path, ".git/refs"))`,`wc -l`))) - Pkg.LibGit2.fetch(repo, path_cache, refspecs = "+refs/*:refs/remotes/cache/*") + Pkg.LibGit2.fetch(repo, remoteurl=path_cache, refspecs =["+refs/*:refs/remotes/cache/*"]) refs2 = parse(Int, readchomp(pipe(`find $(joinpath(path, ".git/refs"))`,`wc -l`))) finalize(repo) From 307aa8bd8efac6d934e0fa39e3598a97fae36042 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Fri, 5 Jun 2015 22:22:49 -0400 Subject: [PATCH 0328/1938] partial fix for #11580 --- base/pkg/entry.jl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 03b4dd3cf782a..307621a7b8223 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -133,8 +133,12 @@ function status(io::IO; pkgname::AbstractString = "") if !isempty(required) showpkg("") && println(io, "$(length(required)) required packages:") for pkg in required - ver,fix = pop!(instd,pkg) - showpkg(pkg) && status(io,pkg,ver,fix) + if !haskey(instd, pkg) + showpkg(pkg) && status(io,pkg,"not found") + else + ver,fix = pop!(instd,pkg) + showpkg(pkg) && status(io,pkg,ver,fix) + end end end additional = sort!(collect(keys(instd))) @@ -181,6 +185,12 @@ function status(io::IO, pkg::AbstractString, ver::VersionNumber, fix::Bool) println(io) end +function status(io::IO, pkg::AbstractString, msg::AbstractString) + @printf io " - %-29s " pkg + @printf io "%-19s" msg + println(io) +end + function clone(url::AbstractString, pkg::AbstractString) info("Cloning $pkg from $url") ispath(pkg) && throw(PkgError("$pkg already exists")) From 70963a91d683dcc317345ccdff45d281a0293ba9 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sat, 27 Jun 2015 19:53:46 -0400 Subject: [PATCH 0329/1938] adopting CString & refactoring --- base/pkg/libgit2.jl | 8 ++--- base/pkg/libgit2/config.jl | 58 ++++++++++++++++++----------------- base/pkg/libgit2/const.jl | 2 +- base/pkg/libgit2/signature.jl | 4 +-- base/pkg/libgit2/strarray.jl | 22 ++++++++++--- base/pkg/libgit2/types.jl | 11 +++++++ 6 files changed, 66 insertions(+), 39 deletions(-) diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 5d180a87c570c..1be848730a9e0 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -154,11 +154,11 @@ function set_remote_url(path::AbstractString, url::AbstractString; remote::Abstr end end -function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, name::Ptr{UInt8}, url::Ptr{UInt8}, payload::Ptr{Void}) +function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, name::Cstring, url::Cstring, payload::Ptr{Void}) # Create the remote with a mirroring url fetch_spec = "+refs/*:refs/*" err = ccall((:git_remote_create_with_fetchspec, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}, Ptr{UInt8}), + (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring, Cstring), remote, repo_ptr, name, url, fetch_spec) err != 0 && return Cint(err) @@ -172,7 +172,7 @@ function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, name::Ptr{ err != 0 && return Cint(err) return Cint(0) end -const mirror_cb = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}, Ptr{Void})) +const mirror_cb = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring, Ptr{Void})) """ git fetch [|] []""" function fetch{T<:AbstractString}(repo::GitRepo; @@ -340,7 +340,7 @@ function clone(url::AbstractString, path::AbstractString; clone_opts_ref = Ref(clone_opts) repo_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_clone, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{UInt8}, Ptr{UInt8}, Ref{CloneOptionsStruct}), + (Ptr{Ptr{Void}}, Cstring, Cstring, Ref{CloneOptionsStruct}), repo_ptr_ptr, url, path, clone_opts_ref) return GitRepo(repo_ptr_ptr[]) end diff --git a/base/pkg/libgit2/config.jl b/base/pkg/libgit2/config.jl index 70170de01c128..6bc77f39ee9e8 100644 --- a/base/pkg/libgit2/config.jl +++ b/base/pkg/libgit2/config.jl @@ -23,30 +23,32 @@ function GitConfig() return GitConfig(cfg_ptr_ptr[]) end -function get{T}(::Type{T}, c::GitConfig, name::AbstractString) - if T<:AbstractString - str_ptr = Ref{Ptr{UInt8}}(C_NULL) - @check ccall((:git_config_get_string, :libgit2), Cint, #TODO: git_config_get_string_buf - (Ptr{Ptr{UInt8}}, Ptr{Void}, Ptr{UInt8}), str_ptr, c.ptr, name) - return bytestring(str_ptr[]) - elseif is(T, Bool) - val_ptr = Ref(Cint(0)) - @check ccall((:git_config_get_bool, :libgit2), Cint, - (Ptr{Cint}, Ptr{Void}, Ptr{UInt8}), val_ptr, c.ptr, name) - return Bool(val_ptr[]) - elseif is(T, Int32) - val_ptr = Ref(Cint(0)) - @check ccall((:git_config_get_bool, :libgit2), Cint, - (Ptr{Cint}, Ptr{Void}, Ptr{UInt8}), val_ptr, c.ptr, name) - return val_ptr[] - elseif is(T, Int64) - val_ptr = Ref(Cintmax_t(0)) - @check ccall((:git_config_get_bool, :libgit2), Cint, - (Ptr{Cintmax_t}, Ptr{Void}, Ptr{UInt8}), val_ptr, c.ptr, name) - return val_ptr[] - else - return nothing - end +function get{T<:AbstractString}(::Type{T}, c::GitConfig, name::AbstractString) + str_ptr = Ref{Ptr{UInt8}}(C_NULL) + @check ccall((:git_config_get_string, :libgit2), Cint, #TODO: git_config_get_string_buf + (Ptr{Ptr{UInt8}}, Ptr{Void}, Cstring), str_ptr, c.ptr, name) + return bytestring(str_ptr[]) +end + +function get(::Type{Bool}, c::GitConfig, name::AbstractString) + val_ptr = Ref(Cint(0)) + @check ccall((:git_config_get_bool, :libgit2), Cint, + (Ptr{Cint}, Ptr{Void}, Cstring), val_ptr, c.ptr, name) + return Bool(val_ptr[]) +end + +function get(::Type{Int32}, c::GitConfig, name::AbstractString) + val_ptr = Ref(Cint(0)) + @check ccall((:git_config_get_int32, :libgit2), Cint, + (Ptr{Cint}, Ptr{Void}, Cstring), val_ptr, c.ptr, name) + return val_ptr[] +end + +function get(::Type{Int64}, c::GitConfig, name::AbstractString) + val_ptr = Ref(Cintmax_t(0)) + @check ccall((:git_config_get_int64, :libgit2), Cint, + (Ptr{Cint}, Ptr{Void}, Cstring), val_ptr, c.ptr, name) + return val_ptr[] end function get{T}(c::GitConfig, name::AbstractString, default::T) @@ -77,21 +79,21 @@ end function set!{T <: AbstractString}(c::GitConfig, name::AbstractString, value::T) @check ccall((:git_config_set_string, :libgit2), Cint, - (Ptr{Void}, Ptr{UInt8}, Ptr{UInt8}), c.ptr, name, value) + (Ptr{Void}, Cstring, Cstring), c.ptr, name, value) end function set!(c::GitConfig, name::AbstractString, value::Bool) bval = Int32(value) @check ccall((:git_config_set_bool, :libgit2), Cint, - (Ptr{Void}, Ptr{UInt8}, Cint), c.ptr, name, bval) + (Ptr{Void}, Cstring, Cint), c.ptr, name, bval) end function set!(c::GitConfig, name::AbstractString, value::Int32) @check ccall((:git_config_set_int32, :libgit2), Cint, - (Ptr{Void}, Ptr{UInt8}, Cint), c.ptr, name, value) + (Ptr{Void}, Cstring, Cint), c.ptr, name, value) end function set!(c::GitConfig, name::AbstractString, value::Int64) @check ccall((:git_config_set_int64, :libgit2), Cint, - (Ptr{Void}, Ptr{UInt8}, Cintmax_t), c.ptr, name, value) + (Ptr{Void}, Cstring, Cintmax_t), c.ptr, name, value) end diff --git a/base/pkg/libgit2/const.jl b/base/pkg/libgit2/const.jl index 44bb89b4d46e8..bfe047ee46387 100644 --- a/base/pkg/libgit2/const.jl +++ b/base/pkg/libgit2/const.jl @@ -138,7 +138,7 @@ module GitConst const INDEXCAP_IGNORE_CASE = Cuint(1) const INDEXCAP_NO_FILEMODE = Cuint(2) const INDEXCAP_NO_SYMLINKS = Cuint(4) - const INDEXCAP_FROM_OWNER = ~(Cuint(0)) + const INDEXCAP_FROM_OWNER = ~Cuint(0) const INDEX_ADD_DEFAULT = Cuint(0) const INDEX_ADD_FORCE = Cuint(1 << 0) diff --git a/base/pkg/libgit2/signature.jl b/base/pkg/libgit2/signature.jl index de57d7640d3f4..cbec2e771736e 100644 --- a/base/pkg/libgit2/signature.jl +++ b/base/pkg/libgit2/signature.jl @@ -13,7 +13,7 @@ Signature(sig::GitSignature) = Signature(sig.ptr) function Signature(name::AbstractString, email::AbstractString) sig_ptr_ptr = Ref{Ptr{SignatureStruct}}(C_NULL) @check ccall((:git_signature_now, :libgit2), Cint, - (Ptr{Ptr{SignatureStruct}}, Ptr{UInt8}, Ptr{UInt8}), sig_ptr_ptr, name, email) + (Ptr{Ptr{SignatureStruct}}, Cstring, Cstring), sig_ptr_ptr, name, email) sig = GitSignature(sig_ptr_ptr[]) s = Signature(sig.ptr) finalize(sig) @@ -30,7 +30,7 @@ end function Base.convert(::Type{GitSignature}, sig::Signature) sig_ptr_ptr = Ref{Ptr{SignatureStruct}}(C_NULL) @check ccall((:git_signature_new, :libgit2), Cint, - (Ptr{Ptr{SignatureStruct}}, Ptr{Uint8}, Ptr{Uint8}, Cint, Cint), + (Ptr{Ptr{SignatureStruct}}, Cstring, Cstring, Cint, Cint), sig_ptr_ptr, sig.name, sig.email, sig.time, sig.time_offset) return GitSignature(sig_ptr_ptr[]) end diff --git a/base/pkg/libgit2/strarray.jl b/base/pkg/libgit2/strarray.jl index 22f4b108672d1..c1d7f6b901fb7 100644 --- a/base/pkg/libgit2/strarray.jl +++ b/base/pkg/libgit2/strarray.jl @@ -2,20 +2,34 @@ function StrArrayStruct{T<:AbstractString}(strs::T...) count = length(strs) - strings = convert(Ptr{Ptr{Uint8}}, Libc.malloc(sizeof(Ptr{Uint8}) * count)) + strings = convert(Ptr{Ptr{UInt8}}, Libc.malloc(sizeof(Ptr{UInt8}) * count)) for i=1:count len = length(strs[i]) #in_ptr = convert(Ptr{Uint8}, bytestring(strs[i])) - in_ptr = pointer(bytestring(strs[i])) - out_ptr = convert(Ptr{Uint8}, Libc.malloc(sizeof(Uint8) * (len + 1))) + in_ptr = unsafe_convert(Cstring, strs[i]) + out_ptr = convert(Ptr{UInt8}, Libc.malloc(sizeof(UInt8) * (len + 1))) unsafe_copy!(out_ptr, in_ptr, len) - unsafe_store!(out_ptr, zero(Uint8), len + 1) # NULL byte + unsafe_store!(out_ptr, zero(UInt8), len + 1) # NULL byte unsafe_store!(strings, out_ptr, i) end return StrArrayStruct(strings, count) end StrArrayStruct{T<:AbstractString}(strs::Vector{T}) = StrArrayStruct(strs...) +function StrArrayStruct2{T<:AbstractString}(strs::T...) + count = length(strs) + strings = convert(Ptr{Cstring}, Libc.malloc(sizeof(Ptr{UInt8}) * count)) + for i=1:count + len = length(strs[i]) + in_ptr = unsafe_convert(Cstring, strs[i]) + out_ptr = convert(Cstring, Libc.malloc(sizeof(UInt8) * (len + 1))) + unsafe_copy!(out_ptr, in_ptr, len) + unsafe_store!(out_ptr, zero(UInt8), len + 1) # NULL byte + unsafe_store!(strings, out_ptr, i) + end + return StrArrayStruct(strings, count) +end + function Base.convert(::Type{Vector{AbstractString}}, sa::StrArrayStruct) arr = Array(AbstractString, sa.count) for i=1:sa.count diff --git a/base/pkg/libgit2/types.jl b/base/pkg/libgit2/types.jl index 2058292bfb233..3815115d41093 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/pkg/libgit2/types.jl @@ -48,6 +48,17 @@ function Base.finalize(sa::StrArrayStruct) return sa_ptr[] end +immutable StrArrayStruct2 + strings::Ptr{Cstring} + count::Csize_t +end +StrArrayStruct2() = StrArrayStruct2(Ptr{Cstring}(C_NULL), zero(Csize_t)) +function Base.finalize(sa::StrArrayStruct2) + sa_ptr = Ref(sa) + ccall((:git_strarray_free, :libgit2), Void, (Ptr{StrArrayStruct2},), sa_ptr) + return sa_ptr[] +end + immutable CheckoutOptionsStruct version::Cuint From f8b0dec0d6afee8ff3295cc1dd82dd419a751a41 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Sun, 28 Jun 2015 00:23:46 -0400 Subject: [PATCH 0330/1938] added prototype of git repl --- base/REPL.jl | 30 +++++++++++++++++++++++++++--- base/pkg/libgit2.jl | 4 +++- base/pkg/libgit2/walker.jl | 6 +++++- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/base/REPL.jl b/base/REPL.jl index af165fec45532..c77d1723cbbf4 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -734,13 +734,26 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep Expr(:call, :(Base.repl_cmd), macroexpand(Expr(:macrocall, symbol("@cmd"),line)), outstream(repl)) end) + # Set up shell mode + git_mode = Prompt("git> "; + prompt_prefix = hascolor ? repl.shell_color : "", + prompt_suffix = hascolor ? + (repl.envcolors ? Base.input_color : repl.input_color) : "", + keymap_func_data = repl, + complete = ShellCompletionProvider(repl), + on_done = respond(repl, julia_prompt) do line + line = strip(line) + :(Pkg.LibGit2.repl($line)) + end) + ################################# Stage II ############################# # Setup history # We will have a unified history for all REPL modes hp = REPLHistoryProvider(Dict{Symbol,Any}(:julia => julia_prompt, :shell => shell_mode, - :help => help_mode)) + :help => help_mode, + :git => git_mode)) if repl.history_file try f = open(find_hist_file(), true, true, true, false, false) @@ -757,6 +770,7 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep julia_prompt.hist = hp shell_mode.hist = hp help_mode.hist = hp + git_mode.hist = hp search_prompt, skeymap = LineEdit.setup_search_keymap(hp) search_prompt.complete = LatexCompletions() @@ -787,6 +801,16 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep edit_insert(s, '?') end end, + '>' => function (s,o...) + if isempty(s) || position(LineEdit.buffer(s)) == 0 + buf = copy(LineEdit.buffer(s)) + transition(s, git_mode) + LineEdit.state(s, git_mode).input_buffer = buf + LineEdit.refresh_line(s) + else + edit_insert(s, '>') + end + end, # Bracketed Paste Mode "\e[200~" => (s,o...)->begin @@ -845,9 +869,9 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep b = Dict{Any,Any}[skeymap, mk, prefix_keymap, LineEdit.history_keymap, LineEdit.default_keymap, LineEdit.escape_defaults] prepend!(b, extra_repl_keymap) - shell_mode.keymap_dict = help_mode.keymap_dict = LineEdit.keymap(b) + shell_mode.keymap_dict = help_mode.keymap_dict = git_mode.keymap_dict = LineEdit.keymap(b) - ModalInterface([julia_prompt, shell_mode, help_mode, search_prompt, prefix_prompt]) + ModalInterface([julia_prompt, shell_mode, help_mode, git_mode, search_prompt, prefix_prompt]) end function run_frontend(repl::LineEditREPL, backend) diff --git a/base/pkg/libgit2.jl b/base/pkg/libgit2.jl index 1be848730a9e0..33da4b1fbafb2 100644 --- a/base/pkg/libgit2.jl +++ b/base/pkg/libgit2.jl @@ -28,6 +28,8 @@ include("libgit2/tag.jl") include("libgit2/blob.jl") include("libgit2/diff.jl") include("libgit2/rebase.jl") +include("libgit2/repl.jl") + immutable State head::Oid @@ -471,7 +473,7 @@ end """ Returns all commit authors """ function authors(repo::GitRepo) athrs = map( - (oid,repo)->author(get(GitCommit, repo, oid))::Signature, + (oid,repo)->author(get(GitCommit, repo, oid))::Signature, #TODO: cleanup repo) #, by = GitConst.SORT_TIME) return athrs end diff --git a/base/pkg/libgit2/walker.jl b/base/pkg/libgit2/walker.jl index 327a68e704a7e..364e8cdbd82bc 100644 --- a/base/pkg/libgit2/walker.jl +++ b/base/pkg/libgit2/walker.jl @@ -50,7 +50,8 @@ function Base.map(f::Function, repo::GitRepo; oid::Oid=Oid(), range::AbstractString="", by::Cint = GitConst.SORT_NONE, - rev::Bool=false) + rev::Bool=false, + count::Int=0) walker = GitRevWalker(repo) res = nothing try @@ -64,6 +65,7 @@ function Base.map(f::Function, repo::GitRepo; end s = start(walker) + c = 0 while !done(walker, s) val = f(s[1], repo) if res == nothing @@ -71,6 +73,8 @@ function Base.map(f::Function, repo::GitRepo; end push!(res, val) val, s = next(walker, s) + c +=1 + count == c && break end finally finalize(walker) From c0d59a0dd454a60211daa6fc429472f13a71da2f Mon Sep 17 00:00:00 2001 From: Art Wild Date: Wed, 1 Jul 2015 20:49:14 -0400 Subject: [PATCH 0331/1938] added `clone` & `init` command --- base/REPL.jl | 4 +- base/pkg/libgit2/repl.jl | 130 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 base/pkg/libgit2/repl.jl diff --git a/base/REPL.jl b/base/REPL.jl index c77d1723cbbf4..be81e66f86c29 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -734,7 +734,7 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep Expr(:call, :(Base.repl_cmd), macroexpand(Expr(:macrocall, symbol("@cmd"),line)), outstream(repl)) end) - # Set up shell mode + # Set up git mode git_mode = Prompt("git> "; prompt_prefix = hascolor ? repl.shell_color : "", prompt_suffix = hascolor ? @@ -743,7 +743,7 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep complete = ShellCompletionProvider(repl), on_done = respond(repl, julia_prompt) do line line = strip(line) - :(Pkg.LibGit2.repl($line)) + :(Pkg.LibGit2.repl_cmd($line)) end) ################################# Stage II ############################# diff --git a/base/pkg/libgit2/repl.jl b/base/pkg/libgit2/repl.jl new file mode 100644 index 0000000000000..eebe1dbd833f1 --- /dev/null +++ b/base/pkg/libgit2/repl.jl @@ -0,0 +1,130 @@ +""" +# Process REPL command in git mode + +List of git commands: +- [/] add +- [ ] branch +- [x] clone +- [ ] commit +- [ ] fetch +- [x] init +- [x] log +- [ ] reset +- [ ] rm +- [/] status +- [ ] tag +""" + +function repl_cmd(ex) + cmd = split(ex) + has_params = length(cmd) > 1 + repopath = pwd() + + if cmd[1] == "init" + if has_params && cmd[2] == "help" + repl_init_help() + return + end + try + repo = init(repopath, has_params ? Cuint(cmd[2] == "--bare") : Cuint(0)) + finalize(repo) + println("Initialized empty Git repository in $repopath") + catch ex + warn(ex) + end + elseif cmd[1] == "clone" + if has_params && cmd[2] == "help" + repl_clone_help() + return + end + try + repourl = cmd[2] + repo = clone(repourl, repopath, bare=("--bare" in cmd), ) + finalize(repo) + println("Cloned $repourl into $repopath") + catch ex + repl_clone_help() + warn(ex) + end + else + repo = GitRepo(repopath) + try + if cmd[1] == "help" + repl_help() + elseif cmd[1] == "log" + msg_count = 10 + if has_params + msg_count = Base.get(tryparse(Int, cmd[2]), 0) + if msg_count == 0 || cmd[2] == "help" + println("usage: log []") + return + end + end + repl_log(repo, msg_count) + elseif cmd[1] == "status" + if has_params && cmd[2] == "help" + repl_status_help() + return + end + elseif cmd[1] == "add" + if has_params && cmd[2] == "help" + repl_add_help() + return + end + else + warn("unknown command: $ex. Use \'help\' command.") + end + finally + finalize(repo) + end + end +end + +function repl_help() + println("""List of git commands: +add\t Add file contents to the index +clone\t Clone a repository into a current directory +init\t Create an empty Git repository or reinitialize an existing one in current directory +log\t Show commit logs +status\t Show the working tree status + +For particular command parameters use: help +""") +# branch\t\tList, create, or delete branches +# commit\t\tRecord changes to the repository +# fetch\t\tDownload objects and refs from another repository +# reset\t\tReset current HEAD to the specified state +# rm\t\tRemove files from the working tree and from the index +# tag\t\tCreate, list, delete or verify a tag object signed with GPG +end + +function repl_init_help() + println("usage: init [--bare]") +end + +function repl_clone_help() + println("usage: clone [--bare] ") +end + +function repl_add_help() + println("usage: add []") +end + +function repl_status_help() + println("usage: status") +end + +function repl_log(repo::GitRepo, msg_count::Int) + msgs = map( + (oid,r)->with(get(GitCommit, r, oid)) do cmt + sig = author(cmt) + msg = message(cmt) + (Oid(cmt), sig, msg) + end, repo, count = msg_count) + for msg in msgs + print_with_color(:yellow, "Commit: $(string(msg[1]))\n") + println("Author:\t$(msg[2].name) $(msg[2].email)") + println("Date:\t$(Dates.unix2datetime(msg[2].time))") + println('\t',join(split(msg[3],'\n'),"\n\t")) + end +end \ No newline at end of file From 09406ada0f927576ba655a6d1e0b22d3d4ac3b73 Mon Sep 17 00:00:00 2001 From: Art Wild Date: Wed, 8 Jul 2015 00:06:42 -0400 Subject: [PATCH 0332/1938] refactoring commit message call --- base/pkg/libgit2/commit.jl | 4 ++-- base/pkg/libgit2/repl.jl | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/base/pkg/libgit2/commit.jl b/base/pkg/libgit2/commit.jl index 1a740b1670ac0..2b781855c3b33 100644 --- a/base/pkg/libgit2/commit.jl +++ b/base/pkg/libgit2/commit.jl @@ -2,8 +2,8 @@ function message(c::GitCommit, raw::Bool=false) local msg_ptr::Ptr{UInt8} - msg_ptr = raw? ccall((:git_commit_message_raw, :libgit2), Ptr{UInt8}, (Ptr{Void},), c.ptr) : - ccall((:git_commit_message, :libgit2), Ptr{UInt8}, (Ptr{Void},), c.ptr) + msg_ptr = raw? ccall((:git_commit_message_raw, :libgit2), CString, (Ptr{Void},), c.ptr) : + ccall((:git_commit_message, :libgit2), CString, (Ptr{Void},), c.ptr) if msg_ptr == C_NULL return nothing end diff --git a/base/pkg/libgit2/repl.jl b/base/pkg/libgit2/repl.jl index eebe1dbd833f1..5731183064147 100644 --- a/base/pkg/libgit2/repl.jl +++ b/base/pkg/libgit2/repl.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + """ # Process REPL command in git mode From 50b81f6fb91b9a92d056aa6dc30d021f8c85483c Mon Sep 17 00:00:00 2001 From: Test User Date: Wed, 8 Jul 2015 23:32:21 -0400 Subject: [PATCH 0333/1938] rebased, moved `libgit2` to `base`, updated to 0.23 & tests --- base/REPL.jl | 2 +- base/exports.jl | 1 + base/{pkg => }/libgit2.jl | 77 +++------ base/{pkg => }/libgit2/blob.jl | 0 base/{pkg => }/libgit2/commit.jl | 8 +- base/{pkg => }/libgit2/config.jl | 2 +- base/{pkg => }/libgit2/const.jl | 24 ++- base/{pkg => }/libgit2/diff.jl | 0 base/{pkg => }/libgit2/error.jl | 4 +- base/{pkg => }/libgit2/index.jl | 0 base/{pkg => }/libgit2/merge.jl | 22 ++- base/{pkg => }/libgit2/oid.jl | 0 base/{pkg => }/libgit2/rebase.jl | 17 +- base/{pkg => }/libgit2/reference.jl | 8 +- base/{pkg => }/libgit2/remote.jl | 32 ++-- base/{pkg => }/libgit2/repl.jl | 2 +- base/{pkg => }/libgit2/repository.jl | 51 +++--- base/{pkg => }/libgit2/signature.jl | 0 base/libgit2/strarray.jl | 33 ++++ base/{pkg => }/libgit2/tag.jl | 0 base/{pkg => }/libgit2/types.jl | 226 ++++++++++++++++----------- base/libgit2/utils.jl | 6 + base/{pkg => }/libgit2/walker.jl | 4 +- base/methodshow.jl | 66 ++++---- base/pkg.jl | 4 +- base/pkg/cache.jl | 4 +- base/pkg/dir.jl | 2 +- base/pkg/entry.jl | 11 +- base/pkg/generate.jl | 4 +- base/pkg/github.jl | 10 +- base/pkg/libgit2/strarray.jl | 39 ----- base/pkg/read.jl | 6 +- base/pkg/write.jl | 4 +- base/sysimg.jl | 4 +- test/libgit2.jl | 182 +++++++++++---------- 35 files changed, 456 insertions(+), 399 deletions(-) rename base/{pkg => }/libgit2.jl (88%) rename base/{pkg => }/libgit2/blob.jl (100%) rename base/{pkg => }/libgit2/commit.jl (93%) rename base/{pkg => }/libgit2/config.jl (98%) rename base/{pkg => }/libgit2/const.jl (89%) rename base/{pkg => }/libgit2/diff.jl (100%) rename base/{pkg => }/libgit2/error.jl (97%) rename base/{pkg => }/libgit2/index.jl (100%) rename base/{pkg => }/libgit2/merge.jl (75%) rename base/{pkg => }/libgit2/oid.jl (100%) rename base/{pkg => }/libgit2/rebase.jl (85%) rename base/{pkg => }/libgit2/reference.jl (97%) rename base/{pkg => }/libgit2/remote.jl (65%) rename base/{pkg => }/libgit2/repl.jl (98%) rename base/{pkg => }/libgit2/repository.jl (73%) rename base/{pkg => }/libgit2/signature.jl (100%) create mode 100644 base/libgit2/strarray.jl rename base/{pkg => }/libgit2/tag.jl (100%) rename base/{pkg => }/libgit2/types.jl (66%) create mode 100644 base/libgit2/utils.jl rename base/{pkg => }/libgit2/walker.jl (96%) delete mode 100644 base/pkg/libgit2/strarray.jl diff --git a/base/REPL.jl b/base/REPL.jl index be81e66f86c29..95b5bfc7ddf86 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -743,7 +743,7 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep complete = ShellCompletionProvider(repl), on_done = respond(repl, julia_prompt) do line line = strip(line) - :(Pkg.LibGit2.repl_cmd($line)) + :(LibGit2.repl_cmd($line)) end) ################################# Stage II ############################# diff --git a/base/exports.jl b/base/exports.jl index cfc1574f57046..5a36c596c25dc 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -7,6 +7,7 @@ export Meta, Operators, Pkg, + LibGit2, Profile, Dates, Sys, diff --git a/base/pkg/libgit2.jl b/base/libgit2.jl similarity index 88% rename from base/pkg/libgit2.jl rename to base/libgit2.jl index 33da4b1fbafb2..2898f53066c74 100644 --- a/base/pkg/libgit2.jl +++ b/base/libgit2.jl @@ -2,13 +2,9 @@ module LibGit2 -import ...Pkg.PkgError +import Base: merge!, cat -export with, with_warn -export GitRepo, GitConfig, GitIndex - -const GITHUB_REGEX = - r"^(?:git@|git://|https://(?:[\w\.\+\-]+@)?)github.com[:/](([^/].+)/(.+?))(?:\.git)?$"i +export with, GitRepo, GitConfig include("libgit2/const.jl") include("libgit2/types.jl") @@ -29,7 +25,7 @@ include("libgit2/blob.jl") include("libgit2/diff.jl") include("libgit2/rebase.jl") include("libgit2/repl.jl") - +include("libgit2/utils.jl") immutable State head::Oid @@ -37,11 +33,6 @@ immutable State work::Oid end -function normalize_url(url::AbstractString) - m = match(GITHUB_REGEX,url) - m == nothing ? url : "https://github.com/$(m.captures[1]).git" -end - """Return HEAD Oid as string""" function head(pkg::AbstractString) with(GitRepo, pkg) do repo @@ -115,22 +106,6 @@ function diff_files(repo::GitRepo, branch1::AbstractString, branch2::AbstractStr return files end -function merge_base(one::AbstractString, two::AbstractString, repo::GitRepo) - oid1_ptr = Ref(Oid(one)) - oid2_ptr = Ref(Oid(two)) - moid_ptr = Ref(Oid()) - moid = try - @check ccall((:git_merge_base, :libgit2), Cint, - (Ptr{Oid}, Ptr{Void}, Ptr{Oid}, Ptr{Oid}), - moid_ptr, repo.ptr, oid1_ptr, oid2_ptr) - moid_ptr[] - catch e - #warn("Pkg:",path(repo),"=>",e.msg) - Oid() - end - return moid -end - function is_ancestor_of(a::AbstractString, b::AbstractString, repo::GitRepo) A = revparseid(repo, a) merge_base(a, b, repo) == A @@ -187,9 +162,7 @@ function fetch{T<:AbstractString}(repo::GitRepo; GitRemoteAnon(repo, remoteurl) end try - with(default_signature(repo)) do sig - fetch(rmt, sig, refspecs, msg="from $(url(rmt))") - end + fetch(rmt, refspecs, msg="from $(url(rmt))") catch err warn("fetch: $err") finally @@ -209,9 +182,7 @@ function push{T<:AbstractString}(repo::GitRepo; GitRemoteAnon(repo, remoteurl) end try - with(default_signature(repo)) do sig - push(rmt, sig, refspecs, force=force, msg="to $(url(rmt))") - end + push(rmt, refspecs, force=force, msg="to $(url(rmt))") catch err warn("push: $err") finally @@ -222,13 +193,13 @@ end """ git branch """ function branch(repo::GitRepo) head_ref = head(repo) - brnch = "" try - brnch = branch(head_ref) + branch(head_ref) + catch + "" finally finalize(head_ref) end - return brnch end """ git checkout [-b|-B] [] [--track /] """ @@ -306,8 +277,8 @@ function checkout!(repo::GitRepo, commit::AbstractString = ""; try peeled = peel(obj, GitConst.OBJ_COMMIT) peeled == nothing && return - opts = force ? CheckoutOptionsStruct(checkout_strategy = GitConst.CHECKOUT_FORCE) : - CheckoutOptionsStruct() + opts = force ? CheckoutOptions(checkout_strategy = GitConst.CHECKOUT_FORCE) : + CheckoutOptions() try # detach commit obj_oid = Oid(peeled) @@ -327,24 +298,18 @@ function checkout!(repo::GitRepo, commit::AbstractString = ""; end """ git clone [-b ] [--bare] """ -function clone(url::AbstractString, path::AbstractString; - branch::AbstractString="", - bare::Bool = false, +function clone(repo_url::AbstractString, repo_path::AbstractString; + checkout_branch::AbstractString="", + isbare::Bool = false, remote_cb::Ptr{Void} = C_NULL) # setup colne options - clone_opts = CloneOptionsStruct( - bare = Int32(bare), - checkout_branch = isempty(branch) ? Ptr{UInt8}(C_NULL) : pointer(branch), + clone_opts = CloneOptions( + bare = Int32(isbare), + checkout_branch = isempty(checkout_branch) ? Cstring_NULL : + convert(Cstring, pointer(checkout_branch)), remote_cb = remote_cb ) - - # start cloning - clone_opts_ref = Ref(clone_opts) - repo_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - @check ccall((:git_clone, :libgit2), Cint, - (Ptr{Ptr{Void}}, Cstring, Cstring, Ref{CloneOptionsStruct}), - repo_ptr_ptr, url, path, clone_opts_ref) - return GitRepo(repo_ptr_ptr[]) + return clone(repo_url, repo_path, clone_opts) end """ git reset [] [--] ... """ @@ -417,7 +382,7 @@ function merge!(repo::GitRepo, committish::AbstractString=""; fast_forward::Bool reset!(repo, brn_ref_oid, GitConst.RESET_HARD) elseif (ma & GitConst.MERGE_ANALYSIS_NORMAL == GitConst.MERGE_ANALYSIS_NORMAL) fast_forward && return false # do not do merge - merge(repo, hua) + merge!(repo, hua) cleanup(repo) info("Review and commit merged changes.") else @@ -507,7 +472,7 @@ function restore(s::State, repo::GitRepo) reset!(repo, GitConst.HEAD_FILE, "*") # unstage everything with(GitIndex, repo) do idx read_tree!(idx, s.work) # move work tree to index - opts = CheckoutOptionsStruct( + opts = CheckoutOptions( checkout_strategy = GitConst.CHECKOUT_FORCE | # check the index out to work GitConst.CHECKOUT_REMOVE_UNTRACKED) # remove everything else checkout_index(repo, Nullable(idx), options = opts) @@ -530,7 +495,7 @@ end function __init__() err = ccall((:git_libgit2_init, :libgit2), Cint, ()) - err > 0 || throw(PkgError("error initializing LibGit2 module")) + err > 0 || throw(ErrorException("error initializing LibGit2 module")) atexit() do ccall((:git_libgit2_shutdown, :libgit2), Cint, ()) end diff --git a/base/pkg/libgit2/blob.jl b/base/libgit2/blob.jl similarity index 100% rename from base/pkg/libgit2/blob.jl rename to base/libgit2/blob.jl diff --git a/base/pkg/libgit2/commit.jl b/base/libgit2/commit.jl similarity index 93% rename from base/pkg/libgit2/commit.jl rename to base/libgit2/commit.jl index 2b781855c3b33..3c3c66072fd4d 100644 --- a/base/pkg/libgit2/commit.jl +++ b/base/libgit2/commit.jl @@ -1,10 +1,10 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license function message(c::GitCommit, raw::Bool=false) - local msg_ptr::Ptr{UInt8} - msg_ptr = raw? ccall((:git_commit_message_raw, :libgit2), CString, (Ptr{Void},), c.ptr) : - ccall((:git_commit_message, :libgit2), CString, (Ptr{Void},), c.ptr) - if msg_ptr == C_NULL + local msg_ptr::Cstring + msg_ptr = raw? ccall((:git_commit_message_raw, :libgit2), Cstring, (Ptr{Void},), c.ptr) : + ccall((:git_commit_message, :libgit2), Cstring, (Ptr{Void},), c.ptr) + if msg_ptr == Cstring_NULL return nothing end return bytestring(msg_ptr) diff --git a/base/pkg/libgit2/config.jl b/base/libgit2/config.jl similarity index 98% rename from base/pkg/libgit2/config.jl rename to base/libgit2/config.jl index 6bc77f39ee9e8..6ed5c5650af9d 100644 --- a/base/pkg/libgit2/config.jl +++ b/base/libgit2/config.jl @@ -3,7 +3,7 @@ function GitConfig(path::AbstractString) cfg_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_config_open_ondisk, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{UInt8}), cfg_ptr_ptr, path) + (Ptr{Ptr{Void}}, Cstring), cfg_ptr_ptr, path) err !=0 && return nothing return GitConfig(cfg_ptr_ptr[]) end diff --git a/base/pkg/libgit2/const.jl b/base/libgit2/const.jl similarity index 89% rename from base/pkg/libgit2/const.jl rename to base/libgit2/const.jl index bfe047ee46387..3ca2f7215e179 100644 --- a/base/pkg/libgit2/const.jl +++ b/base/libgit2/const.jl @@ -40,8 +40,8 @@ module GitConst # checkout const CHECKOUT_NONE = Cuint(0) const CHECKOUT_SAFE = Cuint(1 << 0) - const CHECKOUT_SAFE_CREATE = Cuint(1 << 1) - const CHECKOUT_FORCE = Cuint(1 << 2) + const CHECKOUT_FORCE = Cuint(1 << 1) + const CHECKOUT_RECREATE_MISSING = Cuint(1 << 2) const CHECKOUT_ALLOW_CONFLICTS = Cuint(1 << 4) const CHECKOUT_REMOVE_UNTRACKED = Cuint(1 << 5) const CHECKOUT_REMOVE_IGNORED = Cuint(1 << 6) @@ -54,6 +54,9 @@ module GitConst const CHECKOUT_DISABLE_PATHSPEC_MATCH = Cuint(1 << 13) const CHECKOUT_SKIP_LOCKED_DIRECTORIES = Cuint(1 << 18) const CHECKOUT_DONT_OVERWRITE_IGNORED = Cuint(1 << 19) + const CHECKOUT_CONFLICT_STYLE_MERGE = Cuint(1 << 20) + const CHECKOUT_CONFLICT_STYLE_DIFF3 = Cuint(1 << 21) + const CHECKOUT_DONT_REMOVE_EXISTING = Cuint(1 << 22) const CHECKOUT_UPDATE_SUBMODULES = Cuint(1 << 16) const CHECKOUT_UPDATE_SUBMODULES_IF_CHANGED = Cuint(1 << 17) @@ -191,4 +194,21 @@ module GitConst const CREDTYPE_SSH_KEY = Cuint(1 << 1) const CREDTYPE_SSH_CUSTOM = Cuint(1 << 2) const CREDTYPE_DEFAULT = Cuint(1 << 3) + + # fetch_prune + const FETCH_PRUNE_UNSPECIFIED = Cint(0) + const FETCH_PRUNE = Cint(1) + const FETCH_NO_PRUNE = Cint(2) + + # remote_autotag + const REMOTE_DOWNLOAD_TAGS_UNSPECIFIED = Cint(0) + const REMOTE_DOWNLOAD_TAGS_AUTO = Cint(1) + const REMOTE_DOWNLOAD_TAGS_NONE = Cint(2) + const REMOTE_DOWNLOAD_TAGS_ALL = Cint(3) + + # clone + const CLONE_LOCAL_AUTO = Cint(0) + const CLONE_LOCAL = Cint(1) + const CLONE_NO_LOCAL = Cint(2) + const CLONE_LOCAL_NO_LINKS = Cint(3) end \ No newline at end of file diff --git a/base/pkg/libgit2/diff.jl b/base/libgit2/diff.jl similarity index 100% rename from base/pkg/libgit2/diff.jl rename to base/libgit2/diff.jl diff --git a/base/pkg/libgit2/error.jl b/base/libgit2/error.jl similarity index 97% rename from base/pkg/libgit2/error.jl rename to base/libgit2/error.jl index cb772c0a99f0b..3c34b367a67f2 100644 --- a/base/pkg/libgit2/error.jl +++ b/base/libgit2/error.jl @@ -23,9 +23,7 @@ module Error EPEEL = Cint(-19), # the requested peel operation is not possible PASSTHROUGH = Cint(-30), # internal only ITEROVER = Cint(-31)) # signals end of iteration - -Base.(:(==))(c::Code, e::Integer) = c.val == e -Base.(:(==))(e::Integer, c::Code) = c.val == e +Base.getindex(c::Code) = c.val @enum(Class, None, NoMemory, diff --git a/base/pkg/libgit2/index.jl b/base/libgit2/index.jl similarity index 100% rename from base/pkg/libgit2/index.jl rename to base/libgit2/index.jl diff --git a/base/pkg/libgit2/merge.jl b/base/libgit2/merge.jl similarity index 75% rename from base/pkg/libgit2/merge.jl rename to base/libgit2/merge.jl index 35e09d0997e98..19e835ffa4989 100644 --- a/base/pkg/libgit2/merge.jl +++ b/base/libgit2/merge.jl @@ -41,12 +41,28 @@ function merge_analysis(repo::GitRepo, ann::GitAnnotated) end """ Merge changes into current head """ -function merge(repo::GitRepo, their_head::GitAnnotated; +function merge!(repo::GitRepo, their_head::GitAnnotated; merge_opts = MergeOptionsStruct(), - checkout_opts = CheckoutOptionsStruct(checkout_strategy = GitConst.CHECKOUT_SAFE)) + checkout_opts = CheckoutOptions(checkout_strategy = GitConst.CHECKOUT_SAFE)) return @check ccall((:git_merge, :libgit2), Cint, (Ptr{Void}, Ptr{Ptr{Void}}, Csize_t, - Ptr{MergeOptionsStruct}, Ptr{CheckoutOptionsStruct}), + Ptr{MergeOptionsStruct}, Ptr{CheckoutOptions}), repo.ptr, Ref{Ptr{Void}}(their_head.ptr), 1, Ref(merge_opts), Ref(checkout_opts)) end + +function merge_base(repo::GitRepo, one::AbstractString, two::AbstractString) + oid1_ptr = Ref(Oid(one)) + oid2_ptr = Ref(Oid(two)) + moid_ptr = Ref(Oid()) + moid = try + @check ccall((:git_merge_base, :libgit2), Cint, + (Ptr{Oid}, Ptr{Void}, Ptr{Oid}, Ptr{Oid}), + moid_ptr, repo.ptr, oid1_ptr, oid2_ptr) + moid_ptr[] + catch e + #warn("Pkg:",path(repo),"=>",e.msg) + Oid() + end + return moid +end \ No newline at end of file diff --git a/base/pkg/libgit2/oid.jl b/base/libgit2/oid.jl similarity index 100% rename from base/pkg/libgit2/oid.jl rename to base/libgit2/oid.jl diff --git a/base/pkg/libgit2/rebase.jl b/base/libgit2/rebase.jl similarity index 85% rename from base/pkg/libgit2/rebase.jl rename to base/libgit2/rebase.jl index 9d44cd71466d5..41d533a46e241 100644 --- a/base/pkg/libgit2/rebase.jl +++ b/base/libgit2/rebase.jl @@ -3,13 +3,13 @@ function GitRebase(repo::GitRepo, branch::GitAnnotated, upstream::GitAnnotated; onto::Nullable{GitAnnotated}=Nullable{GitAnnotated}(), sig::Nullable{GitSignature}=Nullable{GitSignature}(), - opts::RebaseOptionsStruct = RebaseOptionsStruct()) + opts::RebaseOptions = RebaseOptions()) rebase_ptr_ptr = Ref{Ptr{Void}}(C_NULL) sig_obj = isnull(sig) ? default_signature(repo) : Base.get(sig) try @check ccall((:git_rebase_init, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, - Ptr{Void}, Ptr{SignatureStruct}, Ptr{RebaseOptionsStruct}), + Ptr{Void}, Ptr{SignatureStruct}, Ptr{RebaseOptions}), rebase_ptr_ptr, repo.ptr, branch.ptr, upstream.ptr, isnull(onto) ? C_NULL : Base.get(onto).ptr, sig_obj.ptr, Ref(opts)) finally @@ -34,21 +34,20 @@ function Base.getindex(rb::GitRebase, i::Csize_t) end Base.getindex(rb::GitRebase, i::Int) = getindex(rb, Csize_t(i)) -function Base.next(rb::GitRebase; opts::CheckoutOptionsStruct = CheckoutOptionsStruct()) +function Base.next(rb::GitRebase; opts::CheckoutOptions = CheckoutOptions()) rb_op_ptr_ptr = Ref{Ptr{RebaseOperation}}(C_NULL) try @check ccall((:git_rebase_next, :libgit2), Cint, - (Ptr{Ptr{RebaseOperation}}, Ptr{Void}, Ptr{CheckoutOptionsStruct}), + (Ptr{Ptr{RebaseOperation}}, Ptr{Void}, Ptr{CheckoutOptions}), rb_op_ptr_ptr, rb.ptr, Ref(opts)) catch err - err.code == Error.ITEROVER && return nothing + err.code == Error.ITEROVER[] && return nothing rethrow(err) end return unsafe_load(convert(Ptr{RebaseOperation}, rb_op_ptr_ptr[]), 1) end -function commit(rb::GitRebase, sig::GitSignature; - opts::CheckoutOptionsStruct = CheckoutOptionsStruct()) +function commit(rb::GitRebase, sig::GitSignature) oid_ptr = Ref(Oid()) @check ccall((:git_rebase_commit, :libgit2), Cint, (Ptr{Oid}, Ptr{Void}, Ptr{SignatureStruct}, Ptr{SignatureStruct}, Ptr{UInt8}, Ptr{UInt8}), @@ -61,8 +60,8 @@ function abort(rb::GitRebase, sig::GitSignature) (Ptr{Void}, Ptr{SignatureStruct}), rb.ptr, sig.ptr) end -function finish(rb::GitRebase, sig::GitSignature; opts::RebaseOptionsStruct = RebaseOptionsStruct()) +function finish(rb::GitRebase, sig::GitSignature; opts::RebaseOptions = RebaseOptions()) return ccall((:git_rebase_finish, :libgit2), Csize_t, - (Ptr{Void}, Ptr{SignatureStruct}, Ptr{RebaseOptionsStruct}), + (Ptr{Void}, Ptr{SignatureStruct}, Ptr{RebaseOptions}), rb.ptr, sig.ptr, Ref(opts)) end diff --git a/base/pkg/libgit2/reference.jl b/base/libgit2/reference.jl similarity index 97% rename from base/pkg/libgit2/reference.jl rename to base/libgit2/reference.jl index 1ae4dc874bf35..780eac18eea6f 100644 --- a/base/pkg/libgit2/reference.jl +++ b/base/libgit2/reference.jl @@ -53,9 +53,9 @@ function peel(ref::GitReference, obj_type::Cint) obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_reference_peel, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Cint), obj_ptr_ptr, ref.ptr, obj_type) - if err == Error.ENOTFOUND + if err == Error.ENOTFOUND[] return Oid() - elseif err != Error.GIT_OK + elseif err != Error.GIT_OK[] if obj_ptr_ptr[] != C_NULL finalize(GitAnyObject(obj_ptr_ptr[])) end @@ -116,9 +116,9 @@ function lookup_branch(repo::GitRepo, branch_name::AbstractString, remote::Bool= err = ccall((:git_branch_lookup, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Cint), ref_ptr_ptr, repo.ptr, branch_name, branch_type) - if err == Error.ENOTFOUND + if err == Error.ENOTFOUND[] return nothing - elseif err != Error.GIT_OK + elseif err != Error.GIT_OK[] if repo_ptr_ptr[] != C_NULL finalize(GitReference(ref_ptr_ptr[])) end diff --git a/base/pkg/libgit2/remote.jl b/base/libgit2/remote.jl similarity index 65% rename from base/pkg/libgit2/remote.jl rename to base/libgit2/remote.jl index fea24c4c4ba04..d258570a9f411 100644 --- a/base/pkg/libgit2/remote.jl +++ b/base/libgit2/remote.jl @@ -11,8 +11,8 @@ end function GitRemoteAnon(repo::GitRepo, url::AbstractString) rmt_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_remote_create_anonymous, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Ptr{UInt8}), - rmt_ptr_ptr, repo.ptr, url, C_NULL) + (Ptr{Ptr{Void}}, Ptr{Void}, Cstring), + rmt_ptr_ptr, repo.ptr, url) return GitRemote(rmt_ptr_ptr[]) end @@ -24,41 +24,37 @@ function get(::Type{GitRemote}, repo::GitRepo, rmt_name::AbstractString) return GitRemote(rmt_ptr_ptr[]) end -function save(rmt::GitRemote) - @check ccall((:git_remote_save, :libgit2), Cint, (Ptr{Void}, ), rmt.ptr) -end - function url(rmt::GitRemote) - url_ptr = ccall((:git_remote_url, :libgit2), Ptr{UInt8}, (Ptr{Void}, ), rmt.ptr) - url_ptr == C_NULL && return "" + url_ptr = ccall((:git_remote_url, :libgit2), Cstring, (Ptr{Void}, ), rmt.ptr) + url_ptr == Cstring_NULL && return "" return bytestring(url_ptr) end -function fetch{T<:AbstractString}(rmt::GitRemote, sig::GitSignature, refspecs::Vector{T}; +function fetch{T<:AbstractString}(rmt::GitRemote, refspecs::Vector{T}; + fetch_opts::FetchOptions = FetchOptions(), msg::AbstractString="") - msg = "pkg.libgit2.fetch: $msg" + msg = "libgit2.fetch: $msg" no_refs = (length(refspecs) == 0) !no_refs && (sa = StrArrayStruct(refspecs...)) try @check ccall((:git_remote_fetch, :libgit2), Cint, - (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{SignatureStruct}, Cstring), - rmt.ptr, no_refs ? C_NULL : Ref(sa), sig.ptr, msg) + (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{FetchOptions}, Cstring), + rmt.ptr, no_refs ? C_NULL : Ref(sa), Ref(fetch_opts), msg) finally !no_refs && finalize(sa) end end -function push{T<:AbstractString}(rmt::GitRemote, sig::GitSignature, refspecs::Vector{T}; - force::Bool=false, - msg::AbstractString="") - msg = "pkg.libgit2.push: $msg" +function push{T<:AbstractString}(rmt::GitRemote, refspecs::Vector{T}; + force::Bool=false) + msg = "libgit2.push: $msg" push_opts = PushOptionsStruct() no_refs = (length(refspecs) == 0) !no_refs && (sa = StrArrayStruct(refspecs...)) try @check ccall((:git_remote_push, :libgit2), Cint, - (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{PushOptionsStruct}, Ptr{Void}, Cstring), - rmt.ptr, no_refs ? C_NULL : Ref(sa), Ref(push_opts), sig.ptr, msg) + (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{PushOptionsStruct}), + rmt.ptr, no_refs ? C_NULL : Ref(sa), Ref(push_opts)) finally !no_refs && finalize(sa) end diff --git a/base/pkg/libgit2/repl.jl b/base/libgit2/repl.jl similarity index 98% rename from base/pkg/libgit2/repl.jl rename to base/libgit2/repl.jl index 5731183064147..0246e9798ea7c 100644 --- a/base/pkg/libgit2/repl.jl +++ b/base/libgit2/repl.jl @@ -41,7 +41,7 @@ function repl_cmd(ex) end try repourl = cmd[2] - repo = clone(repourl, repopath, bare=("--bare" in cmd), ) + repo = clone(repourl, repopath, isbare=("--bare" in cmd), ) finalize(repo) println("Cloned $repourl into $repopath") catch ex diff --git a/base/pkg/libgit2/repository.jl b/base/libgit2/repository.jl similarity index 73% rename from base/pkg/libgit2/repository.jl rename to base/libgit2/repository.jl index d0f606cebdcbf..630c1c0e6363f 100644 --- a/base/pkg/libgit2/repository.jl +++ b/base/libgit2/repository.jl @@ -3,8 +3,8 @@ function GitRepo(path::AbstractString) repo_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_repository_open, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{UInt8}), repo_ptr_ptr, path) - if err != Error.GIT_OK + (Ptr{Ptr{Void}}, Cstring), repo_ptr_ptr, path) + if err != Error.GIT_OK[] if repo_ptr_ptr[] != C_NULL finalize(GitRepo(repo_ptr_ptr[])) end @@ -22,7 +22,7 @@ end function init(path::AbstractString, bare::Cuint = Cuint(0)) repo_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_repository_init, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{UInt8}, Cuint), repo_ptr_ptr, path, bare) + (Ptr{Ptr{Void}}, Cstring, Cuint), repo_ptr_ptr, path, bare) return GitRepo(repo_ptr_ptr[]) end @@ -45,7 +45,7 @@ end function revparse(repo::GitRepo, objname::AbstractString) obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_revparse_single, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), obj_ptr_ptr, repo.ptr, objname) + (Ptr{Ptr{Void}}, Ptr{Void}, Cstring), obj_ptr_ptr, repo.ptr, objname) err != 0 && return nothing return GitAnyObject(obj_ptr_ptr[]) end @@ -73,9 +73,9 @@ function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::Oid, oid_size::Int=OID_ (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}, Cint), obj_ptr_ptr, r.ptr, id_ptr, git_otype) end - if err == Error.ENOTFOUND + if err == Error.ENOTFOUND[] return nothing - elseif err != Error.GIT_OK + elseif err != Error.GIT_OK[] if obj_ptr_ptr[] != C_NULL finalize(GitAnyObject(obj_ptr_ptr[])) end @@ -89,7 +89,7 @@ function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::AbstractString) end function gitdir(repo::GitRepo) - return bytestring(ccall((:git_repository_path, :libgit2), Ptr{UInt8}, + return bytestring(ccall((:git_repository_path, :libgit2), Cstring, (Ptr{Void},), repo.ptr)) end @@ -103,9 +103,9 @@ function peel(obj::GitObject, obj_type::Cint) git_otype = getobjecttype(obj_type) err = ccall((:git_object_peel, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Cint), peeled_ptr_ptr, obj.ptr, obj_type) - if err == Error.ENOTFOUND + if err == Error.ENOTFOUND[] return Oid() - elseif err != Error.GIT_OK + elseif err != Error.GIT_OK[] if peeled_ptr_ptr[] != C_NULL finalize(GitAnyObject(peeled_ptr_ptr[])) end @@ -115,24 +115,24 @@ function peel(obj::GitObject, obj_type::Cint) end function checkout_tree(repo::GitRepo, obj::GitObject; - options::CheckoutOptionsStruct = CheckoutOptionsStruct()) + options::CheckoutOptions = CheckoutOptions()) @check ccall((:git_checkout_tree, :libgit2), Cint, - (Ptr{Void}, Ptr{Void}, Ptr{CheckoutOptionsStruct}), + (Ptr{Void}, Ptr{Void}, Ptr{CheckoutOptions}), repo.ptr, obj.ptr, Ref(options)) end function checkout_index(repo::GitRepo, idx::Nullable{GitIndex} = Nullable{GitIndex}(); - options::CheckoutOptionsStruct = CheckoutOptionsStruct()) + options::CheckoutOptions = CheckoutOptions()) @check ccall((:git_checkout_index, :libgit2), Cint, - (Ptr{Void}, Ptr{Void}, Ptr{CheckoutOptionsStruct}), + (Ptr{Void}, Ptr{Void}, Ptr{CheckoutOptions}), repo.ptr, isnull(idx) ? C_NULL : Base.get(idx).ptr, Ref(options)) end -function checkout_head(repo::GitRepo; options::CheckoutOptionsStruct = CheckoutOptionsStruct()) +function checkout_head(repo::GitRepo; options::CheckoutOptions = CheckoutOptions()) @check ccall((:git_checkout_head, :libgit2), Cint, - (Ptr{Void}, Ptr{CheckoutOptionsStruct}), + (Ptr{Void}, Ptr{CheckoutOptions}), repo.ptr, Ref(options)) end @@ -147,11 +147,18 @@ function reset!(repo::GitRepo, obj::Nullable{GitAnyObject}, pathspecs::AbstractS end function reset!(repo::GitRepo, obj::GitObject, mode::Cint; - checkout_opts::CheckoutOptionsStruct = CheckoutOptionsStruct()) - with(default_signature(repo)) do sig - msg = "pkg.libgit2.reset: moving to $(string(Oid(obj)))" - @check ccall((:git_reset, :libgit2), Cint, - (Ptr{Void}, Ptr{Void}, Cint, Ptr{CheckoutOptionsStruct}, Ptr{SignatureStruct}, Ptr{UInt8}), - repo.ptr, obj.ptr, mode, Ref(checkout_opts), sig.ptr, msg) - end + checkout_opts::CheckoutOptions = CheckoutOptions()) + @check ccall((:git_reset, :libgit2), Cint, + (Ptr{Void}, Ptr{Void}, Cint, Ptr{CheckoutOptions}, Ptr{SignatureStruct}, Ptr{UInt8}), + repo.ptr, obj.ptr, mode, Ref(checkout_opts)) +end + +function clone(repo_url::AbstractString, repo_path::AbstractString, + clone_opts::CloneOptions = CloneOptions()) + clone_opts_ref = Ref(clone_opts) + repo_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_clone, :libgit2), Cint, + (Ptr{Ptr{Void}}, Cstring, Cstring, Ref{CloneOptions}), + repo_ptr_ptr, repo_url, repo_path, clone_opts_ref) + return GitRepo(repo_ptr_ptr[]) end diff --git a/base/pkg/libgit2/signature.jl b/base/libgit2/signature.jl similarity index 100% rename from base/pkg/libgit2/signature.jl rename to base/libgit2/signature.jl diff --git a/base/libgit2/strarray.jl b/base/libgit2/strarray.jl new file mode 100644 index 0000000000000..9e5a4966abea7 --- /dev/null +++ b/base/libgit2/strarray.jl @@ -0,0 +1,33 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + +function StrArrayStruct{T<:AbstractString}(strs::T...) + strcount = length(strs) + map(s->Base.unsafe_convert(Cstring, s), strs) # check for null-strings + sa_strings = convert(Ptr{Cstring}, Libc.malloc(sizeof(Cstring) * strcount)) + for i=1:strcount + len = length(strs[i]) + in_ptr = pointer(bytestring(strs[i])) + out_ptr = convert(Ptr{UInt8}, Libc.malloc(sizeof(UInt8) * (len + 1))) + unsafe_copy!(out_ptr, in_ptr, len) + unsafe_store!(out_ptr, zero(UInt8), len + 1) # NULL byte + unsafe_store!(sa_strings, convert(Cstring, out_ptr), i) + end + return StrArrayStruct(sa_strings, strcount) +end +StrArrayStruct{T<:AbstractString}(strs::Vector{T}) = StrArrayStruct(strs...) + +function Base.convert(::Type{Vector{AbstractString}}, sa::StrArrayStruct) + arr = Array(AbstractString, sa.count) + for i=1:sa.count + arr[i] = bytestring(unsafe_load(sa.strings, i)) + end + return arr +end + +function Base.copy(src::StrArrayStruct) + dst_ptr = Ref(StrArrayStruct()) + @check ccall((:git_strarray_copy, :libgit2), Cint, + (Ptr{StrArrayStruct}, Ptr{StrArrayStruct}), + dst_ptr, Ref(src)) + return dst_ptr[] +end diff --git a/base/pkg/libgit2/tag.jl b/base/libgit2/tag.jl similarity index 100% rename from base/pkg/libgit2/tag.jl rename to base/libgit2/tag.jl diff --git a/base/pkg/libgit2/types.jl b/base/libgit2/types.jl similarity index 66% rename from base/pkg/libgit2/types.jl rename to base/libgit2/types.jl index 3815115d41093..c2e3eacc9d67d 100644 --- a/base/pkg/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -1,5 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +const Cstring_NULL = convert(Cstring, Ptr{UInt8}(C_NULL)) + const OID_RAWSZ = 20 const OID_HEXSZ = OID_RAWSZ * 2 const OID_MINPREFIXLEN = 4 @@ -37,83 +39,99 @@ SignatureStruct() = SignatureStruct(Ptr{UInt8}(0), Ptr{UInt8}(0), TimeStruct()) +# immutable StrArrayStruct +# strings::Ptr{Ptr{UInt8}} +# count::Csize_t +# end +# StrArrayStruct() = StrArrayStruct(Ptr{UInt8}(0), zero(Csize_t)) +# function Base.finalize(sa::StrArrayStruct) +# sa_ptr = Ref(sa) +# ccall((:git_strarray_free, :libgit2), Void, (Ptr{StrArrayStruct},), sa_ptr) +# return sa_ptr[] +# end + immutable StrArrayStruct - strings::Ptr{Ptr{UInt8}} + strings::Ptr{Cstring} count::Csize_t end -StrArrayStruct() = StrArrayStruct(Ptr{UInt8}(0), zero(Csize_t)) +StrArrayStruct() = StrArrayStruct(Ptr{Cstring}(C_NULL), zero(Csize_t)) function Base.finalize(sa::StrArrayStruct) sa_ptr = Ref(sa) ccall((:git_strarray_free, :libgit2), Void, (Ptr{StrArrayStruct},), sa_ptr) return sa_ptr[] end -immutable StrArrayStruct2 - strings::Ptr{Cstring} - count::Csize_t -end -StrArrayStruct2() = StrArrayStruct2(Ptr{Cstring}(C_NULL), zero(Csize_t)) -function Base.finalize(sa::StrArrayStruct2) - sa_ptr = Ref(sa) - ccall((:git_strarray_free, :libgit2), Void, (Ptr{StrArrayStruct2},), sa_ptr) - return sa_ptr[] -end - - -immutable CheckoutOptionsStruct +immutable CheckoutOptions version::Cuint + checkout_strategy::Cuint + disable_filters::Cint dir_mode::Cuint file_mode::Cuint file_open_flags::Cint + notify_flags::Cuint notify_cb::Ptr{Void} notify_payload::Ptr{Void} + progress_cb::Ptr{Void} progress_payload::Ptr{Void} + paths::StrArrayStruct + baseline::Ptr{Void} - target_directory::Ptr{UInt8} - ancestor_label::Ptr{UInt8} - our_label::Ptr{UInt8} - their_label::Ptr{UInt8} + baseline_index::Ptr{Void} + + target_directory::Cstring + ancestor_label::Cstring + our_label::Cstring + their_label::Cstring + + perfdata_cb::Ptr{Void} + perfdata_payload::Ptr{Void} end -CheckoutOptionsStruct(; checkout_strategy::Cuint = GitConst.CHECKOUT_SAFE_CREATE, - disable_filters::Cint = zero(Cint), - dir_mode::Cuint = Cuint(0), # Cuint(0o755), - file_mode::Cuint = Cuint(0), #Cuint(0o644), - file_open_flags::Cint = zero(Cint), - notify_flags::Cuint = GitConst.CHECKOUT_NOTIFY_NONE, - notify_cb::Ptr{Void} = Ptr{Void}(0), - notify_payload::Ptr{Void} = Ptr{Void}(0), - progress_cb::Ptr{Void} = Ptr{Void}(0), - progress_payload::Ptr{Void} = Ptr{Void}(0), - paths::StrArrayStruct = StrArrayStruct(), - baseline::Ptr{Void} = Ptr{Void}(0), - target_directory::Ptr{UInt8} = Ptr{UInt8}(0), - ancestor_label::Ptr{UInt8} =Ptr{UInt8}(0), - our_label::Ptr{UInt8} = Ptr{UInt8}(0), - their_label::Ptr{UInt8} = Ptr{UInt8}(0) -)=CheckoutOptionsStruct(one(Cuint), - checkout_strategy, - disable_filters, - dir_mode, - file_mode, - file_open_flags, - notify_flags, - notify_cb, - notify_payload, - progress_cb, - progress_payload, - paths, - baseline, - target_directory, - ancestor_label, - our_label, - their_label) - -immutable RemoteCallbacksStruct +CheckoutOptions(; checkout_strategy::Cuint = GitConst.CHECKOUT_SAFE, + disable_filters::Cint = zero(Cint), + dir_mode::Cuint = Cuint(0), # Cuint(0o755), + file_mode::Cuint = Cuint(0), #Cuint(0o644), + file_open_flags::Cint = zero(Cint), + notify_flags::Cuint = GitConst.CHECKOUT_NOTIFY_NONE, + notify_cb::Ptr{Void} = Ptr{Void}(0), + notify_payload::Ptr{Void} = Ptr{Void}(0), + progress_cb::Ptr{Void} = Ptr{Void}(0), + progress_payload::Ptr{Void} = Ptr{Void}(0), + paths::StrArrayStruct = StrArrayStruct(), + baseline::Ptr{Void} = Ptr{Void}(0), + baseline_index::Ptr{Void} = Ptr{Void}(0), + target_directory::Cstring = Cstring_NULL, + ancestor_label::Cstring = Cstring_NULL, + our_label::Cstring = Cstring_NULL, + their_label::Cstring = Cstring_NULL, + perfdata_cb::Ptr{Void} = Ptr{Void}(0), + perfdata_payload::Ptr{Void} = Ptr{Void}(0) +)=CheckoutOptions(one(Cuint), + checkout_strategy, + disable_filters, + dir_mode, + file_mode, + file_open_flags, + notify_flags, + notify_cb, + notify_payload, + progress_cb, + progress_payload, + paths, + baseline, + baseline_index, + target_directory, + ancestor_label, + our_label, + their_label, + perfdata_cb, + perfdata_payload) + +immutable RemoteCallbacks version::Cuint sideband_progress::Ptr{Void} completion::Ptr{Void} @@ -124,54 +142,72 @@ immutable RemoteCallbacksStruct pack_progress::Ptr{Void} push_transfer_progress::Ptr{Void} push_update_reference::Ptr{Void} + push_negotiation::Ptr{Void} + transport::Ptr{Void} payload::Ptr{Void} end -RemoteCallbacksStruct() = RemoteCallbacksStruct(one(Cuint), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0)) - -immutable CloneOptionsStruct +RemoteCallbacks() = RemoteCallbacks(one(Cuint), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0), + Ptr{Void}(0)) + +immutable FetchOptions + version::Cuint + callbacks::RemoteCallbacks + prune::Cint + update_fetchhead::Cint + download_tags::Cint +end +FetchOptions(; callbacks::RemoteCallbacks = RemoteCallbacks(), + prune::Cint = GitConst.FETCH_PRUNE_UNSPECIFIED, + update_fetchhead::Cint = one(Cint), + download_tags::Cint = GitConst.REMOTE_DOWNLOAD_TAGS_AUTO +) = FetchOptions(one(Cuint), + callbacks, + prune, + update_fetchhead, + download_tags) + +immutable CloneOptions version::Cuint - checkout_opts::CheckoutOptionsStruct - remote_callbacks::RemoteCallbacksStruct + checkout_opts::CheckoutOptions + fetch_opts::FetchOptions bare::Cint localclone::Cint - checkout_branch::Ptr{UInt8} - signature::Ptr{Void} + checkout_branch::Cstring repository_cb::Ptr{Void} repository_cb_payload::Ptr{Void} remote_cb::Ptr{Void} remote_cb_payload::Ptr{Void} end -CloneOptionsStruct(; checkout_opts::CheckoutOptionsStruct = CheckoutOptionsStruct(), - remote_callbacks::RemoteCallbacksStruct = RemoteCallbacksStruct(), - bare::Cint = zero(Cint), - localclone::Cint = zero(Cint), - checkout_branch::Ptr{UInt8} = Ptr{UInt8}(0), - signature::Ptr{Void} = Ptr{Void}(0), - repository_cb::Ptr{Void} = Ptr{Void}(0), - repository_cb_payload::Ptr{Void} = Ptr{Void}(0), - remote_cb::Ptr{Void} = Ptr{Void}(0), - remote_cb_payload::Ptr{Void} = Ptr{Void}(0) -)=CloneOptionsStruct(one(Cuint), - checkout_opts, - remote_callbacks, - bare, - localclone, - checkout_branch, - signature, - repository_cb, - repository_cb_payload, - remote_cb, - remote_cb_payload) +CloneOptions(; checkout_opts::CheckoutOptions = CheckoutOptions(), + fetch_opts::FetchOptions = FetchOptions(), + bare::Cint = zero(Cint), + localclone::Cint = GitConst.CLONE_LOCAL_AUTO, + checkout_branch::Cstring = Cstring_NULL, + repository_cb::Ptr{Void} = Ptr{Void}(0), + repository_cb_payload::Ptr{Void} = Ptr{Void}(0), + remote_cb::Ptr{Void} = Ptr{Void}(0), + remote_cb_payload::Ptr{Void} = Ptr{Void}(0) +)=CloneOptions(one(Cuint), + checkout_opts, + fetch_opts, + bare, + localclone, + checkout_branch, + repository_cb, + repository_cb_payload, + remote_cb, + remote_cb_payload) # git diff option struct immutable DiffOptionsStruct @@ -301,19 +337,19 @@ IndexEntry() = IndexEntry(IndexTime(), Ptr{UInt8}(0)) Base.show(io::IO, ie::IndexEntry) = print(io, "IndexEntry($(string(ie.id)))") -immutable RebaseOptionsStruct +immutable RebaseOptions version::Cuint quiet::Cint rewrite_notes_ref::Cstring end -RebaseOptionsStruct()=RebaseOptionsStruct(one(Cuint), Cint(1), convert(Cstring, Ptr{UInt8}(C_NULL))) +RebaseOptions()=RebaseOptions(one(Cuint), Cint(1), Cstring_NULL) immutable RebaseOperation optype::Cint id::Oid exec::Cstring end -RebaseOperation()=RebaseOperation(Cint(0),Oid(),convert(Cstring, Ptr{UInt8}(C_NULL))) +RebaseOperation()=RebaseOperation(Cint(0), Oid(), Cstring_NULL) Base.show(io::IO, rbo::RebaseOperation) = print(io, "RebaseOperation($(string(rbo.id)))") # Abstract object types diff --git a/base/libgit2/utils.jl b/base/libgit2/utils.jl new file mode 100644 index 0000000000000..07f844df618d2 --- /dev/null +++ b/base/libgit2/utils.jl @@ -0,0 +1,6 @@ +function version() + major, minor, patch = Cint[0], Cint[0], Cint[0] + ccall((:git_libgit2_version, :libgit2), Void, + (Ptr{Cint}, Ptr{Cint}, Ptr{Cint}), major, minor, patch) + return VersionNumber(major[1], minor[1], patch[1]) +end \ No newline at end of file diff --git a/base/pkg/libgit2/walker.jl b/base/libgit2/walker.jl similarity index 96% rename from base/pkg/libgit2/walker.jl rename to base/libgit2/walker.jl index 364e8cdbd82bc..18ab534a9ede0 100644 --- a/base/pkg/libgit2/walker.jl +++ b/base/libgit2/walker.jl @@ -11,7 +11,7 @@ function Base.start(w::GitRevWalker) id_ptr = Ref(Oid()) err = ccall((:git_revwalk_next, :libgit2), Cint, (Ptr{Oid}, Ptr{Void}), id_ptr, w.ptr) - err != Error.GIT_OK && return (nothing, true) + err != Error.GIT_OK[] && return (nothing, true) return (id_ptr[], false) end @@ -21,7 +21,7 @@ function Base.next(w::GitRevWalker, state) id_ptr = Ref(Oid()) err = ccall((:git_revwalk_next, :libgit2), Cint, (Ptr{Oid}, Ptr{Void}), id_ptr, w.ptr) - err != Error.GIT_OK && return (state[1], (nothing, true)) + err != Error.GIT_OK[] && return (state[1], (nothing, true)) return (state[1], (id_ptr[], false)) end diff --git a/base/methodshow.jl b/base/methodshow.jl index 815d1510cff9e..540912063683b 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -81,42 +81,36 @@ end show(io::IO, mt::MethodTable) = show_method_table(io, mt) -function inbase(m::Module) - if m == Base - true - else - parent = module_parent(m) - parent === m ? false : inbase(parent) - end -end -fileurl(file) = let f = find_source_file(file); f === nothing ? "" : "file://"*f; end -function url(m::Method) - M = m.func.code.module - (m.func.code.file == :null || m.func.code.file == :string) && return "" - file = string(m.func.code.file) - line = m.func.code.line - line <= 0 || ismatch(r"In\[[0-9]+\]", file) && return "" - if inbase(M) - return "https://github.com/JuliaLang/julia/tree/$(Base.GIT_VERSION_INFO.commit)/base/$file#L$line" - else - try - d = dirname(file) - u = Git.readchomp(`config remote.origin.url`, dir=d) - u = match(Git.GITHUB_REGEX,u).captures[1] - root = cd(d) do # dir=d confuses --show-toplevel, apparently - Git.readchomp(`rev-parse --show-toplevel`) - end - if startswith(file, root) - commit = Git.readchomp(`rev-parse HEAD`, dir=d) - return "https://github.com/$u/tree/$commit/"*file[length(root)+2:end]*"#L$line" - else - return fileurl(file) - end - catch - return fileurl(file) - end - end -end +inbase(m::Module) = m == Base ? true : m == Main ? false : inbase(module_parent(m)) +fileurl(file) = let f = find_source_file(file); f == nothing ? "" : "file://"*f; end +#TODO: Convert to libgit2 +# function url(m::Method) +# M = m.func.code.module +# (m.func.code.file == :null || m.func.code.file == :string) && return "" +# file = string(m.func.code.file) +# line = m.func.code.line +# line <= 0 || ismatch(r"In\[[0-9]+\]", file) && return "" +# if inbase(M) +# return "https://github.com/JuliaLang/julia/tree/$(Base.GIT_VERSION_INFO.commit)/base/$file#L$line" +# else +# try +# d = dirname(file) +# u = Git.readchomp(`config remote.origin.url`, dir=d) +# u = match(Git.GITHUB_REGEX,u).captures[1] +# root = cd(d) do # dir=d confuses --show-toplevel, apparently +# Git.readchomp(`rev-parse --show-toplevel`) +# end +# if startswith(file, root) +# commit = Git.readchomp(`rev-parse HEAD`, dir=d) +# return "https://github.com/$u/tree/$commit/"*file[length(root)+2:end]*"#L$line" +# else +# return fileurl(file) +# end +# catch +# return fileurl(file) +# end +# end +# end function writemime(io::IO, ::MIME"text/html", m::Method) print(io, m.func.code.name) diff --git a/base/pkg.jl b/base/pkg.jl index ef5298e9a1524..5f5a153bc071c 100644 --- a/base/pkg.jl +++ b/base/pkg.jl @@ -2,7 +2,7 @@ module Pkg -export Git, Dir, GitHub, Types, Reqs, Cache, Read, Query, Resolve, Write, Generate, Entry +export Dir, GitHub, Types, Reqs, Cache, Read, Query, Resolve, Write, Generate, Entry export dir, init, rm, add, available, installed, status, clone, checkout, update, resolve, register, tag, publish, generate, test, build, free, pin @@ -14,7 +14,7 @@ type PkgError <: Exception msg::AbstractString end -for file in split("git libgit2 dir github types reqs cache read query resolve write generate entry") +for file in split("dir github types reqs cache read query resolve write generate entry") include("pkg/$file.jl") end const cd = Dir.cd diff --git a/base/pkg/cache.jl b/base/pkg/cache.jl index 47fc05f5d8a03..db984caab1a12 100644 --- a/base/pkg/cache.jl +++ b/base/pkg/cache.jl @@ -2,7 +2,7 @@ module Cache -import ..LibGit2, ..Dir, ...Pkg.PkgError +import ...LibGit2, ..Dir, ...Pkg.PkgError using ..Types path(pkg::AbstractString) = abspath(".cache", pkg) @@ -39,7 +39,7 @@ function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) info("Cloning cache of $pkg from $url") try # clone repo, free it at the end - LibGit2.clone(url, cache, bare = true, remote_cb = LibGit2.mirror_cb) + LibGit2.clone(url, cache, isbare = true, remote_cb = LibGit2.mirror_cb) catch err isdir(cache) && rm(cache, recursive=true) throw(PkgError("Cannot clone $pkg from $url. $(err.msg)")) diff --git a/base/pkg/dir.jl b/base/pkg/dir.jl index ca08a0291a686..7566f6ec2c9ad 100644 --- a/base/pkg/dir.jl +++ b/base/pkg/dir.jl @@ -3,7 +3,7 @@ module Dir import ..Pkg: DEFAULT_META, META_BRANCH, PkgError -import ..LibGit2, ..LibGit2.with +import ...LibGit2, ...LibGit2.with const DIR_NAME = ".julia" diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 307621a7b8223..14d981a02e032 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -3,8 +3,9 @@ module Entry import Base: thispatch, nextpatch, nextminor, nextmajor, check_new_version -import ..LibGit2, ..Git, ..Reqs, ..Read, ..Query, ..Resolve, ..Cache, ..Write, ..GitHub, ..Dir -importall ..LibGit2 +import ..Reqs, ..Read, ..Query, ..Resolve, ..Cache, ..Write, ..GitHub, ..Dir +import ...LibGit2 +importall ...LibGit2 import ...Pkg.PkgError using ..Types @@ -224,15 +225,15 @@ function clone(url_or_pkg::AbstractString) clone(url,pkg) end -function checkout(pkg::AbstractString, branch::AbstractString, merge::Bool, pull::Bool) +function checkout(pkg::AbstractString, branch::AbstractString, do_merge::Bool, do_pull::Bool) ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) info("Checking out $pkg $branch...") with(GitRepo, pkg) do r LibGit2.transact(r) do repo LibGit2.isdirty(repo) && throw(PkgError("$pkg is dirty, bailing")) LibGit2.branch!(repo, branch, track=LibGit2.GitConst.REMOTE_ORIGIN) - merge && LibGit2.merge!(repo, fast_forward=true) # merge changes - if pull + do_merge && LibGit2.merge!(repo, fast_forward=true) # merge changes + if do_pull info("Pulling $pkg latest $branch...") LibGit2.fetch(repo) LibGit2.merge!(repo, fast_forward=true) diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index c1668b5208d66..0634c01fc3fa4 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -2,8 +2,8 @@ module Generate -import ..LibGit2, ..Read, ...Pkg.PkgError -importall ..LibGit2 +import ...LibGit2, ..Read, ...Pkg.PkgError +importall ...LibGit2 copyright_year() = string(Dates.year(Dates.today())) copyright_name(repo::GitRepo) = LibGit2.getconfig(repo, "user.name", "") diff --git a/base/pkg/github.jl b/base/pkg/github.jl index f7a081cb03b47..62df505239cbd 100644 --- a/base/pkg/github.jl +++ b/base/pkg/github.jl @@ -2,7 +2,10 @@ module GitHub -import Main, ..LibGit2, ..Dir, ...Pkg.PkgError +import Main, ...LibGit2, ..Dir, ...Pkg.PkgError + +const GITHUB_REGEX = + r"^(?:git@|git://|https://(?:[\w\.\+\-]+@)?)github.com[:/](([^/].+)/(.+?))(?:\.git)?$"i const AUTH_NOTE = "Julia Package Manager" const AUTH_DATA = Dict{Any,Any}( @@ -149,4 +152,9 @@ function fork(owner::AbstractString, repo::AbstractString) return response end +function normalize_url(url::AbstractString) + m = match(GITHUB_REGEX,url) + m == nothing ? url : "https://github.com/$(m.captures[1]).git" +end + end # module diff --git a/base/pkg/libgit2/strarray.jl b/base/pkg/libgit2/strarray.jl deleted file mode 100644 index c1d7f6b901fb7..0000000000000 --- a/base/pkg/libgit2/strarray.jl +++ /dev/null @@ -1,39 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -function StrArrayStruct{T<:AbstractString}(strs::T...) - count = length(strs) - strings = convert(Ptr{Ptr{UInt8}}, Libc.malloc(sizeof(Ptr{UInt8}) * count)) - for i=1:count - len = length(strs[i]) - #in_ptr = convert(Ptr{Uint8}, bytestring(strs[i])) - in_ptr = unsafe_convert(Cstring, strs[i]) - out_ptr = convert(Ptr{UInt8}, Libc.malloc(sizeof(UInt8) * (len + 1))) - unsafe_copy!(out_ptr, in_ptr, len) - unsafe_store!(out_ptr, zero(UInt8), len + 1) # NULL byte - unsafe_store!(strings, out_ptr, i) - end - return StrArrayStruct(strings, count) -end -StrArrayStruct{T<:AbstractString}(strs::Vector{T}) = StrArrayStruct(strs...) - -function StrArrayStruct2{T<:AbstractString}(strs::T...) - count = length(strs) - strings = convert(Ptr{Cstring}, Libc.malloc(sizeof(Ptr{UInt8}) * count)) - for i=1:count - len = length(strs[i]) - in_ptr = unsafe_convert(Cstring, strs[i]) - out_ptr = convert(Cstring, Libc.malloc(sizeof(UInt8) * (len + 1))) - unsafe_copy!(out_ptr, in_ptr, len) - unsafe_store!(out_ptr, zero(UInt8), len + 1) # NULL byte - unsafe_store!(strings, out_ptr, i) - end - return StrArrayStruct(strings, count) -end - -function Base.convert(::Type{Vector{AbstractString}}, sa::StrArrayStruct) - arr = Array(AbstractString, sa.count) - for i=1:sa.count - arr[i] = bytestring(unsafe_load(sa.strings, i)) - end - return arr -end diff --git a/base/pkg/read.jl b/base/pkg/read.jl index f684693045fd7..d18eda3d12d1a 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -2,7 +2,7 @@ module Read -import ..LibGit2, ..Cache, ..Reqs, ...Pkg.PkgError +import ...LibGit2, ..Cache, ..Reqs, ...Pkg.PkgError using ..Types readstrip(path...) = strip(readall(joinpath(path...))) @@ -119,9 +119,9 @@ function installed_version(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::D for (ver,info) in avail sha1 = info.sha1 base = if cache_has_head && LibGit2.iscommit(sha1, crepo) - LibGit2.merge_base(head, sha1, crepo) + LibGit2.merge_base(crepo, head, sha1) elseif LibGit2.iscommit(sha1, prepo) - LibGit2.merge_base(head, sha1, prepo) + LibGit2.merge_base(prepo, head, sha1) else Base.warn_once("unknown $pkg commit $(sha1[1:8]), metadata may be ahead of package cache") continue diff --git a/base/pkg/write.jl b/base/pkg/write.jl index f8d3d13c2cd0a..706d28b883432 100644 --- a/base/pkg/write.jl +++ b/base/pkg/write.jl @@ -2,8 +2,8 @@ module Write -import ..LibGit2, ..Cache, ..Read, ...Pkg.PkgError -importall ..LibGit2 +import ...LibGit2, ..Cache, ..Read, ...Pkg.PkgError +importall ...LibGit2 function prefetch(pkg::AbstractString, sha1::AbstractString) isempty(Cache.prefetch(pkg, Read.url(pkg), sha1)) && return diff --git a/base/sysimg.jl b/base/sysimg.jl index 90e6e066fc285..3d0b4da5e26fe 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -273,9 +273,11 @@ importall .QuadGK include("fastmath.jl") importall .FastMath +# libgit2 support +include("libgit2.jl") + # package manager include("pkg.jl") -const Git = Pkg.Git # profiler include("profile.jl") diff --git a/test/libgit2.jl b/test/libgit2.jl index dddb4429c5391..cb56e6d6c1bb2 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -2,31 +2,54 @@ # check that libgit2 has been installed correctly -const LIBGIT2_VER = v"0.22.1+" +const LIBGIT2_VER = v"0.23.0" function check_version() - major, minor, patch = Cint[0], Cint[0], Cint[0] - ccall((:git_libgit2_version, :libgit2), Void, - (Ptr{Cint}, Ptr{Cint}, Ptr{Cint}), major, minor, patch) - v = VersionNumber(major[1], minor[1], patch[1]) + v = LibGit2.version() if v.major == LIBGIT2_VER.major && v.minor >= LIBGIT2_VER.minor return true else return false end end - @test check_version() -function credentials!(cfg::Pkg.LibGit2.GitConfig, usr="Test User", usr_email="Test@User.com") - git_user = Pkg.LibGit2.get(cfg, "user.name", usr) - usr==git_user && Pkg.LibGit2.set!(cfg, "user.name", usr) - git_user_email = Pkg.LibGit2.get(cfg, "user.email", usr_email) - usr_email==git_user_email && Pkg.LibGit2.set!(cfg, "user.email", usr_email) +# strarray +begin + p1 = "XXX" + p2 = "YYY" + sa1 = LibGit2.StrArrayStruct(p1) + try + arr = convert(Vector{AbstractString}, sa1) + @test arr[1] == p1 + finally + finalize(sa1) + end + + sa2 = LibGit2.StrArrayStruct(p1, p2) + try + arr1 = convert(Vector{AbstractString}, sa2) + @test arr1[1] == p1 + @test arr1[2] == p2 + sa3 = copy(sa2) + arr2 = convert(Vector{AbstractString}, sa3) + @test arr1[1] == arr2[1] + @test arr1[2] == arr2[2] + finalize(sa3) + finally + finalize(sa2) + end +end + +function credentials!(cfg::LibGit2.GitConfig, usr="Test User", usr_email="Test@User.com") + git_user = LibGit2.get(cfg, "user.name", usr) + usr==git_user && LibGit2.set!(cfg, "user.name", usr) + git_user_email = LibGit2.get(cfg, "user.email", usr_email) + usr_email==git_user_email && LibGit2.set!(cfg, "user.email", usr_email) end #TODO: tests need 'user.name' & 'user.email' in config ??? -Pkg.LibGit2.with(Pkg.LibGit2.GitConfig) do cfg +LibGit2.with(LibGit2.GitConfig) do cfg credentials!(cfg) end @@ -44,19 +67,19 @@ end # clone bare temp_dir() do dir - url = "https://github.com/JuliaLang/Example.jl" - path = joinpath(dir, "Example.Bare") - repo = Pkg.LibGit2.clone(url, path, bare = true, remote_cb = Pkg.LibGit2.mirror_cb) + repo_url = "https://github.com/JuliaLang/Example.jl" + repo_path = joinpath(dir, "Example.Bare") + repo = LibGit2.clone(repo_url, repo_path, isbare = true, remote_cb = LibGit2.mirror_cb) finalize(repo) - @test isdir(path) - @test isfile(joinpath(path, Pkg.LibGit2.GitConst.HEAD_FILE)) + @test isdir(repo_path) + @test isfile(joinpath(repo_path, LibGit2.GitConst.HEAD_FILE)) end # clone temp_dir() do dir url = "https://github.com/JuliaLang/Example.jl" path = joinpath(dir, "Example") - repo = Pkg.LibGit2.clone(url, path) + repo = LibGit2.clone(url, path) finalize(repo) @test isdir(path) @test isdir(joinpath(path, ".git")) @@ -65,23 +88,22 @@ end # init temp_dir() do dir path = joinpath(dir, "Example") - repo = Pkg.LibGit2.init(path) + repo = LibGit2.init(path) @test isdir(path) @test isdir(joinpath(path, ".git")) branch = "upstream" url = "https://github.com/JuliaLang/julia.git" - remote = Pkg.LibGit2.GitRemote(repo, branch, url) - Pkg.LibGit2.save(remote) + remote = LibGit2.GitRemote(repo, branch, url) finalize(remote) config = joinpath(path, ".git", "config") lines = split(open(readall, config, "r"), "\n") @test any(map(x->x == "[remote \"upstream\"]", lines)) - remote = Pkg.LibGit2.get(Pkg.LibGit2.GitRemote, repo, branch) - @test Pkg.LibGit2.url(remote) == url + remote = LibGit2.get(LibGit2.GitRemote, repo, branch) + @test LibGit2.url(remote) == url finalize(repo) end @@ -91,26 +113,25 @@ temp_dir() do dir_cache # create cache url = "https://github.com/JuliaLang/Example.jl" path_cache = joinpath(dir_cache, "Example.Bare") - repo = Pkg.LibGit2.clone(url, path_cache, bare = true, remote_cb = Pkg.LibGit2.mirror_cb) - Pkg.LibGit2.with(Pkg.LibGit2.GitConfig, repo) do cfg + repo = LibGit2.clone(url, path_cache, isbare = true, remote_cb = LibGit2.mirror_cb) + LibGit2.with(LibGit2.GitConfig, repo) do cfg credentials!(cfg) end finalize(repo) - # fetch temp_dir() do dir # clone repo path = joinpath(dir, "Example") - repo = Pkg.LibGit2.clone(path_cache, path) - Pkg.LibGit2.with(Pkg.LibGit2.GitConfig, repo) do cfg + repo = LibGit2.clone(path_cache, path) + LibGit2.with(LibGit2.GitConfig, repo) do cfg credentials!(cfg) end - Pkg.LibGit2.fetch(repo) + LibGit2.fetch(repo) refs1 = parse(Int, readchomp(pipe(`find $(joinpath(path, ".git/refs"))`,`wc -l`))) - Pkg.LibGit2.fetch(repo, remoteurl=path_cache, refspecs =["+refs/*:refs/remotes/cache/*"]) + LibGit2.fetch(repo, remoteurl=path_cache, refspecs =["+refs/*:refs/remotes/cache/*"]) refs2 = parse(Int, readchomp(pipe(`find $(joinpath(path, ".git/refs"))`,`wc -l`))) finalize(repo) @@ -119,64 +140,64 @@ temp_dir() do dir_cache @test refs2 > refs1 end - # revwalk - temp_dir() do dir - path = joinpath(dir, "Example") - repo = Pkg.LibGit2.clone(path_cache, path) - Pkg.LibGit2.with(Pkg.LibGit2.GitConfig, repo) do cfg - credentials!(cfg) - end - oids = Pkg.LibGit2.map((oid,repo)->string(oid), repo, by = Pkg.LibGit2.GitConst.SORT_TIME) - @test length(oids) > 0 - finalize(repo) - - Pkg.LibGit2.with(Pkg.LibGit2.GitRepo, path) do repo - oid = Pkg.LibGit2.Oid(oids[end]) - oids = Pkg.LibGit2.map((oid,repo)->(oid,repo), repo, oid=oid, by=Pkg.LibGit2.GitConst.SORT_TIME) - @test length(oids) > 0 - end - end - # signature temp_dir() do dir path = joinpath(dir, "Example") - repo = Pkg.LibGit2.clone(path_cache, path) + repo = LibGit2.clone(path_cache, path) oid = "129eb39c8e0817c616296d1ac5f2cd1cf4f8b312" - c = Pkg.LibGit2.get(Pkg.LibGit2.GitCommit, repo, oid) - auth = Pkg.LibGit2.author(c) - cmtr = Pkg.LibGit2.committer(c) - cmsg = Pkg.LibGit2.message(c) - @test isa(auth, Pkg.LibGit2.Signature) + c = LibGit2.get(LibGit2.GitCommit, repo, oid) + auth = LibGit2.author(c) + cmtr = LibGit2.committer(c) + cmsg = LibGit2.message(c) + @test isa(auth, LibGit2.Signature) @test length(auth.name) > 0 - @test isa(cmtr, Pkg.LibGit2.Signature) + @test isa(cmtr, LibGit2.Signature) @test length(cmtr.email) > 0 @test length(cmsg) > 0 finalize(repo) - sig = Pkg.LibGit2.Signature("AAA", "AAA@BBB.COM", round(time(), 0), 0) - git_sig = convert(Pkg.LibGit2.GitSignature, sig) - sig2 = Pkg.LibGit2.Signature(git_sig) + sig = LibGit2.Signature("AAA", "AAA@BBB.COM", round(time(), 0), 0) + git_sig = convert(LibGit2.GitSignature, sig) + sig2 = LibGit2.Signature(git_sig) finalize(git_sig) @test sig.name == sig2.name @test sig.email == sig2.email @test sig.time == sig2.time end + # revwalk + temp_dir() do dir + path = joinpath(dir, "Example") + repo = LibGit2.clone(path_cache, path) + LibGit2.with(LibGit2.GitConfig, repo) do cfg + credentials!(cfg) + end + oids = LibGit2.map((oid,repo)->string(oid), repo, by = LibGit2.GitConst.SORT_TIME) + @test length(oids) > 0 + finalize(repo) + + LibGit2.with(LibGit2.GitRepo, path) do repo + oid = LibGit2.Oid(oids[end]) + oids = LibGit2.map((oid,repo)->(oid,repo), repo, oid=oid, by=LibGit2.GitConst.SORT_TIME) + @test length(oids) > 0 + end + end + # transact temp_dir() do dir # clone repo path = joinpath(dir, "Example") - repo = Pkg.LibGit2.clone(path_cache, path) - Pkg.LibGit2.with(Pkg.LibGit2.GitConfig, repo) do cfg + repo = LibGit2.clone(path_cache, path) + LibGit2.with(LibGit2.GitConfig, repo) do cfg credentials!(cfg) end cp(joinpath(path, "REQUIRE"), joinpath(path, "CCC")) cp(joinpath(path, "REQUIRE"), joinpath(path, "AAA")) - Pkg.LibGit2.add!(repo, "AAA") - @test_throws ErrorException Pkg.LibGit2.transact(repo) do repo + LibGit2.add!(repo, "AAA") + @test_throws ErrorException LibGit2.transact(repo) do repo mv(joinpath(path, "REQUIRE"), joinpath(path, "BBB")) - Pkg.LibGit2.add!(repo, "BBB") - oid = Pkg.LibGit2.commit(repo, "test commit") + LibGit2.add!(repo, "BBB") + oid = LibGit2.commit(repo, "test commit") error("Force recovery") end @test isfile(joinpath(path, "AAA")) @@ -185,27 +206,20 @@ temp_dir() do dir_cache @test isfile(joinpath(path, "REQUIRE")) end -end - -# strarray -begin - p1 = "XXX" - p2 = "YYY" - sa1 = Pkg.LibGit2.StrArrayStruct(p1) - try - arr = convert(Vector{AbstractString}, sa1) - @test arr[1] == p1 - finally - finalize(sa1) - end + # branch + temp_dir() do dir + path = joinpath(dir, "Example") + repo = LibGit2.clone(path_cache, path) + LibGit2.with(LibGit2.GitConfig, repo) do cfg + credentials!(cfg) + end + brnch = LibGit2.branch(repo) + @test brnch == "master" - sa2 = Pkg.LibGit2.StrArrayStruct(p1, p2) - try - arr = convert(Vector{AbstractString}, sa2) - @test arr[1] == p1 - @test arr[2] == p2 - finally - finalize(sa2) + brnch_name = "test" + LibGit2.branch!(repo, brnch_name) + brnch = LibGit2.branch(repo) + @test brnch == brnch_name end end From c9fcd0b964eb9e2e04357f36318db7afac405ae7 Mon Sep 17 00:00:00 2001 From: wildart Date: Thu, 9 Jul 2015 00:31:32 -0400 Subject: [PATCH 0334/1938] fixed repl tests --- appveyor.yml | 2 +- test/repl.jl | 24 +++++++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 8c3f03524f349..d20034cfb1de5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -49,4 +49,4 @@ build_script: test_script: - usr\bin\julia -e "versioninfo()" - copy usr\lib\julia\sys.ji local.ji && usr\bin\julia -J local.ji -e "true" && del local.ji - - cd test && ..\usr\bin\julia runtests.jl all + - cd test && ..\usr\bin\julia runtests.jl libgit2 repl pkg diff --git a/test/repl.jl b/test/repl.jl index 51968f0e07d27..144c39e0dde2f 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -198,7 +198,10 @@ fakehistory = """ \tls # time: 2014-06-30 20:44:29 EDT # mode: julia -\t2 + 2""" +\t2 + 2 +# time: 2014-06-30 20:44:29 EDT +# mode: git +\tinit""" # Test various history related issues begin @@ -211,17 +214,19 @@ begin repl_mode = repl.interface.modes[1] shell_mode = repl.interface.modes[2] help_mode = repl.interface.modes[3] - histp = repl.interface.modes[4] - prefix_mode = repl.interface.modes[5] + git_mode = repl.interface.modes[4] + histp = repl.interface.modes[5] + prefix_mode = repl.interface.modes[6] hp = REPL.REPLHistoryProvider(Dict{Symbol,Any}(:julia => repl_mode, :shell => shell_mode, - :help => help_mode)) + :help => help_mode, + :git => git_mode)) REPL.hist_from_file(hp, IOBuffer(fakehistory)) REPL.history_reset_state(hp) - histp.hp = repl_mode.hist = shell_mode.hist = help_mode.hist = hp + histp.hp = repl_mode.hist = shell_mode.hist = help_mode.hist = git_mode.hist = hp # Some manual setup s = LineEdit.init_state(repl.t, repl.interface) @@ -229,6 +234,9 @@ begin # Test that navigating history skips invalid modes # (in both directions) LineEdit.history_prev(s, hp) + @test LineEdit.mode(s) == git_mode + @test buffercontents(LineEdit.buffer(s)) == "init" + LineEdit.history_prev(s, hp) @test LineEdit.mode(s) == repl_mode @test buffercontents(LineEdit.buffer(s)) == "2 + 2" LineEdit.history_prev(s, hp) @@ -244,10 +252,16 @@ begin @test LineEdit.mode(s) == repl_mode @test buffercontents(LineEdit.buffer(s)) == "2 + 2" LineEdit.history_next(s, hp) + @test LineEdit.mode(s) == git_mode + @test buffercontents(LineEdit.buffer(s)) == "init" + LineEdit.history_next(s, hp) # Test that the same holds for prefix search ps = LineEdit.state(s, prefix_mode) LineEdit.history_prev_prefix(ps, hp, "") + @test ps.parent == git_mode + @test LineEdit.input_string(ps) == "init" + LineEdit.history_prev_prefix(ps, hp, "") @test ps.parent == repl_mode @test LineEdit.input_string(ps) == "2 + 2" LineEdit.history_prev_prefix(ps, hp, "") From 39aa02b67627920832cc75ba15ec3ccb1866211a Mon Sep 17 00:00:00 2001 From: Test User Date: Thu, 9 Jul 2015 01:26:15 -0400 Subject: [PATCH 0335/1938] tuple-based Oid --- base/libgit2/oid.jl | 27 ++++++++++++--------------- base/libgit2/types.jl | 20 ++++---------------- 2 files changed, 16 insertions(+), 31 deletions(-) diff --git a/base/libgit2/oid.jl b/base/libgit2/oid.jl index c1f7a91667333..59b91826c0fa6 100644 --- a/base/libgit2/oid.jl +++ b/base/libgit2/oid.jl @@ -2,6 +2,7 @@ Oid(id::Oid) = id Oid(ptr::Ptr{Oid}) = unsafe_load(ptr)::Oid +Oid(arr::Vector{UInt8}) = Oid(tuple(arr...)) function Oid(ptr::Ptr{UInt8}) if ptr == C_NULL @@ -62,20 +63,15 @@ function Oid{T<:GitObject}(obj::T) return Oid(obj.ptr) end -function hex(id::Oid) - _hexstr = zeros(UInt8, OID_HEXSZ) - ccall((:git_oid_nfmt, :libgit2), Void, - (Ptr{UInt8}, Csize_t, Ptr{Oid}), Ref(_hexstr), OID_HEXSZ, Ref(id)) - return bytestring(_hexstr) -end +Base.hex(id::Oid) = join([hex(i,2) for i in id.val]) -raw(id::Oid) = hex2bytes(hex(id)) +raw(id::Oid) = collect(id.val) Base.string(id::Oid) = hex(id) Base.show(io::IO, id::Oid) = print(io, "Oid($(string(id)))") -hash(id::Oid) = hash(hex(id)) +Base.hash(id::Oid) = hash(id.val) cmp(id1::Oid, id2::Oid) = int(ccall((:git_oid_cmp, :libgit2), Cint, (Ptr{Oid}, Ptr{Oid}), Ref(id1), Ref(id2))) @@ -83,14 +79,15 @@ cmp(id1::Oid, id2::Oid) = int(ccall((:git_oid_cmp, :libgit2), Cint, Base.isequal(id1::Oid, id2::Oid) = cmp(id1, id2) == 0 Base.isless(id1::Oid, id2::Oid) = cmp(id1, id2) < 0 -iszero(id::Oid) = begin - bytes = raw(id) - for i=1:OID_RAWSZ - if bytes[i] != zero(UInt8) - return false - end +function iszero(id::Oid) + for i in 1:OID_RAWSZ + id.val[i] != zero(UInt8) && return false end return true end -#randoid() = bytestring(rand("0123456789abcdef".data,OID_HEXSZ)) +Base.zero(::Type{Oid}) = Oid() + +Base.rand(::Type{Oid}) = Oid(ntuple(x->rand(UInt8), OID_RAWSZ)) + +Base.isequal(id1::Oid, id2::Oid) = id1.val == id2.val diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index c2e3eacc9d67d..8c57843bbbb66 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -6,23 +6,11 @@ const OID_RAWSZ = 20 const OID_HEXSZ = OID_RAWSZ * 2 const OID_MINPREFIXLEN = 4 -#TODO: The Oid generated code should now use an immutable type that wraps an ntuple of length 20 -# immutable Oid -# id1::UInt8 -# id2::UInt8 -# ... -# id20::UInt8 -# end -@eval begin - $(Expr(:type, false, :Oid, - Expr(:block, - [Expr(:(::), symbol("id$i"), :UInt8) for i=1:OID_RAWSZ]...))) -end - -# default Oid constructor (all zeros) -@generated function Oid() - return Expr(:call, :Oid, [:(zero(UInt8)) for _=1:OID_RAWSZ]...) +immutable Oid + val::NTuple{OID_RAWSZ, UInt8} + Oid(val::NTuple{OID_RAWSZ, UInt8}) = new(val) end +Oid() = Oid(ntuple(i->zero(UInt8), OID_RAWSZ)) immutable TimeStruct time::Int64 # time in seconds from epoch From 8b5dd7826faa4802629850c967f67a164f95db42 Mon Sep 17 00:00:00 2001 From: Test User Date: Fri, 10 Jul 2015 00:46:09 -0400 Subject: [PATCH 0336/1938] added `Oid` tests, modified `free` with tuple parameters, updated `git_reference` calls to 0.23, fixed `branch!`, code cleanup --- base/libgit2.jl | 29 ++++++++------- base/libgit2/reference.jl | 74 ++++++++++++++------------------------- base/libgit2/types.jl | 11 ------ base/pkg/entry.jl | 48 +++++++++++++------------ test/libgit2.jl | 14 ++++++++ 5 files changed, 81 insertions(+), 95 deletions(-) diff --git a/base/libgit2.jl b/base/libgit2.jl index 2898f53066c74..3c9f106c88fed 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -3,6 +3,7 @@ module LibGit2 import Base: merge!, cat +import ..Pkg export with, GitRepo, GitConfig @@ -115,7 +116,7 @@ function set_remote_url(repo::GitRepo, url::AbstractString; remote::AbstractStri with(GitConfig, repo) do cfg set!(cfg, "remote.$remote.url", url) - m = match(GITHUB_REGEX,url) + m = match(Pkg.GitHub.GITHUB_REGEX,url) if m != nothing push = "git@github.com:$(m.captures[1]).git" if push != url @@ -213,11 +214,10 @@ function branch!(repo::GitRepo, branch_name::AbstractString, if branch_ref == nothing # if commit is empty get head commit oid commit_id = if isempty(commit) - head_ref = head(repo) - try - peel(head_ref, GitConst.OBJ_COMMIT) - finally - finalize(head_ref) + with(head(repo)) do head_ref + with(peel(GitCommit, head_ref)) do hrc + Oid(hrc) + end end else Oid(commit) @@ -226,9 +226,7 @@ function branch!(repo::GitRepo, branch_name::AbstractString, cmt = get(GitCommit, repo, commit_id) try - branch_ref = create_branch(repo, cmt, branch_name, - force=force, - msg="pkg.libgit2.branch: moving to $branch_name") + branch_ref = create_branch(repo, cmt, branch_name, force=force) finally finalize(cmt) end @@ -245,6 +243,11 @@ function branch!(repo::GitRepo, branch_name::AbstractString, end end + # checout selected branch + with(peel(GitTree, branch_ref)) do btree + checkout_tree(repo, btree) + end + # switch head to the branch set_head && head!(repo, branch_ref) finally @@ -284,7 +287,7 @@ function checkout!(repo::GitRepo, commit::AbstractString = ""; obj_oid = Oid(peeled) ref = create_reference(repo, obj_oid, force=force, - msg="pkg.libgit2.checkout: moving from $head_name to $(string(obj_oid))") + msg="libgit2.checkout: moving from $head_name to $(string(obj_oid))") finalize(ref) # checkout commit @@ -299,14 +302,14 @@ end """ git clone [-b ] [--bare] """ function clone(repo_url::AbstractString, repo_path::AbstractString; - checkout_branch::AbstractString="", + branch::AbstractString="", isbare::Bool = false, remote_cb::Ptr{Void} = C_NULL) # setup colne options clone_opts = CloneOptions( bare = Int32(isbare), - checkout_branch = isempty(checkout_branch) ? Cstring_NULL : - convert(Cstring, pointer(checkout_branch)), + checkout_branch = isempty(branch) ? Cstring_NULL : + convert(Cstring, pointer(branch)), remote_cb = remote_cb ) return clone(repo_url, repo_path, clone_opts) diff --git a/base/libgit2/reference.jl b/base/libgit2/reference.jl index 780eac18eea6f..99a7f4d941848 100644 --- a/base/libgit2/reference.jl +++ b/base/libgit2/reference.jl @@ -3,15 +3,15 @@ function GitReference(repo::GitRepo, ref_name::AbstractString) ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_reference_lookup, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}), - ref_ptr_ptr, repo.ptr, ref_name) + (Ptr{Ptr{Void}}, Ptr{Void}, Cstring), + ref_ptr_ptr, repo.ptr, ref_name) return GitReference(ref_ptr_ptr[]) end function head(repo::GitRepo) head_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_repository_head, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}), head_ptr_ptr, repo.ptr) + (Ptr{Ptr{Void}}, Ptr{Void}), head_ptr_ptr, repo.ptr) return GitReference(head_ptr_ptr[]) end @@ -43,16 +43,17 @@ end function branch(ref::GitReference) isempty(ref) && return "" - str_ptr_ptr = Ref{Ptr{UInt8}}(C_NULL) + str_ptr_ptr = Ref(LibGit2.Cstring_NULL) @check ccall((:git_branch_name, :libgit2), Cint, - (Ref{Ptr{UInt8}}, Ptr{Void},), str_ptr_ptr, ref.ptr) + (Ptr{Cstring}, Ptr{Void},), str_ptr_ptr, ref.ptr) return bytestring(str_ptr_ptr[]) end -function peel(ref::GitReference, obj_type::Cint) +function peel{T <: GitObject}(::Type{T}, ref::GitReference) + git_otype = getobjecttype(T) obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_reference_peel, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Cint), obj_ptr_ptr, ref.ptr, obj_type) + (Ptr{Ptr{Void}}, Ptr{Void}, Cint), obj_ptr_ptr, ref.ptr, git_otype) if err == Error.ENOTFOUND[] return Oid() elseif err != Error.GIT_OK[] @@ -61,52 +62,32 @@ function peel(ref::GitReference, obj_type::Cint) end throw(Error.GitError(err)) end - id = Oid(obj_ptr_ptr[]) - finalize(GitAnyObject(obj_ptr_ptr[])) - return id + return T(obj_ptr_ptr[]) end function create_reference(repo::GitRepo, obj_oid::Oid, refname::AbstractString = GitConst.HEAD_FILE; force::Bool=false, msg::AbstractString="") - sig = default_signature(repo) ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - try - @check ccall((:git_reference_create, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{Oid}, Cint, Ptr{SignatureStruct}, Ptr{UInt8}), - ref_ptr_ptr, repo.ptr, refname, Ref(obj_oid), Cint(force), - sig.ptr, isempty(msg) ? Ptr{UInt8}(C_NULL) : msg) - finally - finalize(sig) - end + @check ccall((:git_reference_create, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{Oid}, Cint, Cstring), + ref_ptr_ptr, repo.ptr, refname, Ref(obj_oid), Cint(force), + isempty(msg) ? Cstring_NULL : msg) return GitReference(ref_ptr_ptr[]) end function create_branch(repo::GitRepo, commit_obj::GitCommit, bname::AbstractString; - force::Bool=false, msg::AbstractString="") - sig = default_signature(repo) + force::Bool=false) ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - try - @check ccall((:git_branch_create, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{Void}, Cint, Ptr{SignatureStruct}, Ptr{UInt8}), - ref_ptr_ptr, repo.ptr, bname, commit_obj.ptr, Cint(force), - sig.ptr, isempty(msg) ? Ptr{UInt8}(C_NULL) : msg) - finally - finalize(sig) - end + @check ccall((:git_branch_create, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Ptr{Void}, Cint), + ref_ptr_ptr, repo.ptr, bname, commit_obj.ptr, Cint(force)) return GitReference(ref_ptr_ptr[]) end -function head!(repo::GitRepo, ref::GitReference; msg::AbstractString="") +function head!(repo::GitRepo, ref::GitReference) ref_name = name(ref) - sig = default_signature(repo) - msg = "pkg.libgit2.head!: switched to $(branch(ref)) "*msg - try - @check ccall((:git_repository_set_head, :libgit2), Cint, - (Ptr{Void}, Ptr{UInt8}, Ptr{SignatureStruct}, Ptr{UInt8}), - repo.ptr, ref_name, sig.ptr, isempty(msg) ? Ptr{UInt8}(C_NULL) : msg) - finally - finalize(sig) - end + @check ccall((:git_repository_set_head, :libgit2), Cint, + (Ptr{Void}, Cstring), repo.ptr, ref_name) return ref end @@ -115,7 +96,7 @@ function lookup_branch(repo::GitRepo, branch_name::AbstractString, remote::Bool= branch_type = remote ? GitConst.BRANCH_REMOTE : GitConst.BRANCH_LOCAL err = ccall((:git_branch_lookup, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Cint), - ref_ptr_ptr, repo.ptr, branch_name, branch_type) + ref_ptr_ptr, repo.ptr, branch_name, branch_type) if err == Error.ENOTFOUND[] return nothing elseif err != Error.GIT_OK[] @@ -131,23 +112,20 @@ function upstream(ref::GitReference) isempty(ref) && return nothing ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_branch_upstream, :libgit2), Cint, - (Ref{Ptr{Void}}, Ptr{Void},), ref_ptr_ptr, ref.ptr) + (Ref{Ptr{Void}}, Ptr{Void},), ref_ptr_ptr, ref.ptr) return GitReference(ref_ptr_ptr[]) end function owner(ref::GitReference) repo_ptr = ccall((:git_reference_owner, :libgit2), Ptr{Void}, - (Ptr{Void},), ref.ptr) + (Ptr{Void},), ref.ptr) return GitRepo(repo_ptr) end function target!(ref::GitReference, new_oid::Oid; msg::AbstractString="") ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - repo - with(default_signature(owner(ref))) do sig - @check ccall((:git_reference_set_target, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}, Ptr{SignatureStruct}, Ptr{UInt8}), - ref_ptr_ptr, ref.ptr, Ref(new_oid), sig.ptr, isempty(msg) ? Ptr{UInt8}(C_NULL) : msg) - end + @check ccall((:git_reference_set_target, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}, Cstring), + ref_ptr_ptr, ref.ptr, Ref(new_oid), isempty(msg) ? Cstring_NULL : msg) return GitReference(ref_ptr_ptr[]) end diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index 8c57843bbbb66..d8dbc734411bb 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -27,17 +27,6 @@ SignatureStruct() = SignatureStruct(Ptr{UInt8}(0), Ptr{UInt8}(0), TimeStruct()) -# immutable StrArrayStruct -# strings::Ptr{Ptr{UInt8}} -# count::Csize_t -# end -# StrArrayStruct() = StrArrayStruct(Ptr{UInt8}(0), zero(Csize_t)) -# function Base.finalize(sa::StrArrayStruct) -# sa_ptr = Ref(sa) -# ccall((:git_strarray_free, :libgit2), Void, (Ptr{StrArrayStruct},), sa_ptr) -# return sa_ptr[] -# end - immutable StrArrayStruct strings::Ptr{Cstring} count::Csize_t diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 14d981a02e032..ecbb580ffb453 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -268,29 +268,31 @@ function free(pkg::AbstractString) end end -# function free(pkgs) -# try -# for pkg in pkgs -# ispath(pkg,".git") || error("$pkg is not a git repo") -# Read.isinstalled(pkg) || error("$pkg cannot be freed – not an installed package") -# avail = Read.available(pkg) -# isempty(avail) && error("$pkg cannot be freed – not a registered package") -# Git.dirty(dir=pkg) && error("$pkg cannot be freed – repo is dirty") -# info("Freeing $pkg") -# vers = sort!(collect(keys(avail)), rev=true) -# for ver in vers -# sha1 = avail[ver].sha1 -# Git.iscommit(sha1, dir=pkg) || continue -# Git.run(`checkout -q $sha1`, dir=pkg) -# break -# end -# isempty(Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail])) && continue -# error("can't find any registered versions of $pkg to checkout") -# end -# finally -# resolve() -# end -# end +function free(pkgs) + try + for pkg in pkgs + ispath(pkg,".git") || error("$pkg is not a git repo") + Read.isinstalled(pkg) || error("$pkg cannot be freed – not an installed package") + avail = Read.available(pkg) + isempty(avail) && error("$pkg cannot be freed – not a registered package") + with(GitRepo, pkg) do repo + LibGit2.isdirty(repo) && throw(PkgError("$pkg cannot be freed – repo is dirty")) + info("Freeing $pkg") + vers = sort!(collect(keys(avail)), rev=true) + for ver in vers + sha1 = avail[ver].sha1 + LibGit2.iscommit(sha1, repo) || continue + LibGit2.checkout!(repo, sha1) + break + end + end + isempty(Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail])) && continue + error("can't find any registered versions of $pkg to checkout") + end + finally + resolve() + end +end function pin(pkg::AbstractString, head::AbstractString) ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) diff --git a/test/libgit2.jl b/test/libgit2.jl index cb56e6d6c1bb2..49bdc3cc07460 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -223,4 +223,18 @@ temp_dir() do dir_cache end end +# Oid +begin + z = LibGit2.Oid() + r = rand(LibGit2.Oid) + @test LibGit2.iszero(z) + @test z == zero(LibGit2.Oid) + rs = string(r) + rr = LibGit2.raw(r) + @test r == LibGit2.Oid(rr) + @test r == LibGit2.Oid(rs) + @test r == LibGit2.Oid(pointer(rr)) + for i in 11:length(rr); rr[i] = 0; end + @test LibGit2.Oid(rr) == LibGit2.Oid(rs[1:20]) +end From 5a4aad0c024a33f2002e25c915b3c50d2bb76b02 Mon Sep 17 00:00:00 2001 From: wildart Date: Fri, 10 Jul 2015 11:28:41 -0400 Subject: [PATCH 0337/1938] fixed: string value retrieval from config, resource management in tests --- base/libgit2/config.jl | 29 +++++++++---- base/libgit2/types.jl | 12 ++++++ test/libgit2.jl | 94 +++++++++++++++++++++--------------------- 3 files changed, 81 insertions(+), 54 deletions(-) diff --git a/base/libgit2/config.jl b/base/libgit2/config.jl index 6ed5c5650af9d..43ba1b3e1bce1 100644 --- a/base/libgit2/config.jl +++ b/base/libgit2/config.jl @@ -16,18 +16,31 @@ function GitConfig(r::GitRepo) return GitConfig(cfg_ptr_ptr[]) end -function GitConfig() +function GitConfig(isglobal::Bool = true) cfg_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - ccall((:git_config_open_default, :libgit2), Cint, - (Ptr{Ptr{Void}}, ), cfg_ptr_ptr) - return GitConfig(cfg_ptr_ptr[]) + @check ccall((:git_config_open_default, :libgit2), Cint, + (Ptr{Ptr{Void}}, ), cfg_ptr_ptr) + cfg = GitConfig(cfg_ptr_ptr[]) + if isglobal + glb_cfg_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_config_open_global, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}), glb_cfg_ptr_ptr, cfg.ptr) + finalize(cfg) + cfg = GitConfig(glb_cfg_ptr_ptr[]) + end + return cfg end function get{T<:AbstractString}(::Type{T}, c::GitConfig, name::AbstractString) - str_ptr = Ref{Ptr{UInt8}}(C_NULL) - @check ccall((:git_config_get_string, :libgit2), Cint, #TODO: git_config_get_string_buf - (Ptr{Ptr{UInt8}}, Ptr{Void}, Cstring), str_ptr, c.ptr, name) - return bytestring(str_ptr[]) + # str_ptr = Ref{Ptr{UInt8}}(C_NULL) + # @check ccall((:git_config_get_string, :libgit2), Cint, #TODO: git_config_get_string_buf + # (Ptr{Ptr{UInt8}}, Ptr{Void}, Cstring), str_ptr, c.ptr, name) + buf_ptr = Ref(Buffer()) + @check ccall((:git_config_get_string_buf, :libgit2), Cint, + (Ptr{Buffer}, Ptr{Void}, Cstring), buf_ptr, c.ptr, name) + return with(buf_ptr[]) do buf + bytestring(buf.ptr, buf.size) + end end function get(::Type{Bool}, c::GitConfig, name::AbstractString) diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index d8dbc734411bb..e37b45661d089 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -38,6 +38,18 @@ function Base.finalize(sa::StrArrayStruct) return sa_ptr[] end +immutable Buffer + ptr::Ptr{Cchar} + asize::Csize_t + size::Csize_t +end +Buffer() = Buffer(Ptr{Cchar}(C_NULL), zero(Csize_t), zero(Csize_t)) +function Base.finalize(buf::Buffer) + buf_ptr = Ref(buf) + ccall((:git_buf_free, :libgit2), Void, (Ptr{Buffer},), buf_ptr) + return buf_ptr[] +end + immutable CheckoutOptions version::Cuint diff --git a/test/libgit2.jl b/test/libgit2.jl index 49bdc3cc07460..36bc9ef16537b 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -143,26 +143,26 @@ temp_dir() do dir_cache # signature temp_dir() do dir path = joinpath(dir, "Example") - repo = LibGit2.clone(path_cache, path) - oid = "129eb39c8e0817c616296d1ac5f2cd1cf4f8b312" - c = LibGit2.get(LibGit2.GitCommit, repo, oid) - auth = LibGit2.author(c) - cmtr = LibGit2.committer(c) - cmsg = LibGit2.message(c) - @test isa(auth, LibGit2.Signature) - @test length(auth.name) > 0 - @test isa(cmtr, LibGit2.Signature) - @test length(cmtr.email) > 0 - @test length(cmsg) > 0 - finalize(repo) - - sig = LibGit2.Signature("AAA", "AAA@BBB.COM", round(time(), 0), 0) - git_sig = convert(LibGit2.GitSignature, sig) - sig2 = LibGit2.Signature(git_sig) - finalize(git_sig) - @test sig.name == sig2.name - @test sig.email == sig2.email - @test sig.time == sig2.time + LibGit2.with(LibGit2.clone(path_cache, path)) do repo + oid = "129eb39c8e0817c616296d1ac5f2cd1cf4f8b312" + c = LibGit2.get(LibGit2.GitCommit, repo, oid) + auth = LibGit2.author(c) + cmtr = LibGit2.committer(c) + cmsg = LibGit2.message(c) + @test isa(auth, LibGit2.Signature) + @test length(auth.name) > 0 + @test isa(cmtr, LibGit2.Signature) + @test length(cmtr.email) > 0 + @test length(cmsg) > 0 + + sig = LibGit2.Signature("AAA", "AAA@BBB.COM", round(time(), 0), 0) + git_sig = convert(LibGit2.GitSignature, sig) + sig2 = LibGit2.Signature(git_sig) + finalize(git_sig) + @test sig.name == sig2.name + @test sig.email == sig2.email + @test sig.time == sig2.time + end end # revwalk @@ -187,39 +187,41 @@ temp_dir() do dir_cache temp_dir() do dir # clone repo path = joinpath(dir, "Example") - repo = LibGit2.clone(path_cache, path) - LibGit2.with(LibGit2.GitConfig, repo) do cfg - credentials!(cfg) - end - cp(joinpath(path, "REQUIRE"), joinpath(path, "CCC")) - cp(joinpath(path, "REQUIRE"), joinpath(path, "AAA")) - LibGit2.add!(repo, "AAA") - @test_throws ErrorException LibGit2.transact(repo) do repo - mv(joinpath(path, "REQUIRE"), joinpath(path, "BBB")) - LibGit2.add!(repo, "BBB") - oid = LibGit2.commit(repo, "test commit") - error("Force recovery") + LibGit2.with(LibGit2.clone(path_cache, path)) do repo + LibGit2.with(LibGit2.GitConfig, repo) do cfg + credentials!(cfg) + end + cp(joinpath(path, "REQUIRE"), joinpath(path, "CCC")) + cp(joinpath(path, "REQUIRE"), joinpath(path, "AAA")) + LibGit2.add!(repo, "AAA") + @test_throws ErrorException LibGit2.transact(repo) do repo + mv(joinpath(path, "REQUIRE"), joinpath(path, "BBB")) + LibGit2.add!(repo, "BBB") + oid = LibGit2.commit(repo, "test commit") + error("Force recovery") + end + @test isfile(joinpath(path, "AAA")) + @test isfile(joinpath(path, "CCC")) + @test !isfile(joinpath(path, "BBB")) + @test isfile(joinpath(path, "REQUIRE")) end - @test isfile(joinpath(path, "AAA")) - @test isfile(joinpath(path, "CCC")) - @test !isfile(joinpath(path, "BBB")) - @test isfile(joinpath(path, "REQUIRE")) end # branch temp_dir() do dir path = joinpath(dir, "Example") - repo = LibGit2.clone(path_cache, path) - LibGit2.with(LibGit2.GitConfig, repo) do cfg - credentials!(cfg) + LibGit2.with(LibGit2.clone(path_cache, path)) do repo + LibGit2.with(LibGit2.GitConfig, repo) do cfg + credentials!(cfg) + end + brnch = LibGit2.branch(repo) + @test brnch == "master" + + brnch_name = "test" + LibGit2.branch!(repo, brnch_name) + brnch = LibGit2.branch(repo) + @test brnch == brnch_name end - brnch = LibGit2.branch(repo) - @test brnch == "master" - - brnch_name = "test" - LibGit2.branch!(repo, brnch_name) - brnch = LibGit2.branch(repo) - @test brnch == brnch_name end end From c254abb5cd24c936dfbf30f6405aa4282cd8e4b3 Mon Sep 17 00:00:00 2001 From: wildart Date: Sun, 12 Jul 2015 21:12:55 -0400 Subject: [PATCH 0338/1938] fixed 0.23 compatibility errors, added more commands to git-repl --- base/libgit2.jl | 3 +- base/libgit2/const.jl | 41 +++++- base/libgit2/rebase.jl | 32 ++--- base/libgit2/repl.jl | 269 +++++++++++++++++++++++++++++++++---- base/libgit2/repository.jl | 4 +- base/libgit2/status.jl | 24 ++++ base/libgit2/strarray.jl | 2 +- base/libgit2/tag.jl | 4 +- base/libgit2/types.jl | 33 ++++- 9 files changed, 358 insertions(+), 54 deletions(-) create mode 100644 base/libgit2/status.jl diff --git a/base/libgit2.jl b/base/libgit2.jl index 3c9f106c88fed..46f754a448578 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -25,6 +25,7 @@ include("libgit2/tag.jl") include("libgit2/blob.jl") include("libgit2/diff.jl") include("libgit2/rebase.jl") +include("libgit2/status.jl") include("libgit2/repl.jl") include("libgit2/utils.jl") @@ -422,7 +423,7 @@ function rebase!(repo::GitRepo, upstream::AbstractString="", newbase::AbstractSt end finish(rbs, sig) catch err - abort(rbs, sig) + abort(rbs) finally finalize(rbs) end diff --git a/base/libgit2/const.jl b/base/libgit2/const.jl index 3ca2f7215e179..4a8fe9caf2a52 100644 --- a/base/libgit2/const.jl +++ b/base/libgit2/const.jl @@ -211,4 +211,43 @@ module GitConst const CLONE_LOCAL = Cint(1) const CLONE_NO_LOCAL = Cint(2) const CLONE_LOCAL_NO_LINKS = Cint(3) -end \ No newline at end of file + + # status + const STATUS_CURRENT = Cuint(0) + const STATUS_INDEX_NEW = Cuint(1 << 0) + const STATUS_INDEX_MODIFIED = Cuint(1 << 1) + const STATUS_INDEX_DELETED = Cuint(1 << 2) + const STATUS_INDEX_RENAMED = Cuint(1 << 3) + const STATUS_INDEX_TYPECHANGE = Cuint(1 << 4) + const STATUS_WT_NEW = Cuint(1 << 7) + const STATUS_WT_MODIFIED = Cuint(1 << 8) + const STATUS_WT_DELETED = Cuint(1 << 9) + const STATUS_WT_TYPECHANGE = Cuint(1 << 10) + const STATUS_WT_RENAMED = Cuint(1 << 11) + const STATUS_WT_UNREADABLE = Cuint(1 << 12) + const STATUS_IGNORED = Cuint(1 << 14) + const STATUS_CONFLICTED = Cuint(1 << 15) + + # status show + const STATUS_SHOW_INDEX_AND_WORKDIR = Cint(0) + const STATUS_SHOW_INDEX_ONLY = Cint(1) + const STATUS_SHOW_WORKDIR_ONLY = Cint(2) + + # status options + const STATUS_OPT_INCLUDE_UNTRACKED = Cuint(1 << 0) + const STATUS_OPT_INCLUDE_IGNORED = Cuint(1 << 1) + const STATUS_OPT_INCLUDE_UNMODIFIED = Cuint(1 << 2) + const STATUS_OPT_EXCLUDE_SUBMODULES = Cuint(1 << 3) + const STATUS_OPT_RECURSE_UNTRACKED_DIRS = Cuint(1 << 4) + const STATUS_OPT_DISABLE_PATHSPEC_MATCH = Cuint(1 << 5) + const STATUS_OPT_RECURSE_IGNORED_DIRS = Cuint(1 << 6) + const STATUS_OPT_RENAMES_HEAD_TO_INDEX = Cuint(1 << 7) + const STATUS_OPT_RENAMES_INDEX_TO_WORKDIR = Cuint(1 << 8) + const STATUS_OPT_SORT_CASE_SENSITIVELY = Cuint(1 << 9) + const STATUS_OPT_SORT_CASE_INSENSITIVELY = Cuint(1 << 10) + const STATUS_OPT_RENAMES_FROM_REWRITES = Cuint(1 << 11) + const STATUS_OPT_NO_REFRESH = Cuint(1 << 12) + const STATUS_OPT_UPDATE_INDEX = Cuint(1 << 13) + const STATUS_OPT_INCLUDE_UNREADABLE = Cuint(1 << 14) + const STATUS_OPT_INCLUDE_UNREADABLE_AS_UNTRACKED = Cuint(1 << 15) +end diff --git a/base/libgit2/rebase.jl b/base/libgit2/rebase.jl index 41d533a46e241..e4d688c844f88 100644 --- a/base/libgit2/rebase.jl +++ b/base/libgit2/rebase.jl @@ -2,19 +2,13 @@ function GitRebase(repo::GitRepo, branch::GitAnnotated, upstream::GitAnnotated; onto::Nullable{GitAnnotated}=Nullable{GitAnnotated}(), - sig::Nullable{GitSignature}=Nullable{GitSignature}(), opts::RebaseOptions = RebaseOptions()) rebase_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - sig_obj = isnull(sig) ? default_signature(repo) : Base.get(sig) - try - @check ccall((:git_rebase_init, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, - Ptr{Void}, Ptr{SignatureStruct}, Ptr{RebaseOptions}), - rebase_ptr_ptr, repo.ptr, branch.ptr, upstream.ptr, - isnull(onto) ? C_NULL : Base.get(onto).ptr, sig_obj.ptr, Ref(opts)) - finally - isnull(sig) && finalize(sig_obj) - end + @check ccall((:git_rebase_init, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, + Ptr{Void}, Ptr{SignatureStruct}, Ptr{RebaseOptions}), + rebase_ptr_ptr, repo.ptr, branch.ptr, upstream.ptr, + isnull(onto) ? C_NULL : Base.get(onto).ptr, Ref(opts)) return GitRebase(rebase_ptr_ptr[]) end @@ -34,12 +28,12 @@ function Base.getindex(rb::GitRebase, i::Csize_t) end Base.getindex(rb::GitRebase, i::Int) = getindex(rb, Csize_t(i)) -function Base.next(rb::GitRebase; opts::CheckoutOptions = CheckoutOptions()) +function Base.next(rb::GitRebase) rb_op_ptr_ptr = Ref{Ptr{RebaseOperation}}(C_NULL) try @check ccall((:git_rebase_next, :libgit2), Cint, - (Ptr{Ptr{RebaseOperation}}, Ptr{Void}, Ptr{CheckoutOptions}), - rb_op_ptr_ptr, rb.ptr, Ref(opts)) + (Ptr{Ptr{RebaseOperation}}, Ptr{Void}), + rb_op_ptr_ptr, rb.ptr) catch err err.code == Error.ITEROVER[] && return nothing rethrow(err) @@ -55,13 +49,13 @@ function commit(rb::GitRebase, sig::GitSignature) return oid_ptr[] end -function abort(rb::GitRebase, sig::GitSignature) +function abort(rb::GitRebase) return ccall((:git_rebase_abort, :libgit2), Csize_t, - (Ptr{Void}, Ptr{SignatureStruct}), rb.ptr, sig.ptr) + (Ptr{Void},), rb.ptr) end -function finish(rb::GitRebase, sig::GitSignature; opts::RebaseOptions = RebaseOptions()) +function finish(rb::GitRebase, sig::GitSignature) return ccall((:git_rebase_finish, :libgit2), Csize_t, - (Ptr{Void}, Ptr{SignatureStruct}, Ptr{RebaseOptions}), - rb.ptr, sig.ptr, Ref(opts)) + (Ptr{Void}, Ptr{SignatureStruct}), + rb.ptr, sig.ptr) end diff --git a/base/libgit2/repl.jl b/base/libgit2/repl.jl index 0246e9798ea7c..d78469ba3af41 100644 --- a/base/libgit2/repl.jl +++ b/base/libgit2/repl.jl @@ -4,16 +4,16 @@ # Process REPL command in git mode List of git commands: -- [/] add +- [x] add - [ ] branch - [x] clone - [ ] commit - [ ] fetch - [x] init - [x] log -- [ ] reset -- [ ] rm -- [/] status +- [x] reset +- [x] rm +- [x] status - [ ] tag """ @@ -24,7 +24,7 @@ function repl_cmd(ex) if cmd[1] == "init" if has_params && cmd[2] == "help" - repl_init_help() + println("usage: init [--bare]") return end try @@ -45,7 +45,7 @@ function repl_cmd(ex) finalize(repo) println("Cloned $repourl into $repopath") catch ex - repl_clone_help() + println("usage: clone [--bare] ") warn(ex) end else @@ -65,14 +65,29 @@ function repl_cmd(ex) repl_log(repo, msg_count) elseif cmd[1] == "status" if has_params && cmd[2] == "help" - repl_status_help() + println("usage: status [-long]") return end + repl_status(repo, "long" in cmd) elseif cmd[1] == "add" + if has_params && (cmd[2] == "help" || (cmd[2] == "force" && length(cmd) == 2)) + println("usage: add [force] ...") + return + end + force_add = has_params && cmd[2] == "force" + repl_add(repo, force_add, cmd[(force_add ? 3 : 2):end]) + elseif cmd[1] == "rm" + if has_params && cmd[2] == "help" + println("usage: rm ...") + return + end + repl_rm(repo, cmd[2:end]) + elseif cmd[1] == "reset" if has_params && cmd[2] == "help" - repl_add_help() + println("usage: reset ...") return end + repl_reset(repo, cmd[2:end]) else warn("unknown command: $ex. Use \'help\' command.") end @@ -84,36 +99,40 @@ end function repl_help() println("""List of git commands: -add\t Add file contents to the index -clone\t Clone a repository into a current directory -init\t Create an empty Git repository or reinitialize an existing one in current directory -log\t Show commit logs -status\t Show the working tree status + add\t Add file contents to the index + clone\t Clone a repository into a current directory + init\t Create an empty Git repository or reinitialize an existing one in current directory + log\t Show commit logs + reset\t\ Reset current HEAD to the specified state + rm\t Remove files from the working tree and from the index + status\t Show the working tree status For particular command parameters use: help """) # branch\t\tList, create, or delete branches # commit\t\tRecord changes to the repository # fetch\t\tDownload objects and refs from another repository -# reset\t\tReset current HEAD to the specified state -# rm\t\tRemove files from the working tree and from the index # tag\t\tCreate, list, delete or verify a tag object signed with GPG end -function repl_init_help() - println("usage: init [--bare]") -end - -function repl_clone_help() - println("usage: clone [--bare] ") +function repl_add{T<:AbstractString}(repo::GitRepo, force_add::Bool, files::Vector{T}) + forced = force_add ? GitConst.INDEX_ADD_FORCE : GitConst.INDEX_ADD_DEFAULT + add!(repo, files..., flags = forced) + return end -function repl_add_help() - println("usage: add []") +function repl_rm{T<:AbstractString}(repo::GitRepo, files::Vector{T}) + remove!(repo, files...) + return end -function repl_status_help() - println("usage: status") +function repl_reset{T<:AbstractString}(repo::GitRepo, files::Vector{T}) + with(head(repo)) do href + with(peel(GitCommit, href)) do hcommit + reset!(repo, Nullable(hcommit), files...) + end + end + return end function repl_log(repo::GitRepo, msg_count::Int) @@ -129,4 +148,202 @@ function repl_log(repo::GitRepo, msg_count::Int) println("Date:\t$(Dates.unix2datetime(msg[2].time))") println('\t',join(split(msg[3],'\n'),"\n\t")) end -end \ No newline at end of file +end + +function repl_status(repo::GitRepo, islong::Bool=false) + isbare(repo) && error("Cannot report status on bare repository") + + # Show branch + with(head(repo)) do bref + bname = shortname(bref) + if islong + println("# On branch: ", isempty(bname) ? "Not currently on any branch." : bname) + else + println("## ", isempty(bname) ? "HEAD (no branch)" : bname) + end + end + + header = changes_in_index = changed_in_workdir = rm_in_workdir = false + + # Show status + with(GitStatus, repo) do status + for i in 1:length(status) + s = status[i] + s.status == GitConst.STATUS_CURRENT && continue + + istatus = wstatus = " " + + if s.status & GitConst.STATUS_INDEX_NEW > 0 + istatus = islong ? "new file: " : "A" + end + if s.status & GitConst.STATUS_INDEX_MODIFIED > 0 + istatus = islong ? "modified: " : "M" + end + if s.status & GitConst.STATUS_INDEX_DELETED > 0 + istatus = islong ? "deleted: " : "D" + end + if s.status & GitConst.STATUS_INDEX_RENAMED > 0 + istatus = islong ? "renamed: " : "R" + end + if s.status & GitConst.STATUS_INDEX_TYPECHANGE > 0 + istatus = islong ? "typechange:" : "T" + end + + if s.status & GitConst.STATUS_WT_NEW > 0 + if istatus == " " && !islong + istatus = "?" + end + wstatus = "?" + end + if s.status & GitConst.STATUS_WT_MODIFIED > 0; wstatus = "M"; end + if s.status & GitConst.STATUS_WT_DELETED > 0 + wstatus = "D" + rm_in_workdir = true + end + if s.status & GitConst.STATUS_WT_RENAMED > 0; wstatus = "R"; end + if s.status & GitConst.STATUS_WT_TYPECHANGE > 0; wstatus = "T"; end + + if islong + istatus == " " && continue + else + if s.status & GitConst.STATUS_IGNORED > 0 + istatus = "!" + wstatus = "!" + end + istatus == "?" && wstatus == "?" && continue + end + + if islong + if !header + println(" Changes to be committed:"); + println(" (use \"reset ...\" to unstage)\n") + header = true + end + shi_diff = unsafe_load(convert(Ptr{DiffDelta}, s.head_to_index), 1) + old_path = shi_diff.old_file.path == Cstring_NULL ? "" : bytestring(shi_diff.old_file.path) + new_path = shi_diff.new_file.path == Cstring_NULL ? "" : bytestring(shi_diff.new_file.path) + + if !isempty(old_path) && !isempty(new_path) && old_path != new_path + println("\t$istatus $old_path -> $new_path") + else + println("\t$istatus $(isempty(old_path) ? new_path : old_path)") + end + else + a = b = c = "" + if s.head_to_index != C_NULL + shi_diff = unsafe_load(convert(Ptr{DiffDelta}, s.head_to_index), 1) + a = shi_diff.old_file.path == Cstring_NULL ? "" : bytestring(shi_diff.old_file.path) + b = shi_diff.new_file.path == Cstring_NULL ? "" : bytestring(shi_diff.new_file.path) + end + if s.index_to_workdir != C_NULL + siw_diff = unsafe_load(convert(Ptr{DiffDelta}, s.index_to_workdir), 1) + if isempty(a) + a = siw_diff.old_file.path == Cstring_NULL ? "" : bytestring(siw_diff.old_file.path) + end + if isempty(b) + b = siw_diff.old_file.path == Cstring_NULL ? "" : bytestring(siw_diff.old_file.path) + end + c = siw_diff.new_file.path == Cstring_NULL ? "" : bytestring(siw_diff.new_file.path) + end + + if istatus == "R" + if wstatus == "R" + println("$istatus$wstatus $a $b $c") + else + println("$istatus$wstatus $a $b") + end + else + if wstatus == "R" + println("$istatus$wstatus $a $c") + else + println("$istatus$wstatus $a") + end + end + end + end + + if islong + if header + changes_in_index = true + println("") + end + header = false + + # Print workdir changes to tracked files. + for i in 1:length(status) + s = status[i] + (s.status == GitConst.STATUS_CURRENT || s.index_to_workdir == C_NULL) && continue + + wstatus = "" + if s.status & GitConst.STATUS_WT_MODIFIED > 0; wstatus = "modified: "; end + if s.status & GitConst.STATUS_WT_DELETED > 0; wstatus = "deleted: "; end + if s.status & GitConst.STATUS_WT_RENAMED > 0; wstatus = "renamed: "; end + if s.status & GitConst.STATUS_WT_TYPECHANGE > 0; wstatus = "typechange:"; end + + isempty(wstatus) && continue + + if !header + println(" Changes not staged for commit:") + println(" (use \"add$(rm_in_workdir ? "/rm" : "") ...\" to update what will be committed)") + println(" (use \"checkout ...\" to discard changes in working directory)\n") + header = true + end + siw_diff = unsafe_load(convert(Ptr{DiffDelta}, s.index_to_workdir), 1) + old_path = siw_diff.old_file.path == Cstring_NULL ? "" : bytestring(siw_diff.old_file.path) + new_path = siw_diff.new_file.path == Cstring_NULL ? "" : bytestring(siw_diff.new_file.path) + + if !isempty(old_path) && !isempty(new_path) && old_path != new_path + println("\t$wstatus $old_path -> $new_path") + else + println("\t$wstatus $(isempty(old_path) ? new_path : old_path)") + end + end + + if header + changed_in_workdir = true + println("") + end + header = false + end + + # Print untracked files. + for i in 1:length(status) + s = status[i] + if s.status == GitConst.STATUS_WT_NEW + siw_diff = unsafe_load(convert(Ptr{DiffDelta}, s.index_to_workdir), 1) + + if islong + if !header + println(" Untracked files:") + println(" (use \"add ...\" to include in what will be committed)\n") + header = true + end + println("\t", bytestring(siw_diff.old_file.path)) + else + println("?? ", bytestring(siw_diff.old_file.path)) + end + end + end + + # Print ignored files (long version). + if islong + header = false + for i in 1:length(status) + s = status[i] + if s.status == GitConst.STATUS_IGNORED + if !header + println(" Ignored files:") + println(" (use \"add force ...\" to include in what will be committed)\n") + header = true + end + siw_diff = unsafe_load(convert(Ptr{DiffDelta}, s.index_to_workdir), 1) + println("\t", bytestring(siw_diff.old_file.path)) + end + end + + !changes_in_index && changed_in_workdir && println("no changes added to commit (use \"add\" and/or \"commit -a\")") + end + + end + return +end diff --git a/base/libgit2/repository.jl b/base/libgit2/repository.jl index 630c1c0e6363f..36b85630dd0d0 100644 --- a/base/libgit2/repository.jl +++ b/base/libgit2/repository.jl @@ -136,7 +136,7 @@ function checkout_head(repo::GitRepo; options::CheckoutOptions = CheckoutOptions repo.ptr, Ref(options)) end -function reset!(repo::GitRepo, obj::Nullable{GitAnyObject}, pathspecs::AbstractString...) +function reset!{T<:AbstractString, S<:GitObject}(repo::GitRepo, obj::Nullable{S}, pathspecs::T...) with(StrArrayStruct(pathspecs...)) do sa @check ccall((:git_reset_default, :libgit2), Cint, (Ptr{Void}, Ptr{Void}, Ptr{StrArrayStruct}), @@ -149,7 +149,7 @@ end function reset!(repo::GitRepo, obj::GitObject, mode::Cint; checkout_opts::CheckoutOptions = CheckoutOptions()) @check ccall((:git_reset, :libgit2), Cint, - (Ptr{Void}, Ptr{Void}, Cint, Ptr{CheckoutOptions}, Ptr{SignatureStruct}, Ptr{UInt8}), + (Ptr{Void}, Ptr{Void}, Cint, Ptr{CheckoutOptions}), repo.ptr, obj.ptr, mode, Ref(checkout_opts)) end diff --git a/base/libgit2/status.jl b/base/libgit2/status.jl new file mode 100644 index 0000000000000..1cc163f89756d --- /dev/null +++ b/base/libgit2/status.jl @@ -0,0 +1,24 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + +function GitStatus(repo::GitRepo; status_opts=StatusOptions()) + stat_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_status_list_new, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{StatusOptions}), + stat_ptr_ptr, repo.ptr, Ref(status_opts)) + return GitStatus(stat_ptr_ptr[]) +end + +function Base.length(status::GitStatus) + return Int(ccall((:git_status_list_entrycount, :libgit2), Csize_t, + (Ptr{Ptr{Void}}, ), status.ptr)) +end + +function Base.getindex(status::GitStatus, i::Csize_t) + entry_ptr = ccall((:git_status_byindex, :libgit2), Ptr{Void}, + (Ptr{Void}, Csize_t), status.ptr, i-1) + entry_ptr == C_NULL && return nothing + return unsafe_load(convert(Ptr{StatusEntry}, entry_ptr), 1) +end +Base.getindex(status::GitStatus, i::Int) = getindex(status, Csize_t(i)) + + diff --git a/base/libgit2/strarray.jl b/base/libgit2/strarray.jl index 9e5a4966abea7..aa1483980d45d 100644 --- a/base/libgit2/strarray.jl +++ b/base/libgit2/strarray.jl @@ -2,7 +2,7 @@ function StrArrayStruct{T<:AbstractString}(strs::T...) strcount = length(strs) - map(s->Base.unsafe_convert(Cstring, s), strs) # check for null-strings + map(s->Base.unsafe_convert(Cstring, bytestring(s)), strs) # check for null-strings sa_strings = convert(Ptr{Cstring}, Libc.malloc(sizeof(Cstring) * strcount)) for i=1:strcount len = length(strs[i]) diff --git a/base/libgit2/tag.jl b/base/libgit2/tag.jl index aa47cdf9653fc..b20642689cec6 100644 --- a/base/libgit2/tag.jl +++ b/base/libgit2/tag.jl @@ -11,7 +11,7 @@ end function tag_delete(repo::GitRepo, tag::AbstractString) @check ccall((:git_tag_delete, :libgit2), Cint, - (Ptr{Void}, Ptr{UInt8}, ), repo.ptr, tag) + (Ptr{Void}, Cstring, ), repo.ptr, tag) end function tag_create(repo::GitRepo, tag::AbstractString, commit::AbstractString; @@ -21,7 +21,7 @@ function tag_create(repo::GitRepo, tag::AbstractString, commit::AbstractString; with(get(GitCommit, repo, commit)) do commit_obj with(default_signature(repo)) do sig @check ccall((:git_tag_create, :libgit2), Cint, - (Ptr{Oid}, Ptr{Void}, Ptr{UInt8}, Ptr{Void}, Ptr{SignatureStruct}, Ptr{UInt8}, Cint), + (Ptr{Oid}, Ptr{Void}, Cstring, Ptr{Void}, Ptr{SignatureStruct}, Cstring, Cint), oid_ptr, repo.ptr, tag, commit_obj.ptr, sig.ptr, msg, Cint(force)) end end diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index e37b45661d089..a8272b20b7053 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -249,7 +249,7 @@ immutable DiffFile flags::Cuint mode::UInt16 end -DiffFile()=DiffFile(Oid(),convert(Cstring, Ptr{UInt8}(C_NULL)),Coff_t(0),Cuint(0),UInt16(0)) +DiffFile()=DiffFile(Oid(),Cstring_NULL,Coff_t(0),Cuint(0),UInt16(0)) immutable DiffDelta status::Cint @@ -330,8 +330,12 @@ immutable RebaseOptions version::Cuint quiet::Cint rewrite_notes_ref::Cstring + checkout_opts::CheckoutOptions end -RebaseOptions()=RebaseOptions(one(Cuint), Cint(1), Cstring_NULL) +RebaseOptions(; quiet::Cint = Cint(1), + rewrite_notes_ref::Cstring = Cstring_NULL, + checkout_opts::CheckoutOptions = CheckoutOptions() +)=RebaseOptions(one(Cuint), quiet, rewrite_notes_ref, checkout_opts) immutable RebaseOperation optype::Cint @@ -341,6 +345,30 @@ end RebaseOperation()=RebaseOperation(Cint(0), Oid(), Cstring_NULL) Base.show(io::IO, rbo::RebaseOperation) = print(io, "RebaseOperation($(string(rbo.id)))") +immutable StatusOptions + version::Cuint + show::Cint + flags::Cuint + pathspec::StrArrayStruct +end +StatusOptions(; show::Cint = GitConst.STATUS_SHOW_INDEX_AND_WORKDIR, + flags::Cuint = GitConst.STATUS_OPT_INCLUDE_UNTRACKED | + GitConst.STATUS_OPT_RECURSE_UNTRACKED_DIRS | + GitConst.STATUS_OPT_RENAMES_HEAD_TO_INDEX | + GitConst.STATUS_OPT_SORT_CASE_SENSITIVELY, + pathspec::StrArrayStruct = StrArrayStruct() +)=StatusOptions(one(Cuint), + show, + flags, + pathspec) + +immutable StatusEntry + status::Cuint + head_to_index::Ptr{DiffDelta} + index_to_workdir::Ptr{DiffDelta} +end +StatusEntry()=StatusEntry(Cuint(0), C_NULL, C_NULL) + # Abstract object types abstract AbstractGitObject Base.isempty(obj::AbstractGitObject) = (obj.ptr == C_NULL) @@ -364,6 +392,7 @@ for (typ, ref, sup, fnc) in ( (:GitRepo, :Void, :AbstractGitObject, :(:git_repository_free)), (:GitAnnotated, :Void, :AbstractGitObject, :(:git_annotated_commit_free)), (:GitRebase, :Void, :AbstractGitObject, :(:git_rebase_free)), + (:GitStatus, :Void, :AbstractGitObject, :(:git_status_list_free)), (:GitSignature, :SignatureStruct, :AbstractGitObject, :(:git_signature_free)), (:GitAnyObject, :Void, :GitObject, nothing), (:GitCommit, :Void, :GitObject, nothing), From f58f7627aea9b2abd383eefc40daf79348097958 Mon Sep 17 00:00:00 2001 From: wildart Date: Sun, 12 Jul 2015 22:58:44 -0400 Subject: [PATCH 0339/1938] used new enums, changed location of GITHUB_REGEX, exported PkgError --- base/libgit2.jl | 10 ++++++---- base/libgit2/error.jl | 1 - base/libgit2/rebase.jl | 2 +- base/libgit2/reference.jl | 8 ++++---- base/libgit2/repository.jl | 10 +++++----- base/libgit2/walker.jl | 4 ++-- base/pkg.jl | 2 +- base/pkg/github.jl | 5 +---- base/pkg/resolve.jl | 3 ++- test/resolve.jl | 2 +- 10 files changed, 23 insertions(+), 24 deletions(-) diff --git a/base/libgit2.jl b/base/libgit2.jl index 46f754a448578..b74f11adb46e7 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -3,10 +3,12 @@ module LibGit2 import Base: merge!, cat -import ..Pkg export with, GitRepo, GitConfig +const GITHUB_REGEX = + r"^(?:git@|git://|https://(?:[\w\.\+\-]+@)?)github.com[:/](([^/].+)/(.+?))(?:\.git)?$"i + include("libgit2/const.jl") include("libgit2/types.jl") include("libgit2/error.jl") @@ -117,7 +119,7 @@ function set_remote_url(repo::GitRepo, url::AbstractString; remote::AbstractStri with(GitConfig, repo) do cfg set!(cfg, "remote.$remote.url", url) - m = match(Pkg.GitHub.GITHUB_REGEX,url) + m = match(GITHUB_REGEX,url) if m != nothing push = "git@github.com:$(m.captures[1]).git" if push != url @@ -357,9 +359,9 @@ function revcount(repo::GitRepo, fst::AbstractString, snd::AbstractString) snd_id = revparseid(repo, snd) base_id = merge_base(string(fst_id), string(snd_id), repo) fc = count((i,r)->i!=base_id, repo, oid=fst_id, - by=Pkg.LibGit2.GitConst.SORT_TOPOLOGICAL) + by=GitConst.SORT_TOPOLOGICAL) sc = count((i,r)->i!=base_id, repo, oid=snd_id, - by=Pkg.LibGit2.GitConst.SORT_TOPOLOGICAL) + by=GitConst.SORT_TOPOLOGICAL) return (fc-1, sc-1) end diff --git a/base/libgit2/error.jl b/base/libgit2/error.jl index 3c34b367a67f2..193849f9ac622 100644 --- a/base/libgit2/error.jl +++ b/base/libgit2/error.jl @@ -23,7 +23,6 @@ module Error EPEEL = Cint(-19), # the requested peel operation is not possible PASSTHROUGH = Cint(-30), # internal only ITEROVER = Cint(-31)) # signals end of iteration -Base.getindex(c::Code) = c.val @enum(Class, None, NoMemory, diff --git a/base/libgit2/rebase.jl b/base/libgit2/rebase.jl index e4d688c844f88..384aac8142cb2 100644 --- a/base/libgit2/rebase.jl +++ b/base/libgit2/rebase.jl @@ -35,7 +35,7 @@ function Base.next(rb::GitRebase) (Ptr{Ptr{RebaseOperation}}, Ptr{Void}), rb_op_ptr_ptr, rb.ptr) catch err - err.code == Error.ITEROVER[] && return nothing + err.code == Int(Error.ITEROVER) && return nothing rethrow(err) end return unsafe_load(convert(Ptr{RebaseOperation}, rb_op_ptr_ptr[]), 1) diff --git a/base/libgit2/reference.jl b/base/libgit2/reference.jl index 99a7f4d941848..6447285b58bf6 100644 --- a/base/libgit2/reference.jl +++ b/base/libgit2/reference.jl @@ -54,9 +54,9 @@ function peel{T <: GitObject}(::Type{T}, ref::GitReference) obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_reference_peel, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Cint), obj_ptr_ptr, ref.ptr, git_otype) - if err == Error.ENOTFOUND[] + if err == Int(Error.ENOTFOUND) return Oid() - elseif err != Error.GIT_OK[] + elseif err != Int(Error.GIT_OK) if obj_ptr_ptr[] != C_NULL finalize(GitAnyObject(obj_ptr_ptr[])) end @@ -97,9 +97,9 @@ function lookup_branch(repo::GitRepo, branch_name::AbstractString, remote::Bool= err = ccall((:git_branch_lookup, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Cint), ref_ptr_ptr, repo.ptr, branch_name, branch_type) - if err == Error.ENOTFOUND[] + if err == Int(Error.ENOTFOUND) return nothing - elseif err != Error.GIT_OK[] + elseif err != Int(Error.GIT_OK) if repo_ptr_ptr[] != C_NULL finalize(GitReference(ref_ptr_ptr[])) end diff --git a/base/libgit2/repository.jl b/base/libgit2/repository.jl index 36b85630dd0d0..df285627147b0 100644 --- a/base/libgit2/repository.jl +++ b/base/libgit2/repository.jl @@ -4,7 +4,7 @@ function GitRepo(path::AbstractString) repo_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_repository_open, :libgit2), Cint, (Ptr{Ptr{Void}}, Cstring), repo_ptr_ptr, path) - if err != Error.GIT_OK[] + if err != Int(Error.GIT_OK) if repo_ptr_ptr[] != C_NULL finalize(GitRepo(repo_ptr_ptr[])) end @@ -73,9 +73,9 @@ function get{T <: GitObject}(::Type{T}, r::GitRepo, oid::Oid, oid_size::Int=OID_ (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Oid}, Cint), obj_ptr_ptr, r.ptr, id_ptr, git_otype) end - if err == Error.ENOTFOUND[] + if err == Int(Error.ENOTFOUND) return nothing - elseif err != Error.GIT_OK[] + elseif err != Int(Error.GIT_OK) if obj_ptr_ptr[] != C_NULL finalize(GitAnyObject(obj_ptr_ptr[])) end @@ -103,9 +103,9 @@ function peel(obj::GitObject, obj_type::Cint) git_otype = getobjecttype(obj_type) err = ccall((:git_object_peel, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Cint), peeled_ptr_ptr, obj.ptr, obj_type) - if err == Error.ENOTFOUND[] + if err == Int(Error.ENOTFOUND) return Oid() - elseif err != Error.GIT_OK[] + elseif err != Int(Error.GIT_OK) if peeled_ptr_ptr[] != C_NULL finalize(GitAnyObject(peeled_ptr_ptr[])) end diff --git a/base/libgit2/walker.jl b/base/libgit2/walker.jl index 18ab534a9ede0..f07f1b30c3f89 100644 --- a/base/libgit2/walker.jl +++ b/base/libgit2/walker.jl @@ -11,7 +11,7 @@ function Base.start(w::GitRevWalker) id_ptr = Ref(Oid()) err = ccall((:git_revwalk_next, :libgit2), Cint, (Ptr{Oid}, Ptr{Void}), id_ptr, w.ptr) - err != Error.GIT_OK[] && return (nothing, true) + err != Int(Error.GIT_OK) && return (nothing, true) return (id_ptr[], false) end @@ -21,7 +21,7 @@ function Base.next(w::GitRevWalker, state) id_ptr = Ref(Oid()) err = ccall((:git_revwalk_next, :libgit2), Cint, (Ptr{Oid}, Ptr{Void}), id_ptr, w.ptr) - err != Error.GIT_OK[] && return (state[1], (nothing, true)) + err != Int(Error.GIT_OK) && return (state[1], (nothing, true)) return (state[1], (id_ptr[], false)) end diff --git a/base/pkg.jl b/base/pkg.jl index 5f5a153bc071c..3cf3872c66969 100644 --- a/base/pkg.jl +++ b/base/pkg.jl @@ -5,7 +5,7 @@ module Pkg export Dir, GitHub, Types, Reqs, Cache, Read, Query, Resolve, Write, Generate, Entry export dir, init, rm, add, available, installed, status, clone, checkout, update, resolve, register, tag, publish, generate, test, - build, free, pin + build, free, pin, PkgError const DEFAULT_META = "https://github.com/JuliaLang/METADATA.jl" const META_BRANCH = "metadata-v2" diff --git a/base/pkg/github.jl b/base/pkg/github.jl index 62df505239cbd..f0f449d512ce2 100644 --- a/base/pkg/github.jl +++ b/base/pkg/github.jl @@ -4,9 +4,6 @@ module GitHub import Main, ...LibGit2, ..Dir, ...Pkg.PkgError -const GITHUB_REGEX = - r"^(?:git@|git://|https://(?:[\w\.\+\-]+@)?)github.com[:/](([^/].+)/(.+?))(?:\.git)?$"i - const AUTH_NOTE = "Julia Package Manager" const AUTH_DATA = Dict{Any,Any}( "scopes" => ["repo"], @@ -153,7 +150,7 @@ function fork(owner::AbstractString, repo::AbstractString) end function normalize_url(url::AbstractString) - m = match(GITHUB_REGEX,url) + m = match(LibGit2.GITHUB_REGEX,url) m == nothing ? url : "https://github.com/$(m.captures[1]).git" end diff --git a/base/pkg/resolve.jl b/base/pkg/resolve.jl index f3a7a59f8f29c..d0b2005fece0f 100644 --- a/base/pkg/resolve.jl +++ b/base/pkg/resolve.jl @@ -6,7 +6,8 @@ include("resolve/versionweight.jl") include("resolve/interface.jl") include("resolve/maxsum.jl") -using ..Types, ..Query, .PkgToMaxSumInterface, .MaxSum, ...Pkg.PkgError +using ..Types, ..Query, .PkgToMaxSumInterface, .MaxSum +import ...Pkg.PkgError export resolve, sanity_check diff --git a/test/resolve.jl b/test/resolve.jl index 94c99b691fb7c..ffc9aad485fe8 100644 --- a/test/resolve.jl +++ b/test/resolve.jl @@ -4,7 +4,7 @@ using Base.Pkg.Types using Base.Pkg.Query using Base.Pkg.Resolve using Base.Pkg.Resolve.VersionWeights -using Base.Pkg.PkgError +import Base.Pkg.PkgError # Check that VersionWeight keeps the same ordering as VersionNumber From fad9263137a3f470a9b06e71d2d38ae4fdfad46b Mon Sep 17 00:00:00 2001 From: wildart Date: Mon, 13 Jul 2015 21:17:59 -0400 Subject: [PATCH 0340/1938] switch to release build for windows --- deps/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/Makefile b/deps/Makefile index 51c2389c376d4..319de7803d3de 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -2016,7 +2016,7 @@ LIBGIT2_OBJ_TARGET := $(build_shlibdir)/libgit2.$(SHLIB_EXT) LIBGIT2_OPTS := $(CMAKE_COMMON) -DTHREADSAFE=ON ifeq ($(OS),WINNT) -LIBGIT2_OPTS += -DWIN32=ON -DMINGW=ON -DUSE_SSH=OFF -DCMAKE_SYSTEM_NAME=Windows +LIBGIT2_OPTS += -DWIN32=ON -DMINGW=ON -DUSE_SSH=OFF -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_BUILD_TYPE=RelWithDebInfo ifneq ($(ARCH),x86_64) LIBGIT2_OPTS += -DCMAKE_C_FLAGS="-mincoming-stack-boundary=2" endif From 93af677e0b7846338c239ae77b6b394e5fdf5fa0 Mon Sep 17 00:00:00 2001 From: wildart Date: Mon, 27 Jul 2015 18:35:07 -0400 Subject: [PATCH 0341/1938] diff & rebase update to v0.23, error fixes --- base/libgit2.jl | 2 +- base/libgit2/const.jl | 6 ++++++ base/libgit2/diff.jl | 2 +- base/libgit2/rebase.jl | 2 +- base/libgit2/types.jl | 14 +++++++------- base/pkg/entry.jl | 6 +++--- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/base/libgit2.jl b/base/libgit2.jl index b74f11adb46e7..5910cbc130fa7 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -418,7 +418,7 @@ function rebase!(repo::GitRepo, upstream::AbstractString="", newbase::AbstractSt try sig = default_signature(repo) try - rbs = GitRebase(repo, head_ann, upst_ann, sig=Nullable(sig)) + rbs = GitRebase(repo, head_ann, upst_ann) try while (rbs_op = next(rbs)) != nothing commit(rbs, sig) diff --git a/base/libgit2/const.jl b/base/libgit2/const.jl index 4a8fe9caf2a52..ddb597bf88c49 100644 --- a/base/libgit2/const.jl +++ b/base/libgit2/const.jl @@ -250,4 +250,10 @@ module GitConst const STATUS_OPT_UPDATE_INDEX = Cuint(1 << 13) const STATUS_OPT_INCLUDE_UNREADABLE = Cuint(1 << 14) const STATUS_OPT_INCLUDE_UNREADABLE_AS_UNTRACKED = Cuint(1 << 15) + + @enum(GIT_SUBMODULE_IGNORE, SUBMODULE_IGNORE_UNSPECIFIED = -1, # use the submodule's configuration + SUBMODULE_IGNORE_NONE = 1, # any change or untracked == dirty + SUBMODULE_IGNORE_UNTRACKED = 2, # dirty if tracked files change + SUBMODULE_IGNORE_DIRTY = 3, # only dirty if HEAD moved + SUBMODULE_IGNORE_ALL = 4) # never dirty end diff --git a/base/libgit2/diff.jl b/base/libgit2/diff.jl index 221098567d49c..d515b8a223b1b 100644 --- a/base/libgit2/diff.jl +++ b/base/libgit2/diff.jl @@ -4,7 +4,7 @@ function diff_tree(repo::GitRepo, tree::GitTree, pathspecs::AbstractString=""; c emptypathspec = isempty(pathspecs) diff_ptr_ptr = Ref{Ptr{Void}}(C_NULL) if !emptypathspec - sa = StrArrayStruct(paths) + sa = StrArrayStruct(pathspecs) diff_opts = DiffOptionsStruct(pathspec = sa) end try diff --git a/base/libgit2/rebase.jl b/base/libgit2/rebase.jl index 384aac8142cb2..68ce79879005b 100644 --- a/base/libgit2/rebase.jl +++ b/base/libgit2/rebase.jl @@ -6,7 +6,7 @@ function GitRebase(repo::GitRepo, branch::GitAnnotated, upstream::GitAnnotated; rebase_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_rebase_init, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, - Ptr{Void}, Ptr{SignatureStruct}, Ptr{RebaseOptions}), + Ptr{Void}, Ptr{RebaseOptions}), rebase_ptr_ptr, repo.ptr, branch.ptr, upstream.ptr, isnull(onto) ? C_NULL : Base.get(onto).ptr, Ref(opts)) return GitRebase(rebase_ptr_ptr[]) diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index a8272b20b7053..df5620907d4db 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -214,20 +214,20 @@ immutable DiffOptionsStruct interhunk_lines::UInt32 id_abbrev::UInt16 max_size::Coff_t - old_prefix::Ptr{UInt8} - new_prefix::Ptr{UInt8} + old_prefix::Cstring + new_prefix::Cstring end DiffOptionsStruct(; flags::UInt32 = GitConst.DIFF_NORMAL, - ignore_submodules::Cint = GitConst.SUBMODULE_IGNORE_DEFAULT, + ignore_submodules::Cint = Cint(GitConst.SUBMODULE_IGNORE_UNSPECIFIED), pathspec::StrArrayStruct = StrArrayStruct(), - notify_cb::Ptr{Void} = Ptr{Void}(0), - notify_payload::Ptr{Void} = Ptr{Void}(0), + notify_cb::Ptr{Void} = C_NULL, + notify_payload::Ptr{Void} = C_NULL, context_lines::UInt32 = UInt32(3), interhunk_lines::UInt32 = zero(UInt32), id_abbrev::UInt16 = UInt16(7), max_size::Coff_t = Coff_t(512*1024*1024), #zero(Coff_t), #512Mb - old_prefix::Ptr{UInt8} = Ptr{UInt8}(0), - new_prefix::Ptr{UInt8} = Ptr{UInt8}(0) + old_prefix::Cstring = Cstring_NULL, + new_prefix::Cstring = Cstring_NULL )=DiffOptionsStruct(GitConst.DIFF_OPTIONS_VERSION, flags, ignore_submodules, diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index ecbb580ffb453..dcf7b397528e5 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -481,8 +481,8 @@ function resolve( throw(PkgError("$pkg can't be installed because it has no versions that support ", VERSION, " of julia. " * "You may need to update METADATA by running `Pkg.update()`")) else - throw(PkgError("$pkg's requirements can't be satisfied because of the following fixed packages: ", - join(conflicts[pkg], ", ", " and "))) + sconflicts = join(conflicts[pkg], ", ", " and ") + throw(PkgError("$pkg's requirements can't be satisfied because of the following fixed packages: $sconflicts")) end end end @@ -625,7 +625,7 @@ function register(pkg::AbstractString) throw(PkgError("$pkg: $err")) end !isempty(url) || throw(PkgError("$pkg: no URL configured")) - register(pkg, LibGit2.normalize_url(url)) + register(pkg, GitHub.normalize_url(url)) end function isrewritable(v::VersionNumber) From 1347d68085f0dfbccd2aadd1a9b59bd5928c2485 Mon Sep 17 00:00:00 2001 From: wildart Date: Mon, 27 Jul 2015 21:05:29 -0400 Subject: [PATCH 0342/1938] added callbacks, updated push & publish --- base/libgit2.jl | 28 +++++------------------ base/libgit2/callbacks.jl | 36 ++++++++++++++++++++++++++++++ base/libgit2/remote.jl | 17 +++++++++----- base/libgit2/types.jl | 47 ++++++++++++++++++++++++++------------- base/pkg/entry.jl | 13 +++++------ 5 files changed, 91 insertions(+), 50 deletions(-) create mode 100644 base/libgit2/callbacks.jl diff --git a/base/libgit2.jl b/base/libgit2.jl index 5910cbc130fa7..825865b53f60d 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -28,6 +28,7 @@ include("libgit2/blob.jl") include("libgit2/diff.jl") include("libgit2/rebase.jl") include("libgit2/status.jl") +include("libgit2/callbacks.jl") include("libgit2/repl.jl") include("libgit2/utils.jl") @@ -112,7 +113,7 @@ end function is_ancestor_of(a::AbstractString, b::AbstractString, repo::GitRepo) A = revparseid(repo, a) - merge_base(a, b, repo) == A + merge_base(repo, a, b) == A end function set_remote_url(repo::GitRepo, url::AbstractString; remote::AbstractString="origin") @@ -135,26 +136,6 @@ function set_remote_url(path::AbstractString, url::AbstractString; remote::Abstr end end -function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, name::Cstring, url::Cstring, payload::Ptr{Void}) - # Create the remote with a mirroring url - fetch_spec = "+refs/*:refs/*" - err = ccall((:git_remote_create_with_fetchspec, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring, Cstring), - remote, repo_ptr, name, url, fetch_spec) - err != 0 && return Cint(err) - - # And set the configuration option to true for the push command - config = GitConfig(GitRepo(repo_ptr)) - name_str = bytestring(name) - err= try set!(config, "remote.$name_str.mirror", true) - catch -1 - finally finalize(config) - end - err != 0 && return Cint(err) - return Cint(0) -end -const mirror_cb = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring, Ptr{Void})) - """ git fetch [|] []""" function fetch{T<:AbstractString}(repo::GitRepo; remote::AbstractString="origin", @@ -186,7 +167,8 @@ function push{T<:AbstractString}(repo::GitRepo; GitRemoteAnon(repo, remoteurl) end try - push(rmt, refspecs, force=force, msg="to $(url(rmt))") + po = PushOptions(callbacks=RemoteCallbacks(credentials=credentials_cb)) + push(rmt, refspecs, force=force, push_opts=po) catch err warn("push: $err") finally @@ -357,7 +339,7 @@ end function revcount(repo::GitRepo, fst::AbstractString, snd::AbstractString) fst_id = revparseid(repo, fst) snd_id = revparseid(repo, snd) - base_id = merge_base(string(fst_id), string(snd_id), repo) + base_id = merge_base(repo, string(fst_id), string(snd_id)) fc = count((i,r)->i!=base_id, repo, oid=fst_id, by=GitConst.SORT_TOPOLOGICAL) sc = count((i,r)->i!=base_id, repo, oid=snd_id, diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl new file mode 100644 index 0000000000000..5c5d7fc368eb2 --- /dev/null +++ b/base/libgit2/callbacks.jl @@ -0,0 +1,36 @@ +function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, + name::Cstring, url::Cstring, payload::Ptr{Void}) + # Create the remote with a mirroring url + fetch_spec = "+refs/*:refs/*" + err = ccall((:git_remote_create_with_fetchspec, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring, Cstring), + remote, repo_ptr, name, url, fetch_spec) + err != 0 && return Cint(err) + + # And set the configuration option to true for the push command + config = GitConfig(GitRepo(repo_ptr)) + name_str = bytestring(name) + err= try set!(config, "remote.$name_str.mirror", true) + catch -1 + finally finalize(config) + end + err != 0 && return Cint(err) + return Cint(0) +end + +function credentials_callback(cred::Ptr{Ptr{Void}}, url::Cstring, + username_from_url::Cstring, + allowed_types::Cuint, payload::Ptr{Void}) + # Try ssh-agent first + err = ccall((:git_cred_ssh_key_from_agent, :libgit2), Cint, + (Ptr{Ptr{Void}}, Cstring), + cred, username_from_url) + err == 0 && return Cint(0) + + # TODO: deal with other auth types + WIN + + return Cint(1) +end + +const mirror_cb = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring, Ptr{Void})) +const credentials_cb = cfunction(credentials_callback, Cint, (Ptr{Ptr{Void}}, Cstring, Cstring, Cuint, Ptr{Void})) diff --git a/base/libgit2/remote.jl b/base/libgit2/remote.jl index d258570a9f411..1e8feed1736a3 100644 --- a/base/libgit2/remote.jl +++ b/base/libgit2/remote.jl @@ -8,6 +8,14 @@ function GitRemote(repo::GitRepo, rmt_name::AbstractString, rmt_url::AbstractStr return GitRemote(rmt_ptr_ptr[]) end +function GitRemote(repo::GitRepo, rmt_name::AbstractString, rmt_url::AbstractString, fetch_spec::AbstractString) + rmt_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_remote_create_with_fetchspec, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring, Cstring), + rmt_ptr_ptr, repo.ptr, rmt_name, rmt_url, fetch_spec) + return GitRemote(rmt_ptr_ptr[]) +end + function GitRemoteAnon(repo::GitRepo, url::AbstractString) rmt_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_remote_create_anonymous, :libgit2), Cint, @@ -46,15 +54,14 @@ function fetch{T<:AbstractString}(rmt::GitRemote, refspecs::Vector{T}; end function push{T<:AbstractString}(rmt::GitRemote, refspecs::Vector{T}; - force::Bool=false) - msg = "libgit2.push: $msg" - push_opts = PushOptionsStruct() + force::Bool = false, + push_opts::PushOptions = PushOptions()) no_refs = (length(refspecs) == 0) !no_refs && (sa = StrArrayStruct(refspecs...)) try @check ccall((:git_remote_push, :libgit2), Cint, - (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{PushOptionsStruct}), - rmt.ptr, no_refs ? C_NULL : Ref(sa), Ref(push_opts)) + (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{PushOptions}), + rmt.ptr, no_refs ? C_NULL : Ref(sa), Ref(push_opts)) finally !no_refs && finalize(sa) end diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index df5620907d4db..0f69d3ba332e6 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -135,19 +135,31 @@ immutable RemoteCallbacks transport::Ptr{Void} payload::Ptr{Void} end -RemoteCallbacks() = RemoteCallbacks(one(Cuint), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0), - Ptr{Void}(0)) +RemoteCallbacks(; sideband_progress::Ptr{Void} = C_NULL, + completion::Ptr{Void} = C_NULL, + credentials::Ptr{Void} = C_NULL, + certificate_check::Ptr{Void} = C_NULL, + transfer_progress::Ptr{Void} = C_NULL, + update_tips::Ptr{Void} = C_NULL, + pack_progress::Ptr{Void} = C_NULL, + push_transfer_progress::Ptr{Void} = C_NULL, + push_update_reference::Ptr{Void} = C_NULL, + push_negotiation::Ptr{Void} = C_NULL, + transport::Ptr{Void} = C_NULL, + payload::Ptr{Void} = C_NULL +)=RemoteCallbacks(one(Cuint), + sideband_progress, + completion, + credentials, + certificate_check, + transfer_progress, + update_tips, + pack_progress, + push_transfer_progress, + push_update_reference, + push_negotiation, + transport, + payload) immutable FetchOptions version::Cuint @@ -282,11 +294,16 @@ MergeOptionsStruct(; flags::Cint = Cint(0), file_favor ) -immutable PushOptionsStruct +immutable PushOptions version::Cuint parallelism::Cint + callbacks::RemoteCallbacks end -PushOptionsStruct() = PushOptionsStruct(one(Cuint),one(Cuint)) +PushOptions(; parallelism::Cint=one(Cint), + callbacks::RemoteCallbacks=RemoteCallbacks() +)=PushOptions(one(Cuint), + parallelism, + callbacks) immutable IndexTime seconds::Int64 diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index dcf7b397528e5..66e5a358ae12b 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -387,14 +387,14 @@ function pull_request(dir::AbstractString, commit::AbstractString="", url::Abstr m = match(LibGit2.GITHUB_REGEX, url) m == nothing && throw(PkgError("not a GitHub repo URL, can't make a pull request: $url")) - owner, repo = m.captures[2:3] + owner, owner_repo = m.captures[2:3] user = GitHub.user() - info("Forking $owner/$repo to $user") - response = GitHub.fork(owner,repo) + info("Forking $owner/$owner_repo to $user") + response = GitHub.fork(owner,owner_repo) fork = response["ssh_url"] branch = "pull-request/$(commit[1:8])" info("Pushing changes as branch $branch") - LibGit2.push(repo, url=fork, refspecs=["$commit:refs/heads/$branch"]) + LibGit2.push(repo, remoteurl=fork, refspecs=["$commit:refs/heads/$branch"]) pr_url = "$(response["html_url"])/compare/$branch" info("To create a pull-request, open:\n\n $pr_url\n") end @@ -407,6 +407,8 @@ function submit(pkg::AbstractString, commit::AbstractString="") end function publish(branch::AbstractString) + tags = Dict{ByteString,Vector{ASCIIString}}() + with(GitRepo, "METADATA") do repo LibGit2.branch(repo) == branch || throw(PkgError("METADATA must be on $branch to publish changes")) @@ -416,11 +418,8 @@ function publish(branch::AbstractString) ahead_remote > 0 && throw(PkgError("METADATA is behind origin/$branch – run `Pkg.update()` before publishing")) ahead_local == 0 && throw(PkgError("There are no METADATA changes to publish")) - tags = Dict{ByteString,Vector{ASCIIString}}() - # get changed files for path in LibGit2.diff_files(repo, "origin/metadata-v2", LibGit2.GitConst.HEAD_FILE) - println(path) m = match(r"^(.+?)/versions/([^/]+)/sha1$", path) m != nothing && ismatch(Base.VERSION_REGEX, m.captures[2]) || continue pkg, ver = m.captures; ver = convert(VersionNumber,ver) From 25778f71aebc4549f6b473be0f9a2e5e8ce1cada Mon Sep 17 00:00:00 2001 From: wildart Date: Sun, 2 Aug 2015 06:32:13 -0400 Subject: [PATCH 0343/1938] merge overhaul, added credential & fetchhead callbacks, small fixes & cleanup --- base/libgit2.jl | 75 ++++++++++++------------ base/libgit2/callbacks.jl | 81 +++++++++++++++++++++++--- base/libgit2/const.jl | 47 ++++++++------- base/libgit2/merge.jl | 115 +++++++++++++++++++++++++++++++++---- base/libgit2/rebase.jl | 2 +- base/libgit2/reference.jl | 38 ++++++++---- base/libgit2/remote.jl | 8 +-- base/libgit2/repository.jl | 16 ++++++ base/libgit2/types.jl | 38 +++++++----- base/libgit2/utils.jl | 4 +- base/pkg/entry.jl | 8 +-- 11 files changed, 318 insertions(+), 114 deletions(-) diff --git a/base/libgit2.jl b/base/libgit2.jl index 825865b53f60d..f1468a1d346e2 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -147,7 +147,8 @@ function fetch{T<:AbstractString}(repo::GitRepo; GitRemoteAnon(repo, remoteurl) end try - fetch(rmt, refspecs, msg="from $(url(rmt))") + fo = FetchOptions(callbacks=RemoteCallbacks(credentials=credentials_cb)) + fetch(rmt, refspecs, msg="from $(url(rmt))", options = fo) catch err warn("fetch: $err") finally @@ -168,7 +169,7 @@ function push{T<:AbstractString}(repo::GitRepo; end try po = PushOptions(callbacks=RemoteCallbacks(credentials=credentials_cb)) - push(rmt, refspecs, force=force, push_opts=po) + push(rmt, refspecs, force=force, options=po) catch err warn("push: $err") finally @@ -270,8 +271,7 @@ function checkout!(repo::GitRepo, commit::AbstractString = ""; try # detach commit obj_oid = Oid(peeled) - ref = create_reference(repo, obj_oid, - force=force, + ref = GitReference(repo, obj_oid, force=force, msg="libgit2.checkout: moving from $head_name to $(string(obj_oid))") finalize(ref) @@ -347,43 +347,44 @@ function revcount(repo::GitRepo, fst::AbstractString, snd::AbstractString) return (fc-1, sc-1) end -""" git merge [--ff-only] [] """ -function merge!(repo::GitRepo, committish::AbstractString=""; fast_forward::Bool=false) - # get head annotated upstream reference - ret = with(head(repo)) do head_ref - brn_ref = upstream(head_ref) - upst_ann = isempty(committish) ? GitAnnotated(repo, brn_ref) : GitAnnotated(repo, committish) - try - ma, mp = merge_analysis(repo, upst_ann) - (ma & GitConst.MERGE_ANALYSIS_UP_TO_DATE == GitConst.MERGE_ANALYSIS_UP_TO_DATE) && return true - if (ma & GitConst.MERGE_ANALYSIS_FASTFORWARD == GitConst.MERGE_ANALYSIS_FASTFORWARD) - # do fastforward: checkout tree and update branch references - # hur_oid = Oid(hur) - # with(get(GitCommit, repo, hur_oid)) do cmt - # checkout_tree(repo, cmt) - # end - # target!(hr, hur_oid, msg="pkg.libgit2.megre!: fastforward $(name(hur)) into $(name(hr))") - # head!(repo, hur, msg="--fastforward") - - brn_ref_oid = Oid(brn_ref) - target!(head_ref, brn_ref_oid, msg="pkg.libgit2.megre!: fastforward $(name(brn_ref)) into $(name(head_ref))") - reset!(repo, brn_ref_oid, GitConst.RESET_HARD) - elseif (ma & GitConst.MERGE_ANALYSIS_NORMAL == GitConst.MERGE_ANALYSIS_NORMAL) - fast_forward && return false # do not do merge - merge!(repo, hua) - cleanup(repo) - info("Review and commit merged changes.") - else - warn("Unknown merge analysis result. Merging is not possible.") - return false +""" git merge [--ff-only] [ | FETCH_HEAD] """ +function merge!(repo::GitRepo; + committish::AbstractString = "", + branch::AbstractString = "", + fastforward::Bool = false, + options = MergeOptions()) + # merge into head branch + with(head(repo)) do head_ref + upst_anns = if !isempty(committish) # merge committish into HEAD + if committish == GitConst.FETCH_HEAD # merge FETCH_HEAD + fheads = fetchheads(repo) + filter!(fh->fh.ismerge, fheads) + length(fheads) == 0 && throw(Error.GitError(Error.Merge, Error.ERROR, "There is no fetch reference for this branch.")) + map(fh->GitAnnotated(repo,fh), fheads) + else # merge commitish + [GitAnnotated(repo, committish)] + end + else + if !isempty(branch) # merge provided branch into HEAD + with(GitReference(repo, branch)) do brn_ref + [GitAnnotated(repo, brn_ref)] + end + else # try to get tracking remote branch for the head + !isattached(repo) && throw(Error.GitError(Error.Merge, Error.ERROR, "There is no tracking information for the current branch.")) + with(upstream(head_ref)) do tr_brn_ref + [GitAnnotated(repo, tr_brn_ref)] + end end - return true + end + + try + merge!(repo, upst_anns, fastforward, options) finally - finalize(upst_ann) - finalize(brn_ref) + for ann in upst_anns + finalize(ann) + end end end - return ret end """ git rebase --merge [--onto ] [] """ diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 5c5d7fc368eb2..8061bca4faca3 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -1,3 +1,14 @@ +function prompt(msg::AbstractString; default::AbstractString="", password::Bool=false) + msg = length(default) > 0 ? msg*" [$default]:" : msg*":" + uinput = if password + bytestring(ccall(:getpass, Cstring, (Cstring,), msg)) + else + print(msg) + chomp(readline(STDIN)) + end + length(uinput) == 0 ? default : uinput +end + function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, name::Cstring, url::Cstring, payload::Ptr{Void}) # Create the remote with a mirroring url @@ -18,19 +29,71 @@ function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, return Cint(0) end -function credentials_callback(cred::Ptr{Ptr{Void}}, url::Cstring, - username_from_url::Cstring, +function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, + username_ptr::Cstring, allowed_types::Cuint, payload::Ptr{Void}) - # Try ssh-agent first - err = ccall((:git_cred_ssh_key_from_agent, :libgit2), Cint, - (Ptr{Ptr{Void}}, Cstring), - cred, username_from_url) - err == 0 && return Cint(0) + url = bytestring(url_ptr) + println(allowed_types) + + # for HTTPS use keyboard-interactive prompt + if startswith(url, "https") + username = prompt("Username for '$url'") + pass = prompt("Password for '$url'", password=true) + + err = ccall((:git_cred_userpass_plaintext_new, :libgit2), Cint, + (Ptr{Ptr{Void}}, Cstring, Cstring), + cred, username, pass) + err != 0 && return Cint(err) + else + # for SSH we need key info, look for environment vars GITHUB_* as well + + # if username is not provided, then prompt for it + username = if username_ptr == Cstring_NULL + prompt("Username for '$url'") + else + bytestring(username_ptr) + end + println(username) + + publickey = if "GITHUB_PUB_KEY" in keys(ENV) + ENV["GITHUB_PUB_KEY"] + else + keydef = homedir()*"/.ssh/id_rsa.pub" + prompt("Public key location", default=keydef) + end + println(publickey) - # TODO: deal with other auth types + WIN + privatekey = if "GITHUB_PRV_KEY" in keys(ENV) + ENV["GITHUB_PRV_KEY"] + else + keydef = homedir()*"/.ssh/id_rsa" + prompt("Private key location", default=keydef) + end + println(privatekey) - return Cint(1) + passphrase= if "GITHUB_PRV_KEY_PASS" in keys(ENV) + ENV["GITHUB_PRV_KEY"] + else + prompt("Private key passphrase", password=true) + end + println(passphrase) + + err = ccall((:git_cred_ssh_key_new, :libgit2), Cint, + (Ptr{Ptr{Void}}, Cstring, Cstring, Cstring, Cstring), + cred, username, publickey, privatekey, passphrase) + err != 0 && return Cint(err) + end + + return Cint(0) +end + +function fetchhead_foreach_callback(ref_name::Cstring, remote_url::Cstring, + oid::Ptr{Oid}, is_merge::Cuint, payload::Ptr{Void}) + fhead_vec = unsafe_pointer_to_objref(payload)::Vector{FetchHead} + push!(fhead_vec, FetchHead(bytestring(ref_name), bytestring(remote_url), Oid(oid), is_merge == 1)) + return Cint(0) end const mirror_cb = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring, Ptr{Void})) const credentials_cb = cfunction(credentials_callback, Cint, (Ptr{Ptr{Void}}, Cstring, Cstring, Cuint, Ptr{Void})) +const fetchhead_foreach_cb = cfunction(fetchhead_foreach_callback, Cint, (Cstring, Cstring, Ptr{Oid}, Cuint, Ptr{Void})) \ No newline at end of file diff --git a/base/libgit2/const.jl b/base/libgit2/const.jl index ddb597bf88c49..b20c4a62c7307 100644 --- a/base/libgit2/const.jl +++ b/base/libgit2/const.jl @@ -3,6 +3,7 @@ module GitConst const HEAD_FILE = "HEAD" + const FETCH_HEAD = "FETCH_HEAD" const REMOTE_ORIGIN = "origin" # objs @@ -153,28 +154,30 @@ module GitConst # merge const MERGE_TREE_FIND_RENAMES = Cint(1 << 0) - const MERGE_FILE_FAVOR_NORMAL = Cint(0) - const MERGE_FILE_FAVOR_OURS = Cint(1) - const MERGE_FILE_FAVOR_THEIRS = Cint(2) - const MERGE_FILE_FAVOR_UNION = Cint(3) - - const MERGE_AUTOMERGE_NORMAL = Cint(0) - const MERGE_AUTOMERGE_FAVOR_OURS = Cint(1) - const MERGE_AUTOMERGE_FAVOR_THEIRS = Cint(2) - const MERGE_AUTOMERGE_FAVOR_UNION = Cint(3) - - const MERGE_NO_FASTFORWARD = Cint(1) - const MERGE_FASTFORWARD_ONLY = Cint(2) - - const MERGE_ANALYSIS_NONE = Cint(0) - const MERGE_ANALYSIS_NORMAL = Cint(1 << 0) - const MERGE_ANALYSIS_UP_TO_DATE = Cint(1 << 1) - const MERGE_ANALYSIS_FASTFORWARD = Cint(1 << 2) - const MERGE_ANALYSIS_UNBORN = Cint(1 << 3) - - const MERGE_PREFERENCE_NONE = Cint(0) - const MERGE_PREFERENCE_NO_FASTFORWARD = Cint(1 << 0) - const MERGE_PREFERENCE_FASTFORWARD_ONLY = Cint(1 << 1) + @enum(GIT_MERGE_FILE, MERGE_FILE_DEFAULT = 0, # Defaults + MERGE_FILE_STYLE_MERGE = 1 << 0, # Create standard conflicted merge files + MERGE_FILE_STYLE_DIFF3 = 1 << 1, # Create diff3-style files + MERGE_FILE_SIMPLIFY_ALNUM = 1 << 2, # Condense non-alphanumeric regions for simplified diff file + MERGE_FILE_IGNORE_WHITESPACE = 1 << 3, # Ignore all whitespace + MERGE_FILE_IGNORE_WHITESPACE_CHANGE = 1 << 4, # Ignore changes in amount of whitespace + MERGE_FILE_IGNORE_WHITESPACE_EOL = 1 << 5, # Ignore whitespace at end of line + MERGE_FILE_DIFF_PATIENCE = 1 << 6, # Use the "patience diff" algorithm + MERGE_FILE_DIFF_MINIMAL = 1 << 7) # Take extra time to find minimal diff + + @enum(GIT_MERGE_FILE_FAVOR, MERGE_FILE_FAVOR_NORMAL = 0, + MERGE_FILE_FAVOR_OURS = 1, + MERGE_FILE_FAVOR_THEIRS = 2, + MERGE_FILE_FAVOR_UNION = 3) + + @enum(GIT_MERGE_PREFERENCE, MERGE_PREFERENCE_NONE = 0, + MERGE_PREFERENCE_NO_FASTFORWARD = 1, + MERGE_PREFERENCE_FASTFORWARD_ONLY = 2) + + @enum(GIT_MERGE_ANALYSIS, MERGE_ANALYSIS_NONE = 0, + MERGE_ANALYSIS_NORMAL = 1 << 0, + MERGE_ANALYSIS_UP_TO_DATE = 1 << 1, + MERGE_ANALYSIS_FASTFORWARD = 1 << 2, + MERGE_ANALYSIS_UNBORN = 1 << 3) # reset const RESET_SOFT = Cint(1) # Move the head to the given commit diff --git a/base/libgit2/merge.jl b/base/libgit2/merge.jl index 19e835ffa4989..fa0eaca861a0b 100644 --- a/base/libgit2/merge.jl +++ b/base/libgit2/merge.jl @@ -16,6 +16,14 @@ function GitAnnotated(repo::GitRepo, ref::GitReference) return GitAnnotated(ann_ref_ref[]) end +function GitAnnotated(repo::GitRepo, fh::FetchHead) + ann_ref_ref = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_annotated_commit_from_fetchhead, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring, Ptr{Oid}), + ann_ref_ref, repo.ptr, fh.name, fh.url, fh.oid) + return GitAnnotated(ann_ref_ref[]) +end + function GitAnnotated(repo::GitRepo, comittish::AbstractString) obj = revparse(repo, comittish) try @@ -31,24 +39,111 @@ function commit(ann::GitAnnotated) return Oid(ccall((:git_annotated_commit_id, :libgit2), Ptr{Oid}, (Ptr{Void},), ann.ptr)) end -function merge_analysis(repo::GitRepo, ann::GitAnnotated) +function merge_analysis(repo::GitRepo, anns::Vector{GitAnnotated}) analysis = Ref{Cint}(0) preference = Ref{Cint}(0) + anns_ref = Ref(map(a->a.ptr, anns)) + anns_size = Csize_t(length(anns)) @check ccall((:git_merge_analysis, :libgit2), Cint, (Ptr{Cint}, Ptr{Cint}, Ptr{Void}, Ptr{Ptr{Void}}, Csize_t), - analysis, preference, repo.ptr, Ref{Ptr{Void}}(ann.ptr), 1) + analysis, preference, repo.ptr, anns_ref, anns_size) return analysis[], preference[] end +"""Fastforward merge changes into current head """ +function ffmerge!(repo::GitRepo, ann::GitAnnotated) + ann_cmt_oid = commit(ann) + cmt = get(GitCommit, repo, ann_cmt_oid) + cmt == nothing && return false # could not find commit tree + try + checkout_tree(repo, cmt) + with(head(repo)) do head_ref + cmt_oid = Oid(cmt) + msg = "libgit2.merge: fastforward $(string(cmt_oid)) into $(name(head_ref))" + new_head_ref = if reftype(head_ref) == GitConst.REF_OID + target!(head_ref, cmt_oid, msg=msg) + else + GitReference(repo, cmt_oid, fullname(head_ref), msg=msg) + end + finalize(new_head_ref) + end + finally + finalize(cmt) + end + return true +end + """ Merge changes into current head """ -function merge!(repo::GitRepo, their_head::GitAnnotated; - merge_opts = MergeOptionsStruct(), - checkout_opts = CheckoutOptions(checkout_strategy = GitConst.CHECKOUT_SAFE)) - return @check ccall((:git_merge, :libgit2), Cint, - (Ptr{Void}, Ptr{Ptr{Void}}, Csize_t, - Ptr{MergeOptionsStruct}, Ptr{CheckoutOptions}), - repo.ptr, Ref{Ptr{Void}}(their_head.ptr), 1, - Ref(merge_opts), Ref(checkout_opts)) +function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, merge_opts::MergeOptions, + checkout_opts = CheckoutOptions(checkout_strategy = GitConst.CHECKOUT_SAFE)) + anns_size = Csize_t(length(anns)) + @check ccall((:git_merge, :libgit2), Cint, + (Ptr{Void}, Ptr{Ptr{Void}}, Csize_t, + Ptr{MergeOptionsStruct}, Ptr{CheckoutOptions}), + repo.ptr, anns, anns_size, + Ref(merge_opts), Ref(checkout_opts)) + info("Review and commit merged changes.") + return true +end + +"""Internal implementation of merge. +Returns `true` if merge was successful, otherwise `false` +""" +function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, fastforward::Bool, options::MergeOptions) + ma, mp = merge_analysis(repo, anns) + if isset(ma, Cint(GitConst.MERGE_ANALYSIS_UP_TO_DATE)) + return true # no merge - everything is up to date + end + + ffPref = if fastforward + GitConst.MERGE_PREFERENCE_FASTFORWARD_ONLY + elseif isset(mp, Cint(GitConst.MERGE_PREFERENCE_NONE)) + GitConst.MERGE_PREFERENCE_NONE + elseif isset(mp, Cint(GitConst.MERGE_PREFERENCE_NO_FASTFORWARD)) + GitConst.MERGE_PREFERENCE_NO_FASTFORWARD + elseif isset(mp, Cint(GitConst.MERGE_PREFERENCE_FASTFORWARD_ONLY)) + GitConst.MERGE_PREFERENCE_FASTFORWARD_ONLY + end + if ffPref == nothing + warn("Unknown merge preference: $(mp).") + return false + end + + mergeResult = if ffPref == GitConst.MERGE_PREFERENCE_NONE + if isset(ma, Cint(GitConst.MERGE_ANALYSIS_FASTFORWARD)) + if length(anns) > 1 + warn("Unable to perform Fast-Forward merge with mith multiple merge heads.") + false + else + ffmerge!(repo, anns[1]) + end + elseif isset(ma, Cint(GitConst.MERGE_ANALYSIS_NORMAL)) + merge!(repo, anns, options) + end + elseif ffPref == GitConst.MERGE_PREFERENCE_FASTFORWARD_ONLY + if isset(ma, Cint(GitConst.MERGE_ANALYSIS_FASTFORWARD)) + if length(anns) > 1 + warn("Unable to perform Fast-Forward merge with mith multiple merge heads.") + false + else + ffmerge!(repo, anns[1]) + end + else + warn("Cannot perform fast-forward merge.") + false + end + elseif ffPref == GitConst.MERGE_PREFERENCE_NO_FASTFORWARD + if isset(ma, Cint(GitConst.MERGE_ANALYSIS_NORMAL)) + merge!(repo, anns, options) + end + end + + if mergeResult == nothing + warn("Unknown merge analysis result: $(ma). Merging is not possible.") + return false + end + + return mergeResult end function merge_base(repo::GitRepo, one::AbstractString, two::AbstractString) diff --git a/base/libgit2/rebase.jl b/base/libgit2/rebase.jl index 68ce79879005b..32c0da2d2a9af 100644 --- a/base/libgit2/rebase.jl +++ b/base/libgit2/rebase.jl @@ -35,7 +35,7 @@ function Base.next(rb::GitRebase) (Ptr{Ptr{RebaseOperation}}, Ptr{Void}), rb_op_ptr_ptr, rb.ptr) catch err - err.code == Int(Error.ITEROVER) && return nothing + err.code == Error.ITEROVER && return nothing rethrow(err) end return unsafe_load(convert(Ptr{RebaseOperation}, rb_op_ptr_ptr[]), 1) diff --git a/base/libgit2/reference.jl b/base/libgit2/reference.jl index 6447285b58bf6..4ee641f96a657 100644 --- a/base/libgit2/reference.jl +++ b/base/libgit2/reference.jl @@ -1,10 +1,20 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -function GitReference(repo::GitRepo, ref_name::AbstractString) +function GitReference(repo::GitRepo, refname::AbstractString) ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_reference_lookup, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Cstring), - ref_ptr_ptr, repo.ptr, ref_name) + ref_ptr_ptr, repo.ptr, refname) + return GitReference(ref_ptr_ptr[]) +end + +function GitReference(repo::GitRepo, obj_oid::Oid, refname::AbstractString = GitConst.HEAD_FILE; + force::Bool=false, msg::AbstractString="") + ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_reference_create, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{Oid}, Cint, Cstring), + ref_ptr_ptr, repo.ptr, refname, Ref(obj_oid), Cint(force), + isempty(msg) ? Cstring_NULL : msg) return GitReference(ref_ptr_ptr[]) end @@ -49,6 +59,20 @@ function branch(ref::GitReference) return bytestring(str_ptr_ptr[]) end +function ishead(ref::GitReference) + isempty(ref) && return false + err = ccall((:git_branch_is_head, :libgit2), Cint, + (Ptr{Void},), ref.ptr) + return err == 1 +end + +function isbranch(ref::GitReference) + isempty(ref) && return false + err = ccall((:git_reference_is_branch, :libgit2), Cint, + (Ptr{Void},), ref.ptr) + return err == 1 +end + function peel{T <: GitObject}(::Type{T}, ref::GitReference) git_otype = getobjecttype(T) obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @@ -65,16 +89,6 @@ function peel{T <: GitObject}(::Type{T}, ref::GitReference) return T(obj_ptr_ptr[]) end -function create_reference(repo::GitRepo, obj_oid::Oid, refname::AbstractString = GitConst.HEAD_FILE; - force::Bool=false, msg::AbstractString="") - ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - @check ccall((:git_reference_create, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{Oid}, Cint, Cstring), - ref_ptr_ptr, repo.ptr, refname, Ref(obj_oid), Cint(force), - isempty(msg) ? Cstring_NULL : msg) - return GitReference(ref_ptr_ptr[]) -end - function create_branch(repo::GitRepo, commit_obj::GitCommit, bname::AbstractString; force::Bool=false) ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) diff --git a/base/libgit2/remote.jl b/base/libgit2/remote.jl index 1e8feed1736a3..51f324529988c 100644 --- a/base/libgit2/remote.jl +++ b/base/libgit2/remote.jl @@ -39,7 +39,7 @@ function url(rmt::GitRemote) end function fetch{T<:AbstractString}(rmt::GitRemote, refspecs::Vector{T}; - fetch_opts::FetchOptions = FetchOptions(), + options::FetchOptions = FetchOptions(), msg::AbstractString="") msg = "libgit2.fetch: $msg" no_refs = (length(refspecs) == 0) @@ -47,7 +47,7 @@ function fetch{T<:AbstractString}(rmt::GitRemote, refspecs::Vector{T}; try @check ccall((:git_remote_fetch, :libgit2), Cint, (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{FetchOptions}, Cstring), - rmt.ptr, no_refs ? C_NULL : Ref(sa), Ref(fetch_opts), msg) + rmt.ptr, no_refs ? C_NULL : Ref(sa), Ref(options), msg) finally !no_refs && finalize(sa) end @@ -55,13 +55,13 @@ end function push{T<:AbstractString}(rmt::GitRemote, refspecs::Vector{T}; force::Bool = false, - push_opts::PushOptions = PushOptions()) + options::PushOptions = PushOptions()) no_refs = (length(refspecs) == 0) !no_refs && (sa = StrArrayStruct(refspecs...)) try @check ccall((:git_remote_push, :libgit2), Cint, (Ptr{Void}, Ptr{StrArrayStruct}, Ptr{PushOptions}), - rmt.ptr, no_refs ? C_NULL : Ref(sa), Ref(push_opts)) + rmt.ptr, no_refs ? C_NULL : Ref(sa), Ref(options)) finally !no_refs && finalize(sa) end diff --git a/base/libgit2/repository.jl b/base/libgit2/repository.jl index df285627147b0..8a4ade52b5ce7 100644 --- a/base/libgit2/repository.jl +++ b/base/libgit2/repository.jl @@ -162,3 +162,19 @@ function clone(repo_url::AbstractString, repo_path::AbstractString, repo_ptr_ptr, repo_url, repo_path, clone_opts_ref) return GitRepo(repo_ptr_ptr[]) end + +function fetchheads(repo::GitRepo) + fhr = Ref{Vector{FetchHead}}(FetchHead[]) + @check ccall((:git_repository_fetchhead_foreach, :libgit2), Cint, + (Ptr{Void}, Ptr{Void}, Ptr{Void}), + repo.ptr, fetchhead_foreach_cb, fhr) + return fhr[] +end + +function remotes(repo::GitRepo) + out = Ref(StrArrayStruct()) + @check ccall((:git_remote_list, :libgit2), Cint, + (Ptr{Void}, Ptr{Void}), out, repo.ptr) + return convert(Vector{AbstractString}, out[]) +end + diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index 0f69d3ba332e6..cbe5fad4b4df0 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -273,26 +273,29 @@ immutable DiffDelta end DiffDelta()=DiffDelta(Cint(0),Cuint(0),UInt16(0),UInt16(0),DiffFile(),DiffFile()) -immutable MergeOptionsStruct +immutable MergeOptions version::Cuint - flags::Cint + tree_flags::Cint rename_threshold::Cuint target_limit::Cuint metric::Ptr{Void} file_favor::Cint + file_flags::Cuint end -MergeOptionsStruct(; flags::Cint = Cint(0), - rename_threshold::Cuint = Cuint(50), - target_limit::Cuint = Cuint(200), - metric::Ptr{Void} = Ptr{Void}(0), - file_favor::Cint = GitConst.MERGE_FILE_FAVOR_NORMAL -)=MergeOptionsStruct(one(Cuint), - flags, - rename_threshold, - target_limit, - metric, - file_favor - ) +MergeOptions(; tree_flags::Cint = Cint(0), + rename_threshold::Cuint = Cuint(50), + target_limit::Cuint = Cuint(200), + metric::Ptr{Void} = C_NULL, + file_favor::Cint = Cint(GitConst.MERGE_FILE_FAVOR_NORMAL), + file_flags::Cuint =Cuint(GitConst.MERGE_FILE_DEFAULT) +)=MergeOptions(one(Cuint), + tree_flags, + rename_threshold, + target_limit, + metric, + file_favor, + file_flags + ) immutable PushOptions version::Cuint @@ -386,6 +389,13 @@ immutable StatusEntry end StatusEntry()=StatusEntry(Cuint(0), C_NULL, C_NULL) +immutable FetchHead + name::AbstractString + url::AbstractString + oid::Oid + ismerge::Bool +end + # Abstract object types abstract AbstractGitObject Base.isempty(obj::AbstractGitObject) = (obj.ptr == C_NULL) diff --git a/base/libgit2/utils.jl b/base/libgit2/utils.jl index 07f844df618d2..4cb03ceea063d 100644 --- a/base/libgit2/utils.jl +++ b/base/libgit2/utils.jl @@ -3,4 +3,6 @@ function version() ccall((:git_libgit2_version, :libgit2), Void, (Ptr{Cint}, Ptr{Cint}, Ptr{Cint}), major, minor, patch) return VersionNumber(major[1], minor[1], patch[1]) -end \ No newline at end of file +end + +isset(val::Integer, flag::Integer) = (val & flag == flag) \ No newline at end of file diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 66e5a358ae12b..ecac884ad7dea 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -232,11 +232,11 @@ function checkout(pkg::AbstractString, branch::AbstractString, do_merge::Bool, d LibGit2.transact(r) do repo LibGit2.isdirty(repo) && throw(PkgError("$pkg is dirty, bailing")) LibGit2.branch!(repo, branch, track=LibGit2.GitConst.REMOTE_ORIGIN) - do_merge && LibGit2.merge!(repo, fast_forward=true) # merge changes + do_merge && LibGit2.merge!(repo, fastforward=true) # merge changes if do_pull info("Pulling $pkg latest $branch...") LibGit2.fetch(repo) - LibGit2.merge!(repo, fast_forward=true) + LibGit2.merge!(repo, fastforward=true) end resolve() end @@ -332,7 +332,7 @@ function update(branch::AbstractString) end end LibGit2.fetch(repo) - LibGit2.merge!(repo, fast_forward=true) || LibGit2.rebase!(repo, "origin/$branch") + LibGit2.merge!(repo, fastforward=true) || LibGit2.rebase!(repo, "origin/$branch") end avail = Read.available() # this has to happen before computing free/fixed @@ -356,7 +356,7 @@ function update(branch::AbstractString) info("Updating $pkg...") @recover begin LibGit2.fetch(repo) - LibGit2.merge!(repo, fast_forward=true) + LibGit2.merge!(repo, fastforward=true) end end end From 9f3bad9734648a48f23312af7850c0091120f145 Mon Sep 17 00:00:00 2001 From: wildart Date: Sun, 2 Aug 2015 13:17:28 -0400 Subject: [PATCH 0344/1938] added extended repo open, ported Base.url --- base/libgit2/config.jl | 3 -- base/libgit2/const.jl | 5 ++++ base/libgit2/repository.jl | 15 ++++++++++ base/methodshow.jl | 58 ++++++++++++++++++++------------------ 4 files changed, 50 insertions(+), 31 deletions(-) diff --git a/base/libgit2/config.jl b/base/libgit2/config.jl index 43ba1b3e1bce1..532e0b49d0000 100644 --- a/base/libgit2/config.jl +++ b/base/libgit2/config.jl @@ -32,9 +32,6 @@ function GitConfig(isglobal::Bool = true) end function get{T<:AbstractString}(::Type{T}, c::GitConfig, name::AbstractString) - # str_ptr = Ref{Ptr{UInt8}}(C_NULL) - # @check ccall((:git_config_get_string, :libgit2), Cint, #TODO: git_config_get_string_buf - # (Ptr{Ptr{UInt8}}, Ptr{Void}, Cstring), str_ptr, c.ptr, name) buf_ptr = Ref(Buffer()) @check ccall((:git_config_get_string_buf, :libgit2), Cint, (Ptr{Buffer}, Ptr{Void}, Cstring), buf_ptr, c.ptr, name) diff --git a/base/libgit2/const.jl b/base/libgit2/const.jl index b20c4a62c7307..6fa960e79f54b 100644 --- a/base/libgit2/const.jl +++ b/base/libgit2/const.jl @@ -259,4 +259,9 @@ module GitConst SUBMODULE_IGNORE_UNTRACKED = 2, # dirty if tracked files change SUBMODULE_IGNORE_DIRTY = 3, # only dirty if HEAD moved SUBMODULE_IGNORE_ALL = 4) # never dirty + + @enum(GIT_REPOSITORY_OPEN, REPOSITORY_OPEN_DEFAULT = 0, # default value + REPOSITORY_OPEN_NO_SEARCH = 1<<0, # only open the repository if it can be immediately found + REPOSITORY_OPEN_CROSS_FS = 1<<1, # open will not continue searching across FS boundaries + REPOSITORY_OPEN_BARE = 1<<2) # open repository as a bare repo end diff --git a/base/libgit2/repository.jl b/base/libgit2/repository.jl index 8a4ade52b5ce7..2b16abcb41bdb 100644 --- a/base/libgit2/repository.jl +++ b/base/libgit2/repository.jl @@ -13,6 +13,21 @@ function GitRepo(path::AbstractString) return GitRepo(repo_ptr_ptr[]) end +function GitRepoExt(path::AbstractString, flags::Cuint = Cuint(GitConst.REPOSITORY_OPEN_DEFAULT)) + separator = @unix? ":" : ";" + repo_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + err = ccall((:git_repository_open_ext, :libgit2), Cint, + (Ptr{Ptr{Void}}, Cstring, Cuint, Cstring), + repo_ptr_ptr, path, flags, separator) + if err != Int(Error.GIT_OK) + if repo_ptr_ptr[] != C_NULL + finalize(GitRepo(repo_ptr_ptr[])) + end + throw(Error.GitError(err)) + end + return GitRepo(repo_ptr_ptr[]) +end + function cleanup(r::GitRepo) if r.ptr != C_NULL ccall((:git_repository__cleanup, :libgit2), Void, (Ptr{Void},), r.ptr) diff --git a/base/methodshow.jl b/base/methodshow.jl index 540912063683b..f3d94184957ca 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -83,34 +83,36 @@ show(io::IO, mt::MethodTable) = show_method_table(io, mt) inbase(m::Module) = m == Base ? true : m == Main ? false : inbase(module_parent(m)) fileurl(file) = let f = find_source_file(file); f == nothing ? "" : "file://"*f; end -#TODO: Convert to libgit2 -# function url(m::Method) -# M = m.func.code.module -# (m.func.code.file == :null || m.func.code.file == :string) && return "" -# file = string(m.func.code.file) -# line = m.func.code.line -# line <= 0 || ismatch(r"In\[[0-9]+\]", file) && return "" -# if inbase(M) -# return "https://github.com/JuliaLang/julia/tree/$(Base.GIT_VERSION_INFO.commit)/base/$file#L$line" -# else -# try -# d = dirname(file) -# u = Git.readchomp(`config remote.origin.url`, dir=d) -# u = match(Git.GITHUB_REGEX,u).captures[1] -# root = cd(d) do # dir=d confuses --show-toplevel, apparently -# Git.readchomp(`rev-parse --show-toplevel`) -# end -# if startswith(file, root) -# commit = Git.readchomp(`rev-parse HEAD`, dir=d) -# return "https://github.com/$u/tree/$commit/"*file[length(root)+2:end]*"#L$line" -# else -# return fileurl(file) -# end -# catch -# return fileurl(file) -# end -# end -# end + +function url(m::Method) + M = m.func.code.module + (m.func.code.file == :null || m.func.code.file == :string) && return "" + file = string(m.func.code.file) + line = m.func.code.line + line <= 0 || ismatch(r"In\[[0-9]+\]", file) && return "" + if inbase(M) + return "https://github.com/JuliaLang/julia/tree/$(Base.GIT_VERSION_INFO.commit)/base/$file#L$line" + else + try + d = dirname(file) + return LibGit2.with(LibGit2.GitRepoExt(d)) do repo + LibGit2.with(LibGit2.GitConfig(repo)) do cfg + u = LibGit2.get(cfg, "remote.origin.url", "") + u = match(LibGit2.GITHUB_REGEX,u).captures[1] + commit = string(LibGit2.head_oid(repo)) + root = LibGit2.path(repo) + if startswith(file, root) + "https://github.com/$u/tree/$commit/"*file[length(root)+1:end]*"#L$line" + else + fileurl(file) + end + end + end + catch + return fileurl(file) + end + end +end function writemime(io::IO, ::MIME"text/html", m::Method) print(io, m.func.code.name) From 5fa68d570170b6f1ced4dae64e6729a4530506f1 Mon Sep 17 00:00:00 2001 From: wildart Date: Sun, 2 Aug 2015 14:22:58 -0400 Subject: [PATCH 0345/1938] fixed Pkg.pull_request, suppress backtrace PkgError & code cleanup --- base/libgit2.jl | 2 -- base/libgit2/callbacks.jl | 7 +------ base/pkg/dir.jl | 10 +++++++++- base/pkg/entry.jl | 3 ++- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/base/libgit2.jl b/base/libgit2.jl index f1468a1d346e2..593be1d06e19f 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -170,8 +170,6 @@ function push{T<:AbstractString}(repo::GitRepo; try po = PushOptions(callbacks=RemoteCallbacks(credentials=credentials_cb)) push(rmt, refspecs, force=force, options=po) - catch err - warn("push: $err") finally finalize(rmt) end diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 8061bca4faca3..4d1f857f7cb31 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -33,7 +33,6 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, username_ptr::Cstring, allowed_types::Cuint, payload::Ptr{Void}) url = bytestring(url_ptr) - println(allowed_types) # for HTTPS use keyboard-interactive prompt if startswith(url, "https") @@ -53,7 +52,6 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, else bytestring(username_ptr) end - println(username) publickey = if "GITHUB_PUB_KEY" in keys(ENV) ENV["GITHUB_PUB_KEY"] @@ -61,7 +59,6 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, keydef = homedir()*"/.ssh/id_rsa.pub" prompt("Public key location", default=keydef) end - println(publickey) privatekey = if "GITHUB_PRV_KEY" in keys(ENV) ENV["GITHUB_PRV_KEY"] @@ -69,14 +66,12 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, keydef = homedir()*"/.ssh/id_rsa" prompt("Private key location", default=keydef) end - println(privatekey) passphrase= if "GITHUB_PRV_KEY_PASS" in keys(ENV) - ENV["GITHUB_PRV_KEY"] + ENV["GITHUB_PRV_KEY_PASS"] else prompt("Private key passphrase", password=true) end - println(passphrase) err = ccall((:git_cred_ssh_key_new, :libgit2), Cint, (Ptr{Ptr{Void}}, Cstring, Cstring, Cstring, Cstring), diff --git a/base/pkg/dir.jl b/base/pkg/dir.jl index 7566f6ec2c9ad..0505792c0acb0 100644 --- a/base/pkg/dir.jl +++ b/base/pkg/dir.jl @@ -28,7 +28,15 @@ function cd(f::Function, args...; kws...) !haskey(ENV,"JULIA_PKGDIR") ? init() : throw(PkgError("Package metadata directory $metadata_dir doesn't exist; run Pkg.init() to initialize it.")) end - Base.cd(()->f(args...; kws...), dir) + try + Base.cd(()->f(args...; kws...), dir) + catch err + if isa(err, PkgError) + print_with_color(:red, "PkgMgr Error: $(err.msg)") + else + throw(err) + end + end end function init(meta::AbstractString=DEFAULT_META, branch::AbstractString=META_BRANCH) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index ecac884ad7dea..5137a490153a5 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -394,7 +394,8 @@ function pull_request(dir::AbstractString, commit::AbstractString="", url::Abstr fork = response["ssh_url"] branch = "pull-request/$(commit[1:8])" info("Pushing changes as branch $branch") - LibGit2.push(repo, remoteurl=fork, refspecs=["$commit:refs/heads/$branch"]) + refspecs = ["HEAD:refs/heads/$branch"] # workaround for $commit:refs/heads/$branch + LibGit2.push(repo, remoteurl=fork, refspecs=refspecs) pr_url = "$(response["html_url"])/compare/$branch" info("To create a pull-request, open:\n\n $pr_url\n") end From f3301fb8b0f689b3b67f665602ef0c370eedf8e4 Mon Sep 17 00:00:00 2001 From: wildart Date: Sun, 2 Aug 2015 18:23:13 -0400 Subject: [PATCH 0346/1938] switch GitRepo to GitRevWalker --- base/libgit2.jl | 20 ++++++----- base/libgit2/walker.jl | 79 +++++++++++++++++++++--------------------- test/libgit2.jl | 8 +++-- 3 files changed, 58 insertions(+), 49 deletions(-) diff --git a/base/libgit2.jl b/base/libgit2.jl index 593be1d06e19f..c0c21ad51ed0c 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -338,10 +338,12 @@ function revcount(repo::GitRepo, fst::AbstractString, snd::AbstractString) fst_id = revparseid(repo, fst) snd_id = revparseid(repo, snd) base_id = merge_base(repo, string(fst_id), string(snd_id)) - fc = count((i,r)->i!=base_id, repo, oid=fst_id, - by=GitConst.SORT_TOPOLOGICAL) - sc = count((i,r)->i!=base_id, repo, oid=snd_id, - by=GitConst.SORT_TOPOLOGICAL) + fc = with(GitRevWalker(repo)) do walker + count((i,r)->i!=base_id, walker, oid=fst_id, by=GitConst.SORT_TOPOLOGICAL) + end + sc = with(GitRevWalker(repo)) do walker + count((i,r)->i!=base_id, walker, oid=snd_id, by=GitConst.SORT_TOPOLOGICAL) + end return (fc-1, sc-1) end @@ -424,10 +426,12 @@ end """ Returns all commit authors """ function authors(repo::GitRepo) - athrs = map( - (oid,repo)->author(get(GitCommit, repo, oid))::Signature, #TODO: cleanup - repo) #, by = GitConst.SORT_TIME) - return athrs + return with(GitRevWalker(repo)) do walker + map((oid,repo)->with(get(GitCommit, repo, oid)) do cmt + author(cmt)::Signature + end, + walker) #, by = GitConst.SORT_TIME) + end end function snapshot(repo::GitRepo) diff --git a/base/libgit2/walker.jl b/base/libgit2/walker.jl index f07f1b30c3f89..dfde04fee82a1 100644 --- a/base/libgit2/walker.jl +++ b/base/libgit2/walker.jl @@ -46,62 +46,63 @@ function Base.sort!(w::GitRevWalker; by::Cint = GitConst.SORT_NONE, rev::Bool=fa return w end -function Base.map(f::Function, repo::GitRepo; +function repository(w::GitRevWalker) + ptr = ccall((:git_revwalk_repository, :libgit2), Ptr{Void}, (Ptr{Void},), w.ptr) + ptr != C_NULL && return GitRepo(ptr) + return nothing +end + +function Base.map(f::Function, walker::GitRevWalker; oid::Oid=Oid(), range::AbstractString="", by::Cint = GitConst.SORT_NONE, rev::Bool=false, count::Int=0) - walker = GitRevWalker(repo) res = nothing - try - sort!(walker, by=by, rev=rev) - if !iszero(oid) - push!(walker, oid) - elseif !isempty(range) - push!(walker, range) - else - push_head!(walker) - end - s = start(walker) + sort!(walker, by=by, rev=rev) + if !iszero(oid) + push!(walker, oid) + elseif !isempty(range) + push!(walker, range) + else + push_head!(walker) + end + s = start(walker) - c = 0 - while !done(walker, s) - val = f(s[1], repo) - if res == nothing - res = Array(typeof(val),0) - end - push!(res, val) - val, s = next(walker, s) - c +=1 - count == c && break + c = 0 + repo = repository(walker) + while !done(walker, s) + val = f(s[1], repo) + if res == nothing + res = Array(typeof(val),0) end - finally - finalize(walker) + push!(res, val) + val, s = next(walker, s) + c +=1 + count == c && break end return res end -function Base.count(f::Function, repo::GitRepo; +function Base.count(f::Function, walker::GitRevWalker; oid::Oid=Oid(), by::Cint = GitConst.SORT_NONE, rev::Bool=false) c = 0 - with(GitRevWalker(repo)) do walker - sort!(walker, by=by, rev=rev) - if !iszero(oid) - push!(walker, oid) - else - push_head!(walker) - end - s = start(walker) + sort!(walker, by=by, rev=rev) + if !iszero(oid) + push!(walker, oid) + else + push_head!(walker) + end + s = start(walker) - val = true - while !done(walker, s) && val - val = f(s[1], repo) - _, s = next(walker, s) - c += 1 - end + val = true + repo = repository(walker) + while !done(walker, s) && val + val = f(s[1], repo) + _, s = next(walker, s) + c += 1 end return c end diff --git a/test/libgit2.jl b/test/libgit2.jl index 36bc9ef16537b..85795205b85fc 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -172,13 +172,17 @@ temp_dir() do dir_cache LibGit2.with(LibGit2.GitConfig, repo) do cfg credentials!(cfg) end - oids = LibGit2.map((oid,repo)->string(oid), repo, by = LibGit2.GitConst.SORT_TIME) + oids = LibGit2.with(LibGit2.GitRevWalker(repo)) do walker + LibGit2.map((oid,repo)->string(oid), walker, by = LibGit2.GitConst.SORT_TIME) + end @test length(oids) > 0 finalize(repo) LibGit2.with(LibGit2.GitRepo, path) do repo oid = LibGit2.Oid(oids[end]) - oids = LibGit2.map((oid,repo)->(oid,repo), repo, oid=oid, by=LibGit2.GitConst.SORT_TIME) + oids = LibGit2.with(LibGit2.GitRevWalker(repo)) do walker + LibGit2.map((oid,repo)->(oid,repo), walker, oid=oid, by=LibGit2.GitConst.SORT_TIME) + end @test length(oids) > 0 end end From 9690fa619a0e5006a7153aac5a23567879a0a975 Mon Sep 17 00:00:00 2001 From: wildart Date: Sun, 2 Aug 2015 18:23:58 -0400 Subject: [PATCH 0347/1938] added to GitREPL: branch & tag --- base/libgit2/const.jl | 6 +- base/libgit2/reference.jl | 47 ++++++++++++ base/libgit2/repl.jl | 147 +++++++++++++++++++++++++++++++++++--- base/libgit2/types.jl | 1 + 4 files changed, 186 insertions(+), 15 deletions(-) diff --git a/base/libgit2/const.jl b/base/libgit2/const.jl index 6fa960e79f54b..07084bbe31a09 100644 --- a/base/libgit2/const.jl +++ b/base/libgit2/const.jl @@ -26,10 +26,6 @@ module GitConst const REF_SYMBOLIC = Cint(2) const REF_LISTALL = REF_OID | REF_SYMBOLIC - # branch - const BRANCH_LOCAL = Cint(1) - const BRANCH_REMOTE = Cint(2) - # file const FILEMODE_NEW = Cint(00000) const FILEMODE_TREE = Cint(16384) @@ -264,4 +260,6 @@ module GitConst REPOSITORY_OPEN_NO_SEARCH = 1<<0, # only open the repository if it can be immediately found REPOSITORY_OPEN_CROSS_FS = 1<<1, # open will not continue searching across FS boundaries REPOSITORY_OPEN_BARE = 1<<2) # open repository as a bare repo + + @enum(GIT_BRANCH, BRANCH_LOCAL = 1, BRANCH_REMOTE = 2) end diff --git a/base/libgit2/reference.jl b/base/libgit2/reference.jl index 4ee641f96a657..431ea4c146963 100644 --- a/base/libgit2/reference.jl +++ b/base/libgit2/reference.jl @@ -98,6 +98,10 @@ function create_branch(repo::GitRepo, commit_obj::GitCommit, bname::AbstractStri return GitReference(ref_ptr_ptr[]) end +function delete_branch(branch::GitReference) + @check ccall((:git_branch_delete, :libgit2), Cint, (Ptr{Void},), branch.ptr) +end + function head!(repo::GitRepo, ref::GitReference) ref_name = name(ref) @check ccall((:git_repository_set_head, :libgit2), Cint, @@ -143,3 +147,46 @@ function target!(ref::GitReference, new_oid::Oid; msg::AbstractString="") ref_ptr_ptr, ref.ptr, Ref(new_oid), isempty(msg) ? Cstring_NULL : msg) return GitReference(ref_ptr_ptr[]) end + +function GitBranchIter(r::GitRepo, flags::Cint=Cint(GitConst.BRANCH_LOCAL)) + bi_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_branch_iterator_new, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Cint), bi_ptr, r.ptr, flags) + return GitBranchIter(bi_ptr[]) +end + +function Base.start(bi::GitBranchIter) + ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + btype = Cint[0] + err = ccall((:git_branch_next, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Cint}, Ptr{Void}), + ref_ptr_ptr, btype, bi.ptr) + err != Int(Error.GIT_OK) && return (nothing, -1, true) + return (GitReference(ref_ptr_ptr[]), btype[1], false) +end + +Base.done(bi::GitBranchIter, state) = Bool(state[3]) + +function Base.next(bi::GitBranchIter, state) + ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + btype = Cint[0] + err = ccall((:git_branch_next, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Cint}, Ptr{Void}), + ref_ptr_ptr, btype, bi.ptr) + err != Int(Error.GIT_OK) && return (state[1:2], (nothing, -1, true)) + return (state[1:2], (GitReference(ref_ptr_ptr[]), btype[1], false)) +end + +function Base.map(f::Function, bi::GitBranchIter) + res = nothing + s = start(bi) + while !done(bi, s) + val = f(s[1:2]) + if res == nothing + res = Array(typeof(val),0) + end + push!(res, val) + val, s = next(bi, s) + end + return res +end diff --git a/base/libgit2/repl.jl b/base/libgit2/repl.jl index d78469ba3af41..5977125f7947e 100644 --- a/base/libgit2/repl.jl +++ b/base/libgit2/repl.jl @@ -5,16 +5,16 @@ List of git commands: - [x] add -- [ ] branch +- [x] branch - [x] clone +- [ ] checkout - [ ] commit -- [ ] fetch - [x] init - [x] log - [x] reset - [x] rm - [x] status -- [ ] tag +- [x] tag """ function repl_cmd(ex) @@ -49,7 +49,7 @@ function repl_cmd(ex) warn(ex) end else - repo = GitRepo(repopath) + repo = GitRepoExt(repopath) try if cmd[1] == "help" repl_help() @@ -88,6 +88,54 @@ function repl_cmd(ex) return end repl_reset(repo, cmd[2:end]) + elseif cmd[1] == "branch" + runcmd = if has_params + if cmd[2] == "local" + :listlocal + elseif cmd[2] == "remote" + :listremote + elseif cmd[2] == "all" + :listall + elseif length(cmd) == 3 + if cmd[2] == "add" + :add + elseif cmd[2] == "delete" + :delete + else + :help + end + else + :help + end + else + :listlocal + end + if runcmd == :help + println("usage: branch [(local|remote|all)|(add|delete) ]") + return + end + repl_branch(repo, runcmd, cmd[end]) + elseif cmd[1] == "tag" + runcmd = if has_params + if length(cmd) == 3 + if cmd[2] == "add" + :add + elseif cmd[2] == "delete" + :delete + else + :help + end + else + :help + end + else + :list + end + if runcmd == :help + println("usage: tag [(add|delete) ]") + return + end + repl_tag(repo, runcmd, cmd[end]) else warn("unknown command: $ex. Use \'help\' command.") end @@ -100,19 +148,19 @@ end function repl_help() println("""List of git commands: add\t Add file contents to the index + branch\t List, create, or delete branches clone\t Clone a repository into a current directory init\t Create an empty Git repository or reinitialize an existing one in current directory log\t Show commit logs reset\t\ Reset current HEAD to the specified state rm\t Remove files from the working tree and from the index status\t Show the working tree status + tag\t Create, list or delete a tag objects For particular command parameters use: help """) -# branch\t\tList, create, or delete branches -# commit\t\tRecord changes to the repository -# fetch\t\tDownload objects and refs from another repository -# tag\t\tCreate, list, delete or verify a tag object signed with GPG +# checkout\t Checkout a branch or paths to the working tree +# commit\t Record changes to the repository end function repl_add{T<:AbstractString}(repo::GitRepo, force_add::Bool, files::Vector{T}) @@ -135,13 +183,90 @@ function repl_reset{T<:AbstractString}(repo::GitRepo, files::Vector{T}) return end +function repl_branch{T<:AbstractString}(repo::GitRepo, cmd::Symbol, branch::T) + if cmd == :listlocal || cmd == :listremote || cmd == :listall + flags = cmd == :listlocal ? Cint(GitConst.BRANCH_LOCAL) : ( + cmd == :listremote ? Cint(GitConst.BRANCH_REMOTE) : + Cint(GitConst.BRANCH_LOCAL) | Cint(GitConst.BRANCH_REMOTE)) + # walk through branches + branches = with(GitBranchIter(repo, flags)) do bi + map(x-> try + (shortname(x[1]), x[2]) + finally + finalize(x[1]) + end, bi) + end + # get HEAD name + head_name = with(head(repo)) do href + if isattached(repo) + shortname(href) + else + "(detached from $(string(Oid(href))[1:7]))" + end + end + + # prepare output + output = Tuple{AbstractString,Symbol}[] + for b in branches + c = if b[1] == head_name + :green + elseif b[2] == 1 + :white + else + :red + end + push!(output, (b[1], c)) + end + + # if HEAD does not have branch insert it in output + cmd == :listremote || any(map(b->b[2] == :green, output)) || insert!(output, 1, (head_name, :green)) + for (b, c) in output + c == :green ? print("* ") : print(" ") + print_with_color(c, "$b\n") + end + elseif cmd == :add + with(head(repo)) do href + with(peel(GitCommit, href)) do hcommit + create_branch(repo, hcommit, branch) + end + end + elseif cmd == :delete + bref = lookup_branch(repo, branch) + bref != nothing && try + delete_branch(bref) + finally + finalize(bref) + end + end + return +end + +function repl_tag{T<:AbstractString}(repo::GitRepo, cmd::Symbol, tag::T) + if cmd == :list + for t in sort(tag_list(repo)) + println(t) + end + elseif cmd == :add + with(head(repo)) do href + with(peel(GitCommit, href)) do hcommit + tag_create(repo, tag, string(Oid(hcommit))) + end + end + elseif cmd == :delete + tag_delete(repo, tag) + end + return +end + function repl_log(repo::GitRepo, msg_count::Int) - msgs = map( - (oid,r)->with(get(GitCommit, r, oid)) do cmt + msgs = with(GitRevWalker(repo)) do walker + map((oid,r)->with(get(GitCommit, r, oid)) do cmt sig = author(cmt) msg = message(cmt) (Oid(cmt), sig, msg) - end, repo, count = msg_count) + end, + walker, count = msg_count) + end for msg in msgs print_with_color(:yellow, "Commit: $(string(msg[1]))\n") println("Author:\t$(msg[2].name) $(msg[2].email)") diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index cbe5fad4b4df0..4fad99d2cfdf9 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -420,6 +420,7 @@ for (typ, ref, sup, fnc) in ( (:GitAnnotated, :Void, :AbstractGitObject, :(:git_annotated_commit_free)), (:GitRebase, :Void, :AbstractGitObject, :(:git_rebase_free)), (:GitStatus, :Void, :AbstractGitObject, :(:git_status_list_free)), + (:GitBranchIter, :Void, :AbstractGitObject, :(:git_branch_iterator_free)), (:GitSignature, :SignatureStruct, :AbstractGitObject, :(:git_signature_free)), (:GitAnyObject, :Void, :GitObject, nothing), (:GitCommit, :Void, :GitObject, nothing), From c5350e5db68110a66f6fb2a60cdf576e3354ff72 Mon Sep 17 00:00:00 2001 From: wildart Date: Sun, 2 Aug 2015 18:38:13 -0400 Subject: [PATCH 0348/1938] added `commit` to Git REPL & fixed commands help info --- base/libgit2/repl.jl | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/base/libgit2/repl.jl b/base/libgit2/repl.jl index 5977125f7947e..fac11d573d0ae 100644 --- a/base/libgit2/repl.jl +++ b/base/libgit2/repl.jl @@ -24,11 +24,11 @@ function repl_cmd(ex) if cmd[1] == "init" if has_params && cmd[2] == "help" - println("usage: init [--bare]") + println("usage: init [bare]") return end try - repo = init(repopath, has_params ? Cuint(cmd[2] == "--bare") : Cuint(0)) + repo = init(repopath, has_params ? Cuint(cmd[2] == "bare") : Cuint(0)) finalize(repo) println("Initialized empty Git repository in $repopath") catch ex @@ -41,11 +41,11 @@ function repl_cmd(ex) end try repourl = cmd[2] - repo = clone(repourl, repopath, isbare=("--bare" in cmd), ) + repo = clone(repourl, repopath, isbare=("bare" in cmd), ) finalize(repo) println("Cloned $repourl into $repopath") catch ex - println("usage: clone [--bare] ") + println("usage: clone [bare] ") warn(ex) end else @@ -65,7 +65,7 @@ function repl_cmd(ex) repl_log(repo, msg_count) elseif cmd[1] == "status" if has_params && cmd[2] == "help" - println("usage: status [-long]") + println("usage: status [long]") return end repl_status(repo, "long" in cmd) @@ -136,6 +136,12 @@ function repl_cmd(ex) return end repl_tag(repo, runcmd, cmd[end]) + elseif cmd[1] == "commit" + if !has_params || cmd[2] == "help" + println("usage: commit ") + return + end + repl_commit(repo, cmd[2:end]) else warn("unknown command: $ex. Use \'help\' command.") end @@ -150,6 +156,7 @@ function repl_help() add\t Add file contents to the index branch\t List, create, or delete branches clone\t Clone a repository into a current directory + commit\t Commit record changes to the repository init\t Create an empty Git repository or reinitialize an existing one in current directory log\t Show commit logs reset\t\ Reset current HEAD to the specified state @@ -160,7 +167,7 @@ function repl_help() For particular command parameters use: help """) # checkout\t Checkout a branch or paths to the working tree -# commit\t Record changes to the repository + end function repl_add{T<:AbstractString}(repo::GitRepo, force_add::Bool, files::Vector{T}) @@ -258,6 +265,11 @@ function repl_tag{T<:AbstractString}(repo::GitRepo, cmd::Symbol, tag::T) return end +function repl_commit{T<:AbstractString}(repo::GitRepo, msg::Vector{T}) + m = chomp(join(msg, " ")) + commit(repo, m) +end + function repl_log(repo::GitRepo, msg_count::Int) msgs = with(GitRevWalker(repo)) do walker map((oid,r)->with(get(GitCommit, r, oid)) do cmt From 243e1a8dd32b8018e0bef6c10b6e5b8beedea1c2 Mon Sep 17 00:00:00 2001 From: wildart Date: Sun, 2 Aug 2015 18:44:24 -0400 Subject: [PATCH 0349/1938] fix Git REPL `commit` output --- base/libgit2/repl.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/libgit2/repl.jl b/base/libgit2/repl.jl index fac11d573d0ae..11d1bb990ee35 100644 --- a/base/libgit2/repl.jl +++ b/base/libgit2/repl.jl @@ -8,7 +8,7 @@ List of git commands: - [x] branch - [x] clone - [ ] checkout -- [ ] commit +- [x] commit - [x] init - [x] log - [x] reset @@ -266,8 +266,9 @@ function repl_tag{T<:AbstractString}(repo::GitRepo, cmd::Symbol, tag::T) end function repl_commit{T<:AbstractString}(repo::GitRepo, msg::Vector{T}) - m = chomp(join(msg, " ")) + m = chomp(join(msg, " "))*"\n" commit(repo, m) + return end function repl_log(repo::GitRepo, msg_count::Int) From 7aa333b749bbba1df10c0a90719b0ed3374cb1df Mon Sep 17 00:00:00 2001 From: wildart Date: Sun, 2 Aug 2015 19:40:08 -0400 Subject: [PATCH 0350/1938] added `checkout` to Git REPL --- base/libgit2.jl | 3 +- base/libgit2/repl.jl | 76 +++++++++++++++++++++++++------------- base/libgit2/repository.jl | 10 +++++ 3 files changed, 62 insertions(+), 27 deletions(-) diff --git a/base/libgit2.jl b/base/libgit2.jl index c0c21ad51ed0c..7e496c8319773 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -216,7 +216,8 @@ function branch!(repo::GitRepo, branch_name::AbstractString, end end try - if !isempty(track) # setup tracking #TODO: what is branch tracks other then "origin" remote + #TODO: what if branch tracks other then "origin" remote + if !isempty(track) # setup tracking try with(GitConfig, repo) do cfg set!(cfg, "branch.$branch_name.remote", GitConst.REMOTE_ORIGIN) diff --git a/base/libgit2/repl.jl b/base/libgit2/repl.jl index 11d1bb990ee35..179321b408739 100644 --- a/base/libgit2/repl.jl +++ b/base/libgit2/repl.jl @@ -7,7 +7,7 @@ List of git commands: - [x] add - [x] branch - [x] clone -- [ ] checkout +- [x] checkout - [x] commit - [x] init - [x] log @@ -142,6 +142,12 @@ function repl_cmd(ex) return end repl_commit(repo, cmd[2:end]) + elseif cmd[1] == "checkout" + if !has_params || cmd[2] == "help" + println("usage: checkout (|)") + return + end + repl_checkout(repo, cmd[2]) else warn("unknown command: $ex. Use \'help\' command.") end @@ -152,22 +158,20 @@ function repl_cmd(ex) end function repl_help() - println("""List of git commands: - add\t Add file contents to the index - branch\t List, create, or delete branches - clone\t Clone a repository into a current directory - commit\t Commit record changes to the repository - init\t Create an empty Git repository or reinitialize an existing one in current directory - log\t Show commit logs - reset\t\ Reset current HEAD to the specified state - rm\t Remove files from the working tree and from the index - status\t Show the working tree status - tag\t Create, list or delete a tag objects - -For particular command parameters use: help -""") -# checkout\t Checkout a branch or paths to the working tree - + println("""List of commands: + add Add file contents to the index + branch List, create, or delete branches + checkout Checkout a branch or paths to the working tree + clone Clone a repository into a current directory + commit Commit record changes to the repository + init Create an empty Git repository or reinitialize an existing one in current directory + log Show commit logs + reset Reset current HEAD to the specified state + rm Remove files from the working tree and from the index + status Show the working tree status + tag Create, list or delete a tag objects + +For particular command parameters use: help""") end function repl_add{T<:AbstractString}(repo::GitRepo, force_add::Bool, files::Vector{T}) @@ -203,14 +207,8 @@ function repl_branch{T<:AbstractString}(repo::GitRepo, cmd::Symbol, branch::T) finalize(x[1]) end, bi) end - # get HEAD name - head_name = with(head(repo)) do href - if isattached(repo) - shortname(href) - else - "(detached from $(string(Oid(href))[1:7]))" - end - end + + head_name = headname(repo) # get HEAD name # prepare output output = Tuple{AbstractString,Symbol}[] @@ -267,7 +265,33 @@ end function repl_commit{T<:AbstractString}(repo::GitRepo, msg::Vector{T}) m = chomp(join(msg, " "))*"\n" - commit(repo, m) + cmd_oid = commit(repo, m) + head_name = headname(repo) # get HEAD name + print("[$head_name $(string(cmd_oid)[1:7])] $m") + return +end + +function repl_checkout{T<:AbstractString}(repo::GitRepo, branch::T) + # get branch tree + btree_oid = revparseid(repo, "$branch^{tree}") + + # checout selected branch + with(get(GitTree, repo, btree_oid)) do btree + checkout_tree(repo, btree) + end + + # switch head to the branch + branch_ref = lookup_branch(repo, branch) + if branch_ref != nothing + head!(repo, branch_ref) + finalize(branch_ref) + else + # detatch HEAD + boid = revparseid(repo, branch) + checkout!(repo, string(boid)) + end + + println("Switched to ",(branch_ref == nothing ? "" : "branch "),"'$branch'") return end diff --git a/base/libgit2/repository.jl b/base/libgit2/repository.jl index 2b16abcb41bdb..6ef36e051e2f5 100644 --- a/base/libgit2/repository.jl +++ b/base/libgit2/repository.jl @@ -48,6 +48,16 @@ function head_oid(repo::GitRepo) return oid end +function headname(repo::GitRepo) + with(head(repo)) do href + if isattached(repo) + shortname(href) + else + "(detached from $(string(Oid(href))[1:7]))" + end + end +end + function isbare(repo::GitRepo) return ccall((:git_repository_is_bare, :libgit2), Cint, (Ptr{Void},), repo.ptr) == 1 end From 2f8c7b7ac44f851d8847b214bcd9f354eef84b88 Mon Sep 17 00:00:00 2001 From: wildart Date: Sun, 2 Aug 2015 19:54:01 -0400 Subject: [PATCH 0351/1938] fixed remote branch name --- base/pkg/entry.jl | 2 +- base/pkg/github.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 5137a490153a5..b84f694c78108 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -420,7 +420,7 @@ function publish(branch::AbstractString) ahead_local == 0 && throw(PkgError("There are no METADATA changes to publish")) # get changed files - for path in LibGit2.diff_files(repo, "origin/metadata-v2", LibGit2.GitConst.HEAD_FILE) + for path in LibGit2.diff_files(repo, "origin/$branch", LibGit2.GitConst.HEAD_FILE) m = match(r"^(.+?)/versions/([^/]+)/sha1$", path) m != nothing && ismatch(Base.VERSION_REGEX, m.captures[2]) || continue pkg, ver = m.captures; ver = convert(VersionNumber,ver) diff --git a/base/pkg/github.jl b/base/pkg/github.jl index f0f449d512ce2..cef5028f2e569 100644 --- a/base/pkg/github.jl +++ b/base/pkg/github.jl @@ -13,7 +13,7 @@ const AUTH_DATA = Dict{Any,Any}( function user() usr = LibGit2.getconfig("github.user", "") - if isempty(usr) + if isempty(usr) #TODO: add `config` command to Git REPL and change below info throw(PkgError(""" no GitHub user name configured; please configure it with: From 2853db0fd9c90eb56bbe0e5b1647c1c7fa9659a8 Mon Sep 17 00:00:00 2001 From: wildart Date: Sun, 2 Aug 2015 19:54:35 -0400 Subject: [PATCH 0352/1938] reomoved Git module and its tests --- base/pkg/git.jl | 124 ----------------------------------------------- test/git.jl | 39 --------------- test/gitutils.jl | 114 ------------------------------------------- 3 files changed, 277 deletions(-) delete mode 100644 base/pkg/git.jl delete mode 100644 test/git.jl delete mode 100644 test/gitutils.jl diff --git a/base/pkg/git.jl b/base/pkg/git.jl deleted file mode 100644 index 32e492f074dc1..0000000000000 --- a/base/pkg/git.jl +++ /dev/null @@ -1,124 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -module Git -# -# some utility functions for working with git repos -# -import Base: shell_escape - -function dir(d) - g = joinpath(d,".git") - isdir(g) && return g - normpath(d, Base.readchomp(setenv(`git rev-parse --git-dir`, dir=d))) -end - -function git(d) - isempty(d) && return `git` - work_tree = abspath(d) - git_dir = joinpath(work_tree, dir(work_tree)) - normpath(work_tree, ".") == normpath(git_dir, ".") ? # is it a bare repo? - `git --git-dir=$work_tree` : `git --work-tree=$work_tree --git-dir=$git_dir` -end - -cmd(args::Cmd; dir="") = `$(git(dir)) $args` -run(args::Cmd; dir="", out=STDOUT) = Base.run(pipeline(cmd(args,dir=dir), out)) -readall(args::Cmd; dir="") = Base.readall(cmd(args,dir=dir)) -readchomp(args::Cmd; dir="") = Base.readchomp(cmd(args,dir=dir)) - -function success(args::Cmd; dir="") - g = git(dir) - Base.readchomp(`$g rev-parse --is-bare-repository`) == "false" && - Base.run(`$g update-index -q --really-refresh`) - Base.success(`$g $args`) -end - -function version() - vs = split(readchomp(`version`), ' ')[3] - ns = split(vs, '.') - if length(ns) > 3 - VersionNumber(join(ns[1:3], '.')) - else - VersionNumber(join(ns, '.')) - end -end - -modules(args::Cmd; dir="") = readchomp(`config -f .gitmodules $args`, dir=dir) -different(verA::AbstractString, verB::AbstractString, path::AbstractString; dir="") = - !success(`diff-tree --quiet $verA $verB -- $path`, dir=dir) - -dirty(; dir="") = !success(`diff-index --quiet HEAD`, dir=dir) -staged(; dir="") = !success(`diff-index --quiet --cached HEAD`, dir=dir) -unstaged(; dir="") = !success(`diff-files --quiet`, dir=dir) -dirty(paths; dir="") = !success(`diff-index --quiet HEAD -- $paths`, dir=dir) -staged(paths; dir="") = !success(`diff-index --quiet --cached HEAD -- $paths`, dir=dir) -unstaged(paths; dir="") = !success(`diff-files --quiet -- $paths`, dir=dir) - -iscommit(name; dir="") = success(`cat-file commit $name`, dir=dir) -attached(; dir="") = success(`symbolic-ref -q HEAD`, dir=dir) -branch(; dir="") = readchomp(`rev-parse --symbolic-full-name --abbrev-ref HEAD`, dir=dir) -head(; dir="") = readchomp(`rev-parse HEAD`, dir=dir) - -function iscommit(sha1s::Vector; dir="") - indexin(sha1s,split(readchomp(`log --all --format=%H`, dir=dir),"\n")).!=0 -end - -immutable State - head::ASCIIString - index::ASCIIString - work::ASCIIString -end - -function snapshot(; dir="") - head = readchomp(`rev-parse HEAD`, dir=dir) - index = readchomp(`write-tree`, dir=dir) - work = try - if length(readdir(abspath(dir))) > 1 - run(`add --all`, dir=dir) - run(`add .`, dir=dir) - end - readchomp(`write-tree`, dir=dir) - finally - run(`read-tree $index`, dir=dir) # restore index - end - State(head, index, work) -end - -function restore(s::State; dir="") - run(`reset -q --`, dir=dir) # unstage everything - run(`read-tree $(s.work)`, dir=dir) # move work tree to index - run(`checkout-index -fa`, dir=dir) # check the index out to work - run(`clean -qdf`, dir=dir) # remove everything else - run(`read-tree $(s.index)`, dir=dir) # restore index - run(`reset -q --soft $(s.head)`, dir=dir) # restore head -end - -function transact(f::Function; dir="") - state = snapshot(dir=dir) - try f() catch - restore(state, dir=dir) - rethrow() - end -end - -function is_ancestor_of(a::AbstractString, b::AbstractString; dir="") - A = readchomp(`rev-parse $a`, dir=dir) - readchomp(`merge-base $A $b`, dir=dir) == A -end - -const GITHUB_REGEX = - r"^(?:git@|git://|https://(?:[\w\.\+\-]+@)?)github.com[:/](([^/].+)/(.+?))(?:\.git)?$"i - -function set_remote_url(url::AbstractString; remote::AbstractString="origin", dir="") - run(`config remote.$remote.url $url`, dir=dir) - m = match(GITHUB_REGEX,url) - m === nothing && return - push = "git@github.com:$(m.captures[1]).git" - push != url && run(`config remote.$remote.pushurl $push`, dir=dir) -end - -function normalize_url(url::AbstractString) - m = match(GITHUB_REGEX,url) - m === nothing ? url : "git://github.com/$(m.captures[1]).git" -end - -end # module diff --git a/test/git.jl b/test/git.jl deleted file mode 100644 index a5e5ed7a2673e..0000000000000 --- a/test/git.jl +++ /dev/null @@ -1,39 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -import Base.Git -include("gitutils.jl") - -@test Git.version() >= v"1.7.3" - -dir = string("tmp.",randstring()) -@test !ispath(dir) -mkdir(dir) -@test isdir(dir) -try cd(dir) do - - run(`git init -q`) - run(`git config user.name "Julia Tester"`) - run(`git config user.email test@julialang.org`) - run(`git commit -q --allow-empty -m "initial empty commit"`) - git_verify(Dict(), Dict(), Dict()) - - # each path can have one of these content in each of head, index, work - # for a total of length(contents)^3 = 4^3 = 64 combinations. - # each path can be in any of these 64 "superpositions" before & after - # for a total of 64^2 = 4096 files needed to test all transitions - # between before and after superpositions of git repo states. - - contents = [nothing, "foo", "bar", Dict{Any,Any}("baz"=>"qux")] - b = length(contents) - states = [ [ base(b,k,6) => contents[rem(div(k,b^p),b)+1] for k=0:(b^3)^2-1 ] for p=0:5 ] - - git_setup(states[1:3]...) - try Git.transact() do - git_setup(states[4:6]...) - throw(nothing) - end catch x - is(x,nothing) || rethrow() - end - git_verify(states[1:3]...) - -end finally rm(dir, recursive=true) end diff --git a/test/gitutils.jl b/test/gitutils.jl deleted file mode 100644 index 9485719ebf880..0000000000000 --- a/test/gitutils.jl +++ /dev/null @@ -1,114 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -function write_and_readchomp(data, cmd::Cmd) - r, w, p = readandwrite(cmd) - print(w,data); close(w) - v = readchomp(r) - wait(p) - return v -end - -function mktree(d::Dict) - lstree = "" - for (name, data) in d - if isa(data, AbstractString) - sha1 = write_and_readchomp(data, `git hash-object -w --stdin`) - lstree *= "100644 blob $sha1\t$name\n" - elseif isa(data, Dict) - sha1 = mktree(data) - lstree *= "040000 tree $sha1\t$name\n" - elseif is(data, nothing) - # ignore it - else - error("mktree: don't know what to do with $name => $data") - end - end - write_and_readchomp(lstree, `git mktree`) -end - -function verify_tree(d::Dict, tree::AbstractString) - # check that tree matches d - seen = Set() - for line in eachline(`git ls-tree $tree`) - m = match(r"^(\d{6}) (\w+) ([0-9a-f]{40})\t(.*)$", line) - @test m != nothing - perm, kind, sha1, name = m.captures - @test haskey(d,name) - data = d[name] - if isa(data, AbstractString) - @test kind == "blob" - @test data == readall(`git cat-file blob $sha1`) - elseif isa(data, Dict) - @test kind == "tree" - verify_tree(data, sha1) - else - error("verify_tree: don't know what to do with $name => $data") - end - push!(seen, name) - end - # check that nothing was missing from tree - for (name, data) in d - @test is(data,nothing) || in(name,seen) - end -end - -function verify_work(d::Dict) - # check what's in d - for (name, data) in d - if is(data, nothing) - @test !ispath(name) - continue - end - @test ispath(name) - if isa(data, AbstractString) - @test isfile(name) - @test readall(name) == data - elseif isa(data, Dict) - cd(name) do - verify_work(data) - end - else - error("verify_work: don't know what to do with $name => $data") - end - end - # check for anything that's not in d - for line in eachline(`ls -A`) - name = chomp(line) - @test name == ".git" || haskey(d,name) - end -end - -function git_verify(h::Dict, i::Dict, w::Dict) - verify_tree(h, "HEAD") - verify_tree(i, readchomp(`git write-tree`)) - verify_work(w) -end - -function git_setup(h::Dict, i::Dict, w::Dict, parents::AbstractString...) - # create tree objects - headt = mktree(h) - index = mktree(i) - work = mktree(w) - - # clear the repo - for line in eachline(`ls -A`) - name = chomp(line) - name == ".git" || rm(name, recursive=true) - end - - # create the head commit - commit_tree = `git commit-tree $headt` - for parent in parents - commit_tree = `$commit_tree -p $parent` - end - head = write_and_readchomp(headt, commit_tree) - run(`git reset -q --soft $head`) - - run(`git read-tree $work`) # read work into the index - run(`git checkout-index -fa`) # check the index out - run(`git read-tree $index`) # setup the index - - # verify that everything is as expected - git_verify(h, i, w) -end -git_setup(h::Dict, i::Dict, w::Dict) = git_setup(h, i, w, readchomp(`git rev-parse HEAD`)) From 0ccfdfe567db1497195bc5c15af08e923cc04e93 Mon Sep 17 00:00:00 2001 From: wildart Date: Sun, 2 Aug 2015 20:04:48 -0400 Subject: [PATCH 0353/1938] fix error message & add all tests --- appveyor.yml | 2 +- base/pkg/dir.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d20034cfb1de5..1634ea19f6f6a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -49,4 +49,4 @@ build_script: test_script: - usr\bin\julia -e "versioninfo()" - copy usr\lib\julia\sys.ji local.ji && usr\bin\julia -J local.ji -e "true" && del local.ji - - cd test && ..\usr\bin\julia runtests.jl libgit2 repl pkg + - cd test && ..\usr\bin\julia runtests.jl all && ..\usr\bin\julia runtests.jl pkg diff --git a/base/pkg/dir.jl b/base/pkg/dir.jl index 0505792c0acb0..4925eee06640f 100644 --- a/base/pkg/dir.jl +++ b/base/pkg/dir.jl @@ -32,7 +32,7 @@ function cd(f::Function, args...; kws...) Base.cd(()->f(args...; kws...), dir) catch err if isa(err, PkgError) - print_with_color(:red, "PkgMgr Error: $(err.msg)") + print_with_color(:red, "ERROR: $(err.msg)") else throw(err) end From 4beb9af21246e10de4d8f2b9762f6f18324dd3f0 Mon Sep 17 00:00:00 2001 From: wildart Date: Mon, 3 Aug 2015 19:33:23 -0400 Subject: [PATCH 0354/1938] fixed cache update, remove redundant `chomp` call from Git REPL `commit`, added `fetch` and `push` to Git REPL --- base/libgit2.jl | 8 ++++++-- base/libgit2/repl.jl | 41 ++++++++++++++++++++++++++++++++++++++--- base/pkg/cache.jl | 4 ++-- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/base/libgit2.jl b/base/libgit2.jl index 7e496c8319773..4322ae948c4e5 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -58,7 +58,11 @@ function iscommit(id::AbstractString, repo::GitRepo) res = true try c = get(GitCommit, repo, id) - finalize(c) + if c === nothing + res = false + else + finalize(c) + end catch res = false end @@ -158,9 +162,9 @@ end """ git push [|] []""" function push{T<:AbstractString}(repo::GitRepo; - refspecs::Vector{T}=AbstractString[], remote::AbstractString="origin", remoteurl::AbstractString="", + refspecs::Vector{T}=AbstractString[], force::Bool=false) rmt = if isempty(remoteurl) get(GitRemote, repo, remote) diff --git a/base/libgit2/repl.jl b/base/libgit2/repl.jl index 179321b408739..9ce706b6e091c 100644 --- a/base/libgit2/repl.jl +++ b/base/libgit2/repl.jl @@ -9,14 +9,15 @@ List of git commands: - [x] clone - [x] checkout - [x] commit +- [x] fetch - [x] init - [x] log +- [x] push - [x] reset - [x] rm - [x] status - [x] tag """ - function repl_cmd(ex) cmd = split(ex) has_params = length(cmd) > 1 @@ -36,7 +37,7 @@ function repl_cmd(ex) end elseif cmd[1] == "clone" if has_params && cmd[2] == "help" - repl_clone_help() + println("usage: clone [bare] ") return end try @@ -148,6 +149,28 @@ function repl_cmd(ex) return end repl_checkout(repo, cmd[2]) + elseif cmd[1] == "fetch" + if has_params && cmd[2] == "help" + println("usage: fetch [] []") + return + end + url, rs = if has_params + length(cmd) == 3 ? (cmd[2], cmd[3]) : (cmd[2], "") + else + ("", "") + end + repl_fetch(repo, url, rs) + elseif cmd[1] == "push" + if has_params && cmd[2] == "help" + println("usage: push [] []") + return + end + url, rs = if has_params + length(cmd) == 3 ? (cmd[2], cmd[3]) : (cmd[2], "") + else + ("", "") + end + repl_push(repo, url, rs) else warn("unknown command: $ex. Use \'help\' command.") end @@ -164,8 +187,10 @@ function repl_help() checkout Checkout a branch or paths to the working tree clone Clone a repository into a current directory commit Commit record changes to the repository + fetch Download objects and refs from another repository init Create an empty Git repository or reinitialize an existing one in current directory log Show commit logs + push Update remote refs along with associated objects reset Reset current HEAD to the specified state rm Remove files from the working tree and from the index status Show the working tree status @@ -185,6 +210,16 @@ function repl_rm{T<:AbstractString}(repo::GitRepo, files::Vector{T}) return end +function repl_fetch{T<:AbstractString}(repo::GitRepo, url::T, refspec::T) + fetch(repo, remoteurl=url, refspecs=[refspec]) + return +end + +function repl_push{T<:AbstractString}(repo::GitRepo, url::T, refspec::T) + push(repo, remoteurl=url, refspecs=[refspec]) + return +end + function repl_reset{T<:AbstractString}(repo::GitRepo, files::Vector{T}) with(head(repo)) do href with(peel(GitCommit, href)) do hcommit @@ -264,7 +299,7 @@ function repl_tag{T<:AbstractString}(repo::GitRepo, cmd::Symbol, tag::T) end function repl_commit{T<:AbstractString}(repo::GitRepo, msg::Vector{T}) - m = chomp(join(msg, " "))*"\n" + m = join(msg, " ")*"\n" cmd_oid = commit(repo, m) head_name = headname(repo) # get HEAD name print("[$head_name $(string(cmd_oid)[1:7])] $m") diff --git a/base/pkg/cache.jl b/base/pkg/cache.jl index db984caab1a12..e074147eb632e 100644 --- a/base/pkg/cache.jl +++ b/base/pkg/cache.jl @@ -47,11 +47,11 @@ function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) end try LibGit2.set_remote_url(repo, url) - in_cache = map(sha1->LibGit2.iscommit(sha1, repo), sha1s) + in_cache = BitArray(map(sha1->LibGit2.iscommit(sha1, repo), sha1s)) if !all(in_cache) info("Updating cache of $pkg...") LibGit2.fetch(repo) - in_cache = map(sha1->LibGit2.iscommit(sha1, repo), sha1s) + in_cache = BitArray(map(sha1->LibGit2.iscommit(sha1, repo), sha1s)) end sha1s[!in_cache] finally From 7e9caae8eb7f897b585aa6bc25a70f019194d0c4 Mon Sep 17 00:00:00 2001 From: wildart Date: Sat, 8 Aug 2015 02:19:04 -0400 Subject: [PATCH 0355/1938] fixed: `tag` requirements creation, empty callbacks --- base/libgit2.jl | 4 ++-- base/libgit2/callbacks.jl | 6 +++--- base/libgit2/repository.jl | 3 ++- base/pkg/cache.jl | 2 +- base/pkg/entry.jl | 5 +++-- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/base/libgit2.jl b/base/libgit2.jl index 4322ae948c4e5..1f9b4e95dd03a 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -151,7 +151,7 @@ function fetch{T<:AbstractString}(repo::GitRepo; GitRemoteAnon(repo, remoteurl) end try - fo = FetchOptions(callbacks=RemoteCallbacks(credentials=credentials_cb)) + fo = FetchOptions(callbacks=RemoteCallbacks(credentials=credentials_cb())) fetch(rmt, refspecs, msg="from $(url(rmt))", options = fo) catch err warn("fetch: $err") @@ -172,7 +172,7 @@ function push{T<:AbstractString}(repo::GitRepo; GitRemoteAnon(repo, remoteurl) end try - po = PushOptions(callbacks=RemoteCallbacks(credentials=credentials_cb)) + po = PushOptions(callbacks=RemoteCallbacks(credentials=credentials_cb())) push(rmt, refspecs, force=force, options=po) finally finalize(rmt) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 4d1f857f7cb31..d9896ccbf12d9 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -89,6 +89,6 @@ function fetchhead_foreach_callback(ref_name::Cstring, remote_url::Cstring, return Cint(0) end -const mirror_cb = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring, Ptr{Void})) -const credentials_cb = cfunction(credentials_callback, Cint, (Ptr{Ptr{Void}}, Cstring, Cstring, Cuint, Ptr{Void})) -const fetchhead_foreach_cb = cfunction(fetchhead_foreach_callback, Cint, (Cstring, Cstring, Ptr{Oid}, Cuint, Ptr{Void})) \ No newline at end of file +mirror_cb() = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring, Ptr{Void})) +credentials_cb() = cfunction(credentials_callback, Cint, (Ptr{Ptr{Void}}, Cstring, Cstring, Cuint, Ptr{Void})) +fetchhead_foreach_cb() = cfunction(fetchhead_foreach_callback, Cint, (Cstring, Cstring, Ptr{Oid}, Cuint, Ptr{Void})) \ No newline at end of file diff --git a/base/libgit2/repository.jl b/base/libgit2/repository.jl index 6ef36e051e2f5..d11fa270de374 100644 --- a/base/libgit2/repository.jl +++ b/base/libgit2/repository.jl @@ -190,9 +190,10 @@ end function fetchheads(repo::GitRepo) fhr = Ref{Vector{FetchHead}}(FetchHead[]) + ffcb = fetchhead_foreach_cb() @check ccall((:git_repository_fetchhead_foreach, :libgit2), Cint, (Ptr{Void}, Ptr{Void}, Ptr{Void}), - repo.ptr, fetchhead_foreach_cb, fhr) + repo.ptr, ffcb, fhr) return fhr[] end diff --git a/base/pkg/cache.jl b/base/pkg/cache.jl index e074147eb632e..1f8f9bf0f0925 100644 --- a/base/pkg/cache.jl +++ b/base/pkg/cache.jl @@ -39,7 +39,7 @@ function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) info("Cloning cache of $pkg from $url") try # clone repo, free it at the end - LibGit2.clone(url, cache, isbare = true, remote_cb = LibGit2.mirror_cb) + LibGit2.clone(url, cache, isbare = true, remote_cb = LibGit2.mirror_cb()) catch err isdir(cache) && rm(cache, recursive=true) throw(PkgError("Cannot clone $pkg from $url. $(err.msg)")) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index b84f694c78108..b094ef761a8ff 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -551,7 +551,9 @@ function resolve( end function write_tag_metadata(repo::GitRepo, pkg::AbstractString, ver::VersionNumber, commit::AbstractString, force::Bool=false) - content = LibGit2.cat(repo, LibGit2.GitBlob, "$commit:REQUIRE") + content = with(GitRepo,pkg) do pkg_repo + LibGit2.cat(pkg_repo, LibGit2.GitBlob, "$commit:REQUIRE") + end reqs = content != nothing ? Reqs.read(split(content, '\n', keep=false)) : Reqs.Line[] cd("METADATA") do d = joinpath(pkg,"versions",string(ver)) @@ -686,7 +688,6 @@ function tag(pkg::AbstractString, ver::Union(Symbol,VersionNumber), force::Bool= if LibGit2.isdirty(repo) info("Committing METADATA for $pkg") LibGit2.commit(repo, "Tag $pkg v$ver") - #run(`commit -q -m "Tag $pkg v$ver" -- $pkg`, dir="METADATA") else info("No METADATA changes to commit") end From 78929ad8daf796d62f526c45382eee02f457b1e5 Mon Sep 17 00:00:00 2001 From: wildart Date: Sun, 13 Sep 2015 23:19:42 -0400 Subject: [PATCH 0356/1938] fixed tests --- test/libgit2.jl | 8 ++++---- test/pkg.jl | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/test/libgit2.jl b/test/libgit2.jl index 85795205b85fc..69e0cdf9490af 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -69,7 +69,7 @@ end temp_dir() do dir repo_url = "https://github.com/JuliaLang/Example.jl" repo_path = joinpath(dir, "Example.Bare") - repo = LibGit2.clone(repo_url, repo_path, isbare = true, remote_cb = LibGit2.mirror_cb) + repo = LibGit2.clone(repo_url, repo_path, isbare = true, remote_cb = LibGit2.mirror_cb()) finalize(repo) @test isdir(repo_path) @test isfile(joinpath(repo_path, LibGit2.GitConst.HEAD_FILE)) @@ -113,7 +113,7 @@ temp_dir() do dir_cache # create cache url = "https://github.com/JuliaLang/Example.jl" path_cache = joinpath(dir_cache, "Example.Bare") - repo = LibGit2.clone(url, path_cache, isbare = true, remote_cb = LibGit2.mirror_cb) + repo = LibGit2.clone(url, path_cache, isbare = true, remote_cb = LibGit2.mirror_cb()) LibGit2.with(LibGit2.GitConfig, repo) do cfg credentials!(cfg) end @@ -129,10 +129,10 @@ temp_dir() do dir_cache end LibGit2.fetch(repo) - refs1 = parse(Int, readchomp(pipe(`find $(joinpath(path, ".git/refs"))`,`wc -l`))) + refs1 = parse(Int, readchomp(pipeline(`find $(joinpath(path, ".git/refs"))`,`wc -l`))) LibGit2.fetch(repo, remoteurl=path_cache, refspecs =["+refs/*:refs/remotes/cache/*"]) - refs2 = parse(Int, readchomp(pipe(`find $(joinpath(path, ".git/refs"))`,`wc -l`))) + refs2 = parse(Int, readchomp(pipeline(`find $(joinpath(path, ".git/refs"))`,`wc -l`))) finalize(repo) @test refs1 > 0 diff --git a/test/pkg.jl b/test/pkg.jl index 506ad8787e827..784e4e7ae295d 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -41,7 +41,6 @@ temp_pkg_dir() do Pkg.rm("Example") @test isempty(Pkg.installed()) @test !isempty(Pkg.available("Example")) - @test_throws ErrorException Pkg.available("FakePackageDoesn'tExist") Pkg.clone("https://github.com/JuliaLang/Example.jl.git") @test [keys(Pkg.installed())...] == ["Example"] Pkg.status("Example", iob) From 29004a47ebaa0baaee698189e610b83fe421ec4f Mon Sep 17 00:00:00 2001 From: wildart Date: Mon, 14 Sep 2015 00:35:21 -0400 Subject: [PATCH 0357/1938] removed Git REPL --- base/REPL.jl | 29 +-- base/libgit2.jl | 1 - base/libgit2/repl.jl | 546 ------------------------------------------- test/repl.jl | 23 +- 4 files changed, 8 insertions(+), 591 deletions(-) delete mode 100644 base/libgit2/repl.jl diff --git a/base/REPL.jl b/base/REPL.jl index 95b5bfc7ddf86..f4cc49f49a527 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -734,17 +734,6 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep Expr(:call, :(Base.repl_cmd), macroexpand(Expr(:macrocall, symbol("@cmd"),line)), outstream(repl)) end) - # Set up git mode - git_mode = Prompt("git> "; - prompt_prefix = hascolor ? repl.shell_color : "", - prompt_suffix = hascolor ? - (repl.envcolors ? Base.input_color : repl.input_color) : "", - keymap_func_data = repl, - complete = ShellCompletionProvider(repl), - on_done = respond(repl, julia_prompt) do line - line = strip(line) - :(LibGit2.repl_cmd($line)) - end) ################################# Stage II ############################# @@ -752,8 +741,7 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep # We will have a unified history for all REPL modes hp = REPLHistoryProvider(Dict{Symbol,Any}(:julia => julia_prompt, :shell => shell_mode, - :help => help_mode, - :git => git_mode)) + :help => help_mode)) if repl.history_file try f = open(find_hist_file(), true, true, true, false, false) @@ -770,7 +758,6 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep julia_prompt.hist = hp shell_mode.hist = hp help_mode.hist = hp - git_mode.hist = hp search_prompt, skeymap = LineEdit.setup_search_keymap(hp) search_prompt.complete = LatexCompletions() @@ -801,16 +788,6 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep edit_insert(s, '?') end end, - '>' => function (s,o...) - if isempty(s) || position(LineEdit.buffer(s)) == 0 - buf = copy(LineEdit.buffer(s)) - transition(s, git_mode) - LineEdit.state(s, git_mode).input_buffer = buf - LineEdit.refresh_line(s) - else - edit_insert(s, '>') - end - end, # Bracketed Paste Mode "\e[200~" => (s,o...)->begin @@ -869,9 +846,9 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep b = Dict{Any,Any}[skeymap, mk, prefix_keymap, LineEdit.history_keymap, LineEdit.default_keymap, LineEdit.escape_defaults] prepend!(b, extra_repl_keymap) - shell_mode.keymap_dict = help_mode.keymap_dict = git_mode.keymap_dict = LineEdit.keymap(b) + shell_mode.keymap_dict = help_mode.keymap_dict = LineEdit.keymap(b) - ModalInterface([julia_prompt, shell_mode, help_mode, git_mode, search_prompt, prefix_prompt]) + ModalInterface([julia_prompt, shell_mode, help_mode, search_prompt, prefix_prompt]) end function run_frontend(repl::LineEditREPL, backend) diff --git a/base/libgit2.jl b/base/libgit2.jl index 1f9b4e95dd03a..c871b350ff17f 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -29,7 +29,6 @@ include("libgit2/diff.jl") include("libgit2/rebase.jl") include("libgit2/status.jl") include("libgit2/callbacks.jl") -include("libgit2/repl.jl") include("libgit2/utils.jl") immutable State diff --git a/base/libgit2/repl.jl b/base/libgit2/repl.jl deleted file mode 100644 index 9ce706b6e091c..0000000000000 --- a/base/libgit2/repl.jl +++ /dev/null @@ -1,546 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -""" -# Process REPL command in git mode - -List of git commands: -- [x] add -- [x] branch -- [x] clone -- [x] checkout -- [x] commit -- [x] fetch -- [x] init -- [x] log -- [x] push -- [x] reset -- [x] rm -- [x] status -- [x] tag -""" -function repl_cmd(ex) - cmd = split(ex) - has_params = length(cmd) > 1 - repopath = pwd() - - if cmd[1] == "init" - if has_params && cmd[2] == "help" - println("usage: init [bare]") - return - end - try - repo = init(repopath, has_params ? Cuint(cmd[2] == "bare") : Cuint(0)) - finalize(repo) - println("Initialized empty Git repository in $repopath") - catch ex - warn(ex) - end - elseif cmd[1] == "clone" - if has_params && cmd[2] == "help" - println("usage: clone [bare] ") - return - end - try - repourl = cmd[2] - repo = clone(repourl, repopath, isbare=("bare" in cmd), ) - finalize(repo) - println("Cloned $repourl into $repopath") - catch ex - println("usage: clone [bare] ") - warn(ex) - end - else - repo = GitRepoExt(repopath) - try - if cmd[1] == "help" - repl_help() - elseif cmd[1] == "log" - msg_count = 10 - if has_params - msg_count = Base.get(tryparse(Int, cmd[2]), 0) - if msg_count == 0 || cmd[2] == "help" - println("usage: log []") - return - end - end - repl_log(repo, msg_count) - elseif cmd[1] == "status" - if has_params && cmd[2] == "help" - println("usage: status [long]") - return - end - repl_status(repo, "long" in cmd) - elseif cmd[1] == "add" - if has_params && (cmd[2] == "help" || (cmd[2] == "force" && length(cmd) == 2)) - println("usage: add [force] ...") - return - end - force_add = has_params && cmd[2] == "force" - repl_add(repo, force_add, cmd[(force_add ? 3 : 2):end]) - elseif cmd[1] == "rm" - if has_params && cmd[2] == "help" - println("usage: rm ...") - return - end - repl_rm(repo, cmd[2:end]) - elseif cmd[1] == "reset" - if has_params && cmd[2] == "help" - println("usage: reset ...") - return - end - repl_reset(repo, cmd[2:end]) - elseif cmd[1] == "branch" - runcmd = if has_params - if cmd[2] == "local" - :listlocal - elseif cmd[2] == "remote" - :listremote - elseif cmd[2] == "all" - :listall - elseif length(cmd) == 3 - if cmd[2] == "add" - :add - elseif cmd[2] == "delete" - :delete - else - :help - end - else - :help - end - else - :listlocal - end - if runcmd == :help - println("usage: branch [(local|remote|all)|(add|delete) ]") - return - end - repl_branch(repo, runcmd, cmd[end]) - elseif cmd[1] == "tag" - runcmd = if has_params - if length(cmd) == 3 - if cmd[2] == "add" - :add - elseif cmd[2] == "delete" - :delete - else - :help - end - else - :help - end - else - :list - end - if runcmd == :help - println("usage: tag [(add|delete) ]") - return - end - repl_tag(repo, runcmd, cmd[end]) - elseif cmd[1] == "commit" - if !has_params || cmd[2] == "help" - println("usage: commit ") - return - end - repl_commit(repo, cmd[2:end]) - elseif cmd[1] == "checkout" - if !has_params || cmd[2] == "help" - println("usage: checkout (|)") - return - end - repl_checkout(repo, cmd[2]) - elseif cmd[1] == "fetch" - if has_params && cmd[2] == "help" - println("usage: fetch [] []") - return - end - url, rs = if has_params - length(cmd) == 3 ? (cmd[2], cmd[3]) : (cmd[2], "") - else - ("", "") - end - repl_fetch(repo, url, rs) - elseif cmd[1] == "push" - if has_params && cmd[2] == "help" - println("usage: push [] []") - return - end - url, rs = if has_params - length(cmd) == 3 ? (cmd[2], cmd[3]) : (cmd[2], "") - else - ("", "") - end - repl_push(repo, url, rs) - else - warn("unknown command: $ex. Use \'help\' command.") - end - finally - finalize(repo) - end - end -end - -function repl_help() - println("""List of commands: - add Add file contents to the index - branch List, create, or delete branches - checkout Checkout a branch or paths to the working tree - clone Clone a repository into a current directory - commit Commit record changes to the repository - fetch Download objects and refs from another repository - init Create an empty Git repository or reinitialize an existing one in current directory - log Show commit logs - push Update remote refs along with associated objects - reset Reset current HEAD to the specified state - rm Remove files from the working tree and from the index - status Show the working tree status - tag Create, list or delete a tag objects - -For particular command parameters use: help""") -end - -function repl_add{T<:AbstractString}(repo::GitRepo, force_add::Bool, files::Vector{T}) - forced = force_add ? GitConst.INDEX_ADD_FORCE : GitConst.INDEX_ADD_DEFAULT - add!(repo, files..., flags = forced) - return -end - -function repl_rm{T<:AbstractString}(repo::GitRepo, files::Vector{T}) - remove!(repo, files...) - return -end - -function repl_fetch{T<:AbstractString}(repo::GitRepo, url::T, refspec::T) - fetch(repo, remoteurl=url, refspecs=[refspec]) - return -end - -function repl_push{T<:AbstractString}(repo::GitRepo, url::T, refspec::T) - push(repo, remoteurl=url, refspecs=[refspec]) - return -end - -function repl_reset{T<:AbstractString}(repo::GitRepo, files::Vector{T}) - with(head(repo)) do href - with(peel(GitCommit, href)) do hcommit - reset!(repo, Nullable(hcommit), files...) - end - end - return -end - -function repl_branch{T<:AbstractString}(repo::GitRepo, cmd::Symbol, branch::T) - if cmd == :listlocal || cmd == :listremote || cmd == :listall - flags = cmd == :listlocal ? Cint(GitConst.BRANCH_LOCAL) : ( - cmd == :listremote ? Cint(GitConst.BRANCH_REMOTE) : - Cint(GitConst.BRANCH_LOCAL) | Cint(GitConst.BRANCH_REMOTE)) - # walk through branches - branches = with(GitBranchIter(repo, flags)) do bi - map(x-> try - (shortname(x[1]), x[2]) - finally - finalize(x[1]) - end, bi) - end - - head_name = headname(repo) # get HEAD name - - # prepare output - output = Tuple{AbstractString,Symbol}[] - for b in branches - c = if b[1] == head_name - :green - elseif b[2] == 1 - :white - else - :red - end - push!(output, (b[1], c)) - end - - # if HEAD does not have branch insert it in output - cmd == :listremote || any(map(b->b[2] == :green, output)) || insert!(output, 1, (head_name, :green)) - for (b, c) in output - c == :green ? print("* ") : print(" ") - print_with_color(c, "$b\n") - end - elseif cmd == :add - with(head(repo)) do href - with(peel(GitCommit, href)) do hcommit - create_branch(repo, hcommit, branch) - end - end - elseif cmd == :delete - bref = lookup_branch(repo, branch) - bref != nothing && try - delete_branch(bref) - finally - finalize(bref) - end - end - return -end - -function repl_tag{T<:AbstractString}(repo::GitRepo, cmd::Symbol, tag::T) - if cmd == :list - for t in sort(tag_list(repo)) - println(t) - end - elseif cmd == :add - with(head(repo)) do href - with(peel(GitCommit, href)) do hcommit - tag_create(repo, tag, string(Oid(hcommit))) - end - end - elseif cmd == :delete - tag_delete(repo, tag) - end - return -end - -function repl_commit{T<:AbstractString}(repo::GitRepo, msg::Vector{T}) - m = join(msg, " ")*"\n" - cmd_oid = commit(repo, m) - head_name = headname(repo) # get HEAD name - print("[$head_name $(string(cmd_oid)[1:7])] $m") - return -end - -function repl_checkout{T<:AbstractString}(repo::GitRepo, branch::T) - # get branch tree - btree_oid = revparseid(repo, "$branch^{tree}") - - # checout selected branch - with(get(GitTree, repo, btree_oid)) do btree - checkout_tree(repo, btree) - end - - # switch head to the branch - branch_ref = lookup_branch(repo, branch) - if branch_ref != nothing - head!(repo, branch_ref) - finalize(branch_ref) - else - # detatch HEAD - boid = revparseid(repo, branch) - checkout!(repo, string(boid)) - end - - println("Switched to ",(branch_ref == nothing ? "" : "branch "),"'$branch'") - return -end - -function repl_log(repo::GitRepo, msg_count::Int) - msgs = with(GitRevWalker(repo)) do walker - map((oid,r)->with(get(GitCommit, r, oid)) do cmt - sig = author(cmt) - msg = message(cmt) - (Oid(cmt), sig, msg) - end, - walker, count = msg_count) - end - for msg in msgs - print_with_color(:yellow, "Commit: $(string(msg[1]))\n") - println("Author:\t$(msg[2].name) $(msg[2].email)") - println("Date:\t$(Dates.unix2datetime(msg[2].time))") - println('\t',join(split(msg[3],'\n'),"\n\t")) - end -end - -function repl_status(repo::GitRepo, islong::Bool=false) - isbare(repo) && error("Cannot report status on bare repository") - - # Show branch - with(head(repo)) do bref - bname = shortname(bref) - if islong - println("# On branch: ", isempty(bname) ? "Not currently on any branch." : bname) - else - println("## ", isempty(bname) ? "HEAD (no branch)" : bname) - end - end - - header = changes_in_index = changed_in_workdir = rm_in_workdir = false - - # Show status - with(GitStatus, repo) do status - for i in 1:length(status) - s = status[i] - s.status == GitConst.STATUS_CURRENT && continue - - istatus = wstatus = " " - - if s.status & GitConst.STATUS_INDEX_NEW > 0 - istatus = islong ? "new file: " : "A" - end - if s.status & GitConst.STATUS_INDEX_MODIFIED > 0 - istatus = islong ? "modified: " : "M" - end - if s.status & GitConst.STATUS_INDEX_DELETED > 0 - istatus = islong ? "deleted: " : "D" - end - if s.status & GitConst.STATUS_INDEX_RENAMED > 0 - istatus = islong ? "renamed: " : "R" - end - if s.status & GitConst.STATUS_INDEX_TYPECHANGE > 0 - istatus = islong ? "typechange:" : "T" - end - - if s.status & GitConst.STATUS_WT_NEW > 0 - if istatus == " " && !islong - istatus = "?" - end - wstatus = "?" - end - if s.status & GitConst.STATUS_WT_MODIFIED > 0; wstatus = "M"; end - if s.status & GitConst.STATUS_WT_DELETED > 0 - wstatus = "D" - rm_in_workdir = true - end - if s.status & GitConst.STATUS_WT_RENAMED > 0; wstatus = "R"; end - if s.status & GitConst.STATUS_WT_TYPECHANGE > 0; wstatus = "T"; end - - if islong - istatus == " " && continue - else - if s.status & GitConst.STATUS_IGNORED > 0 - istatus = "!" - wstatus = "!" - end - istatus == "?" && wstatus == "?" && continue - end - - if islong - if !header - println(" Changes to be committed:"); - println(" (use \"reset ...\" to unstage)\n") - header = true - end - shi_diff = unsafe_load(convert(Ptr{DiffDelta}, s.head_to_index), 1) - old_path = shi_diff.old_file.path == Cstring_NULL ? "" : bytestring(shi_diff.old_file.path) - new_path = shi_diff.new_file.path == Cstring_NULL ? "" : bytestring(shi_diff.new_file.path) - - if !isempty(old_path) && !isempty(new_path) && old_path != new_path - println("\t$istatus $old_path -> $new_path") - else - println("\t$istatus $(isempty(old_path) ? new_path : old_path)") - end - else - a = b = c = "" - if s.head_to_index != C_NULL - shi_diff = unsafe_load(convert(Ptr{DiffDelta}, s.head_to_index), 1) - a = shi_diff.old_file.path == Cstring_NULL ? "" : bytestring(shi_diff.old_file.path) - b = shi_diff.new_file.path == Cstring_NULL ? "" : bytestring(shi_diff.new_file.path) - end - if s.index_to_workdir != C_NULL - siw_diff = unsafe_load(convert(Ptr{DiffDelta}, s.index_to_workdir), 1) - if isempty(a) - a = siw_diff.old_file.path == Cstring_NULL ? "" : bytestring(siw_diff.old_file.path) - end - if isempty(b) - b = siw_diff.old_file.path == Cstring_NULL ? "" : bytestring(siw_diff.old_file.path) - end - c = siw_diff.new_file.path == Cstring_NULL ? "" : bytestring(siw_diff.new_file.path) - end - - if istatus == "R" - if wstatus == "R" - println("$istatus$wstatus $a $b $c") - else - println("$istatus$wstatus $a $b") - end - else - if wstatus == "R" - println("$istatus$wstatus $a $c") - else - println("$istatus$wstatus $a") - end - end - end - end - - if islong - if header - changes_in_index = true - println("") - end - header = false - - # Print workdir changes to tracked files. - for i in 1:length(status) - s = status[i] - (s.status == GitConst.STATUS_CURRENT || s.index_to_workdir == C_NULL) && continue - - wstatus = "" - if s.status & GitConst.STATUS_WT_MODIFIED > 0; wstatus = "modified: "; end - if s.status & GitConst.STATUS_WT_DELETED > 0; wstatus = "deleted: "; end - if s.status & GitConst.STATUS_WT_RENAMED > 0; wstatus = "renamed: "; end - if s.status & GitConst.STATUS_WT_TYPECHANGE > 0; wstatus = "typechange:"; end - - isempty(wstatus) && continue - - if !header - println(" Changes not staged for commit:") - println(" (use \"add$(rm_in_workdir ? "/rm" : "") ...\" to update what will be committed)") - println(" (use \"checkout ...\" to discard changes in working directory)\n") - header = true - end - siw_diff = unsafe_load(convert(Ptr{DiffDelta}, s.index_to_workdir), 1) - old_path = siw_diff.old_file.path == Cstring_NULL ? "" : bytestring(siw_diff.old_file.path) - new_path = siw_diff.new_file.path == Cstring_NULL ? "" : bytestring(siw_diff.new_file.path) - - if !isempty(old_path) && !isempty(new_path) && old_path != new_path - println("\t$wstatus $old_path -> $new_path") - else - println("\t$wstatus $(isempty(old_path) ? new_path : old_path)") - end - end - - if header - changed_in_workdir = true - println("") - end - header = false - end - - # Print untracked files. - for i in 1:length(status) - s = status[i] - if s.status == GitConst.STATUS_WT_NEW - siw_diff = unsafe_load(convert(Ptr{DiffDelta}, s.index_to_workdir), 1) - - if islong - if !header - println(" Untracked files:") - println(" (use \"add ...\" to include in what will be committed)\n") - header = true - end - println("\t", bytestring(siw_diff.old_file.path)) - else - println("?? ", bytestring(siw_diff.old_file.path)) - end - end - end - - # Print ignored files (long version). - if islong - header = false - for i in 1:length(status) - s = status[i] - if s.status == GitConst.STATUS_IGNORED - if !header - println(" Ignored files:") - println(" (use \"add force ...\" to include in what will be committed)\n") - header = true - end - siw_diff = unsafe_load(convert(Ptr{DiffDelta}, s.index_to_workdir), 1) - println("\t", bytestring(siw_diff.old_file.path)) - end - end - - !changes_in_index && changed_in_workdir && println("no changes added to commit (use \"add\" and/or \"commit -a\")") - end - - end - return -end diff --git a/test/repl.jl b/test/repl.jl index 144c39e0dde2f..cec032d089628 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -199,9 +199,7 @@ fakehistory = """ # time: 2014-06-30 20:44:29 EDT # mode: julia \t2 + 2 -# time: 2014-06-30 20:44:29 EDT -# mode: git -\tinit""" +""" # Test various history related issues begin @@ -214,19 +212,17 @@ begin repl_mode = repl.interface.modes[1] shell_mode = repl.interface.modes[2] help_mode = repl.interface.modes[3] - git_mode = repl.interface.modes[4] - histp = repl.interface.modes[5] - prefix_mode = repl.interface.modes[6] + histp = repl.interface.modes[4] + prefix_mode = repl.interface.modes[5] hp = REPL.REPLHistoryProvider(Dict{Symbol,Any}(:julia => repl_mode, :shell => shell_mode, - :help => help_mode, - :git => git_mode)) + :help => help_mode)) REPL.hist_from_file(hp, IOBuffer(fakehistory)) REPL.history_reset_state(hp) - histp.hp = repl_mode.hist = shell_mode.hist = help_mode.hist = git_mode.hist = hp + histp.hp = repl_mode.hist = shell_mode.hist = help_mode.hist = hp # Some manual setup s = LineEdit.init_state(repl.t, repl.interface) @@ -234,9 +230,6 @@ begin # Test that navigating history skips invalid modes # (in both directions) LineEdit.history_prev(s, hp) - @test LineEdit.mode(s) == git_mode - @test buffercontents(LineEdit.buffer(s)) == "init" - LineEdit.history_prev(s, hp) @test LineEdit.mode(s) == repl_mode @test buffercontents(LineEdit.buffer(s)) == "2 + 2" LineEdit.history_prev(s, hp) @@ -252,16 +245,10 @@ begin @test LineEdit.mode(s) == repl_mode @test buffercontents(LineEdit.buffer(s)) == "2 + 2" LineEdit.history_next(s, hp) - @test LineEdit.mode(s) == git_mode - @test buffercontents(LineEdit.buffer(s)) == "init" - LineEdit.history_next(s, hp) # Test that the same holds for prefix search ps = LineEdit.state(s, prefix_mode) LineEdit.history_prev_prefix(ps, hp, "") - @test ps.parent == git_mode - @test LineEdit.input_string(ps) == "init" - LineEdit.history_prev_prefix(ps, hp, "") @test ps.parent == repl_mode @test LineEdit.input_string(ps) == "2 + 2" LineEdit.history_prev_prefix(ps, hp, "") From 116bb4c37f186b43bb964890213821f2d66b30bd Mon Sep 17 00:00:00 2001 From: wildart Date: Mon, 14 Sep 2015 19:47:02 -0400 Subject: [PATCH 0358/1938] fixed bad rebase commit of `inbase` --- base/methodshow.jl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/base/methodshow.jl b/base/methodshow.jl index f3d94184957ca..1d379ce8773a1 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -81,7 +81,14 @@ end show(io::IO, mt::MethodTable) = show_method_table(io, mt) -inbase(m::Module) = m == Base ? true : m == Main ? false : inbase(module_parent(m)) +function inbase(m::Module) + if m == Base + true + else + parent = module_parent(m) + parent === m ? false : inbase(parent) + end +end fileurl(file) = let f = find_source_file(file); f == nothing ? "" : "file://"*f; end function url(m::Method) From b0f07a8214dfde6fc4094c60778946afc3ef2291 Mon Sep 17 00:00:00 2001 From: wildart Date: Wed, 23 Sep 2015 02:44:00 -0400 Subject: [PATCH 0359/1938] added: tree entry type, tree walker, blob lookup, some refinements fixed: revwalker map type instability & count, typos --- base/libgit2.jl | 1 + base/libgit2/blob.jl | 8 ++++++++ base/libgit2/const.jl | 16 +++++++-------- base/libgit2/repository.jl | 3 ++- base/libgit2/tree.jl | 40 ++++++++++++++++++++++++++++++++++++++ base/libgit2/types.jl | 1 + base/libgit2/utils.jl | 6 ++++-- base/libgit2/walker.jl | 8 ++------ base/pkg/entry.jl | 7 ++----- test/libgit2.jl | 2 +- 10 files changed, 69 insertions(+), 23 deletions(-) create mode 100644 base/libgit2/tree.jl diff --git a/base/libgit2.jl b/base/libgit2.jl index c871b350ff17f..7ac4dd1739357 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -28,6 +28,7 @@ include("libgit2/blob.jl") include("libgit2/diff.jl") include("libgit2/rebase.jl") include("libgit2/status.jl") +include("libgit2/tree.jl") include("libgit2/callbacks.jl") include("libgit2/utils.jl") diff --git a/base/libgit2/blob.jl b/base/libgit2/blob.jl index 3c10dfd6fd706..8f8a25bf87c78 100644 --- a/base/libgit2/blob.jl +++ b/base/libgit2/blob.jl @@ -11,3 +11,11 @@ end function Base.length(blob::GitBlob) return ccall((:git_blob_rawsize, :libgit2), Coff_t, (Ptr{Void},), blob.ptr) end + +function lookup(repo::GitRepo, oid::Oid) + blob_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_blob_lookup, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ref{Oid}), + blob_ptr_ptr, repo.ptr, Ref(oid)) + return GitBlob(blob_ptr_ptr[]) +end diff --git a/base/libgit2/const.jl b/base/libgit2/const.jl index 07084bbe31a09..4aeb909108dfb 100644 --- a/base/libgit2/const.jl +++ b/base/libgit2/const.jl @@ -26,14 +26,6 @@ module GitConst const REF_SYMBOLIC = Cint(2) const REF_LISTALL = REF_OID | REF_SYMBOLIC - # file - const FILEMODE_NEW = Cint(00000) - const FILEMODE_TREE = Cint(16384) - const FILEMODE_BLOB = Cint(33188) - const FILEMODE_BLOB_EXECUTABLE = Cint(33261) - const FILEMODE_LINK = Cint(40960) - const FILEMODE_COMMIT = Cint(57344) - # checkout const CHECKOUT_NONE = Cuint(0) const CHECKOUT_SAFE = Cuint(1 << 0) @@ -262,4 +254,12 @@ module GitConst REPOSITORY_OPEN_BARE = 1<<2) # open repository as a bare repo @enum(GIT_BRANCH, BRANCH_LOCAL = 1, BRANCH_REMOTE = 2) + + @enum(GIT_FILEMODE, FILEMODE_UNREADABLE = 0o000000, + FILEMODE_TREE = 0o040000, + FILEMODE_BLOB = 0o100644, + FILEMODE_BLOB_EXECUTABLE = 0o100755, + FILEMODE_LINK = 0o120000, + FILEMODE_COMMIT = 0o160000) + end diff --git a/base/libgit2/repository.jl b/base/libgit2/repository.jl index d11fa270de374..000dfd17134a0 100644 --- a/base/libgit2/repository.jl +++ b/base/libgit2/repository.jl @@ -139,6 +139,8 @@ function peel(obj::GitObject, obj_type::Cint) return git_otype(peeled_ptr_ptr[]) end +peel{T <: GitObject}(::Type{T}, obj::GitObject) = peel(obj, getobjecttype(T)) + function checkout_tree(repo::GitRepo, obj::GitObject; options::CheckoutOptions = CheckoutOptions()) @check ccall((:git_checkout_tree, :libgit2), Cint, @@ -203,4 +205,3 @@ function remotes(repo::GitRepo) (Ptr{Void}, Ptr{Void}), out, repo.ptr) return convert(Vector{AbstractString}, out[]) end - diff --git a/base/libgit2/tree.jl b/base/libgit2/tree.jl new file mode 100644 index 0000000000000..4d77ddce4d7fb --- /dev/null +++ b/base/libgit2/tree.jl @@ -0,0 +1,40 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + +""" +Traverse the entries in a tree and its subtrees in post or pre order. + +Function parameter should have following signature: + + (Cstring, Ptr{Void}, Ptr{Void}) -> Cint +""" +function treewalk(f::Function, tree::GitTree, payload=Any[], post::Bool = false) + cbf = cfunction(f, Cint, (Cstring, Ptr{Void}, Ptr{Void})) + cbf_payload = Ref{typeof(payload)}(payload) + @check ccall((:git_tree_walk, :libgit2), Cint, + (Ptr{Void}, Cint, Ptr{Void}, Ptr{Void}), + tree.ptr, post, cbf, cbf_payload) + return cbf_payload +end + +function filename(te::GitTreeEntry) + str = ccall((:git_tree_entry_name, :libgit2), Cstring, (Ptr{Void},), te.ptr) + str != C_NULL && return bytestring(str) + return nothing +end + +function filemode(te::GitTreeEntry) + return ccall((:git_tree_entry_filemode, :libgit2), Cint, (Ptr{Void},), te.ptr) +end + +function filemode(te::GitTreeEntry) + return ccall((:git_tree_entry_filemode, :libgit2), Cint, (Ptr{Void},), te.ptr) +end + + +function object(repo::GitRepo, te::GitTreeEntry) + obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + @check ccall((:git_tree_entry_to_object, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Ref{Void}), + obj_ptr_ptr, repo.ptr, te.ptr) + return GitAnyObject(obj_ptr_ptr[]) +end diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index 4fad99d2cfdf9..8de0bfc368fc5 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -421,6 +421,7 @@ for (typ, ref, sup, fnc) in ( (:GitRebase, :Void, :AbstractGitObject, :(:git_rebase_free)), (:GitStatus, :Void, :AbstractGitObject, :(:git_status_list_free)), (:GitBranchIter, :Void, :AbstractGitObject, :(:git_branch_iterator_free)), + (:GitTreeEntry, :Void, :AbstractGitObject, :(:git_tree_entry_free)), (:GitSignature, :SignatureStruct, :AbstractGitObject, :(:git_signature_free)), (:GitAnyObject, :Void, :GitObject, nothing), (:GitCommit, :Void, :GitObject, nothing), diff --git a/base/libgit2/utils.jl b/base/libgit2/utils.jl index 4cb03ceea063d..bd6cfc72806b3 100644 --- a/base/libgit2/utils.jl +++ b/base/libgit2/utils.jl @@ -1,8 +1,10 @@ function version() - major, minor, patch = Cint[0], Cint[0], Cint[0] + major = Ref{Cint}(0) + minor = Ref{Cint}(0) + patch = Ref{Cint}(0) ccall((:git_libgit2_version, :libgit2), Void, (Ptr{Cint}, Ptr{Cint}, Ptr{Cint}), major, minor, patch) - return VersionNumber(major[1], minor[1], patch[1]) + return VersionNumber(major[], minor[], patch[]) end isset(val::Integer, flag::Integer) = (val & flag == flag) \ No newline at end of file diff --git a/base/libgit2/walker.jl b/base/libgit2/walker.jl index dfde04fee82a1..82918877424e3 100644 --- a/base/libgit2/walker.jl +++ b/base/libgit2/walker.jl @@ -58,7 +58,7 @@ function Base.map(f::Function, walker::GitRevWalker; by::Cint = GitConst.SORT_NONE, rev::Bool=false, count::Int=0) - res = nothing + res = [] sort!(walker, by=by, rev=rev) if !iszero(oid) push!(walker, oid) @@ -73,9 +73,6 @@ function Base.map(f::Function, walker::GitRevWalker; repo = repository(walker) while !done(walker, s) val = f(s[1], repo) - if res == nothing - res = Array(typeof(val),0) - end push!(res, val) val, s = next(walker, s) c +=1 @@ -97,12 +94,11 @@ function Base.count(f::Function, walker::GitRevWalker; end s = start(walker) - val = true repo = repository(walker) while !done(walker, s) && val val = f(s[1], repo) _, s = next(walker, s) - c += 1 + c += (val == true) end return c end diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index b094ef761a8ff..0aa66af314120 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -55,8 +55,7 @@ function add(pkg::AbstractString, vers::VersionSet) end branch = Dir.getmetabranch() outdated = with(GitRepo, "METADATA") do repo - current_barnch = LibGit2.branch(repo) - if current_barnch == branch + if LibGit2.branch(repo) == branch if LibGit2.isdiff(repo, "origin/$branch") outdated = :yes else @@ -187,9 +186,7 @@ function status(io::IO, pkg::AbstractString, ver::VersionNumber, fix::Bool) end function status(io::IO, pkg::AbstractString, msg::AbstractString) - @printf io " - %-29s " pkg - @printf io "%-19s" msg - println(io) + @printf io " - %-29s %-19s\n" pkg msg end function clone(url::AbstractString, pkg::AbstractString) diff --git a/test/libgit2.jl b/test/libgit2.jl index 69e0cdf9490af..782cea47e3732 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -2,7 +2,7 @@ # check that libgit2 has been installed correctly -const LIBGIT2_VER = v"0.23.0" +const LIBGIT2_VER = v"0.23.1" function check_version() v = LibGit2.version() From 80f0db7f1ed0295c0de0fc26cba5b1871e88216d Mon Sep 17 00:00:00 2001 From: wildart Date: Wed, 23 Sep 2015 03:45:54 -0400 Subject: [PATCH 0360/1938] Union() => Union{} --- base/pkg/entry.jl | 2 +- base/pkg/generate.jl | 4 ++-- base/pkg/reqs.jl | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 0aa66af314120..4c52452a10c31 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -635,7 +635,7 @@ end nextbump(v::VersionNumber) = isrewritable(v) ? v : nextpatch(v) -function tag(pkg::AbstractString, ver::Union(Symbol,VersionNumber), force::Bool=false, commitish::AbstractString="HEAD") +function tag(pkg::AbstractString, ver::Union{Symbol,VersionNumber}, force::Bool=false, commitish::AbstractString="HEAD") ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) with(GitRepo,"METADATA") do repo LibGit2.isdirty(repo, pkg) && throw(PkgError("METADATA/$pkg is dirty – commit or stash changes to tag")) diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index 0634c01fc3fa4..e21a0adf476a9 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -135,8 +135,8 @@ end function license(pkg::AbstractString, license::AbstractString, - years::Union(Int,AbstractString), - authors::Union(AbstractString,Array); + years::Union{Int,AbstractString}, + authors::Union{AbstractString,Array}; force::Bool=false) file = genfile(pkg,"LICENSE.md",force) do io if !haskey(LICENSES,license) diff --git a/base/pkg/reqs.jl b/base/pkg/reqs.jl index 6b86e26a77bcd..80f67a268031f 100644 --- a/base/pkg/reqs.jl +++ b/base/pkg/reqs.jl @@ -63,7 +63,7 @@ function read{T<:AbstractString}(readable::Vector{T}) return lines end -function read(readable::Union(IO,Base.AbstractCmd)) +function read(readable::Union{IO,Base.AbstractCmd}) lines = Line[] for line in eachline(readable) line = chomp(line) From 0663672bf447f7df684014e2dc41e2298d555565 Mon Sep 17 00:00:00 2001 From: wildart Date: Fri, 25 Sep 2015 00:39:30 -0400 Subject: [PATCH 0361/1938] fixed comparisons to `nothing` --- base/libgit2.jl | 14 +++++++------- base/libgit2/merge.jl | 8 ++++---- base/libgit2/oid.jl | 2 +- base/libgit2/reference.jl | 2 +- base/libgit2/repository.jl | 2 +- base/libgit2/types.jl | 2 +- base/methodshow.jl | 2 +- base/pkg/entry.jl | 12 ++++++------ base/pkg/github.jl | 2 +- base/pkg/read.jl | 2 +- 10 files changed, 24 insertions(+), 24 deletions(-) diff --git a/base/libgit2.jl b/base/libgit2.jl index 7ac4dd1739357..a27b807a6b9c7 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -102,7 +102,7 @@ function diff_files(repo::GitRepo, branch1::AbstractString, branch2::AbstractStr diff = diff_tree(repo, tree1, tree2) for i in 1:count(diff) delta = diff[i] - delta == nothing && break + delta === nothing && break if delta.status in filter push!(files, bytestring(delta.new_file.path)) end @@ -125,7 +125,7 @@ function set_remote_url(repo::GitRepo, url::AbstractString; remote::AbstractStri set!(cfg, "remote.$remote.url", url) m = match(GITHUB_REGEX,url) - if m != nothing + if m !== nothing push = "git@github.com:$(m.captures[1]).git" if push != url set!(cfg, "remote.$remote.pushurl", push) @@ -199,7 +199,7 @@ function branch!(repo::GitRepo, branch_name::AbstractString, set_head::Bool=true) # set as head reference on exit # try to lookup branch first branch_ref = force ? nothing : lookup_branch(repo, branch_name) - if branch_ref == nothing + if branch_ref === nothing # if commit is empty get head commit oid commit_id = if isempty(commit) with(head(repo)) do head_ref @@ -265,10 +265,10 @@ function checkout!(repo::GitRepo, commit::AbstractString = ""; # search for commit to get a commit object obj = get(GitAnyObject, repo, Oid(commit)) - obj == nothing && return + obj === nothing && return try peeled = peel(obj, GitConst.OBJ_COMMIT) - peeled == nothing && return + peeled === nothing && return opts = force ? CheckoutOptions(checkout_strategy = GitConst.CHECKOUT_FORCE) : CheckoutOptions() try @@ -317,7 +317,7 @@ end """ git reset [--soft | --mixed | --hard] """ function reset!(repo::GitRepo, commit::Oid, mode::Cint = GitConst.RESET_MIXED) obj = get(GitAnyObject, repo, commit) - obj == nothing && return + obj === nothing && return try reset!(repo, obj, mode) finally @@ -408,7 +408,7 @@ function rebase!(repo::GitRepo, upstream::AbstractString="", newbase::AbstractSt try rbs = GitRebase(repo, head_ann, upst_ann) try - while (rbs_op = next(rbs)) != nothing + while (rbs_op = next(rbs)) !== nothing commit(rbs, sig) end finish(rbs, sig) diff --git a/base/libgit2/merge.jl b/base/libgit2/merge.jl index fa0eaca861a0b..a8b6cc00d5e56 100644 --- a/base/libgit2/merge.jl +++ b/base/libgit2/merge.jl @@ -28,7 +28,7 @@ function GitAnnotated(repo::GitRepo, comittish::AbstractString) obj = revparse(repo, comittish) try cmt = peel(obj, GitConst.OBJ_COMMIT) - cmt == nothing && return nothing + cmt === nothing && return nothing return GitAnnotated(repo, Oid(cmt)) finally finalize(obj) @@ -54,7 +54,7 @@ end function ffmerge!(repo::GitRepo, ann::GitAnnotated) ann_cmt_oid = commit(ann) cmt = get(GitCommit, repo, ann_cmt_oid) - cmt == nothing && return false # could not find commit tree + cmt === nothing && return false # could not find commit tree try checkout_tree(repo, cmt) with(head(repo)) do head_ref @@ -104,7 +104,7 @@ function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, fastforward::Bool, op elseif isset(mp, Cint(GitConst.MERGE_PREFERENCE_FASTFORWARD_ONLY)) GitConst.MERGE_PREFERENCE_FASTFORWARD_ONLY end - if ffPref == nothing + if ffPref === nothing warn("Unknown merge preference: $(mp).") return false end @@ -138,7 +138,7 @@ function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, fastforward::Bool, op end end - if mergeResult == nothing + if mergeResult === nothing warn("Unknown merge analysis result: $(ma). Merging is not possible.") return false end diff --git a/base/libgit2/oid.jl b/base/libgit2/oid.jl index 59b91826c0fa6..171c29efeefc5 100644 --- a/base/libgit2/oid.jl +++ b/base/libgit2/oid.jl @@ -59,7 +59,7 @@ function Oid(obj::Ptr{Void}) end function Oid{T<:GitObject}(obj::T) - obj == nothing && return Oid() + obj === nothing && return Oid() return Oid(obj.ptr) end diff --git a/base/libgit2/reference.jl b/base/libgit2/reference.jl index 431ea4c146963..4ffba828a9516 100644 --- a/base/libgit2/reference.jl +++ b/base/libgit2/reference.jl @@ -182,7 +182,7 @@ function Base.map(f::Function, bi::GitBranchIter) s = start(bi) while !done(bi, s) val = f(s[1:2]) - if res == nothing + if res === nothing res = Array(typeof(val),0) end push!(res, val) diff --git a/base/libgit2/repository.jl b/base/libgit2/repository.jl index 000dfd17134a0..6d25aea5981bc 100644 --- a/base/libgit2/repository.jl +++ b/base/libgit2/repository.jl @@ -78,7 +78,7 @@ end """ Returns id of a found object """ function revparseid(repo::GitRepo, objname::AbstractString) obj = revparse(repo, objname) - obj == nothing && return Oid() + obj === nothing && return Oid() oid = Oid(obj.ptr) finalize(obj) return oid diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index 8de0bfc368fc5..8eb0413a9d682 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -439,7 +439,7 @@ for (typ, ref, sup, fnc) in ( end end - if fnc != nothing + if fnc !== nothing @eval function Base.finalize(obj::$typ) if obj.ptr != C_NULL ccall(($fnc, :libgit2), Void, (Ptr{$ref},), obj.ptr) diff --git a/base/methodshow.jl b/base/methodshow.jl index 1d379ce8773a1..5f3783332d92c 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -89,7 +89,7 @@ function inbase(m::Module) parent === m ? false : inbase(parent) end end -fileurl(file) = let f = find_source_file(file); f == nothing ? "" : "file://"*f; end +fileurl(file) = let f = find_source_file(file); f === nothing ? "" : "file://"*f; end function url(m::Method) M = m.func.code.module diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 4c52452a10c31..53ecfe59fd0bb 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -35,7 +35,7 @@ end function edit() editor = get(ENV,"VISUAL",get(ENV,"EDITOR",nothing)) - editor != nothing || + editor !== nothing || throw(PkgError("set the EDITOR environment variable to an edit command")) editor = Base.shell_split(editor) reqs = Reqs.parse("REQUIRE") @@ -216,7 +216,7 @@ function clone(url_or_pkg::AbstractString) else url = url_or_pkg m = match(r"(?:^|[/\\])(\w+?)(?:\.jl)?(?:\.git)?$", url) - m != nothing || throw(PkgError("can't determine package name from URL: $url")) + m !== nothing || throw(PkgError("can't determine package name from URL: $url")) pkg = m.captures[1] end clone(url,pkg) @@ -383,7 +383,7 @@ function pull_request(dir::AbstractString, commit::AbstractString="", url::Abstr end m = match(LibGit2.GITHUB_REGEX, url) - m == nothing && throw(PkgError("not a GitHub repo URL, can't make a pull request: $url")) + m === nothing && throw(PkgError("not a GitHub repo URL, can't make a pull request: $url")) owner, owner_repo = m.captures[2:3] user = GitHub.user() info("Forking $owner/$owner_repo to $user") @@ -419,11 +419,11 @@ function publish(branch::AbstractString) # get changed files for path in LibGit2.diff_files(repo, "origin/$branch", LibGit2.GitConst.HEAD_FILE) m = match(r"^(.+?)/versions/([^/]+)/sha1$", path) - m != nothing && ismatch(Base.VERSION_REGEX, m.captures[2]) || continue + m !== nothing && ismatch(Base.VERSION_REGEX, m.captures[2]) || continue pkg, ver = m.captures; ver = convert(VersionNumber,ver) sha1 = readchomp(joinpath("METADATA",path)) old = LibGit2.cat(repo, LibGit2.GitBlob, "origin/$branch:$path") - old != nothing && old != sha1 && throw(PkgError("$pkg v$ver SHA1 changed in METADATA – refusing to publish")) + old !== nothing && old != sha1 && throw(PkgError("$pkg v$ver SHA1 changed in METADATA – refusing to publish")) with(GitRepo, pkg) do pkg_repo tag_name = "v$ver" tag_commit = LibGit2.revparseid(pkg_repo, "$(tag_name)^{commit}") @@ -551,7 +551,7 @@ function write_tag_metadata(repo::GitRepo, pkg::AbstractString, ver::VersionNumb content = with(GitRepo,pkg) do pkg_repo LibGit2.cat(pkg_repo, LibGit2.GitBlob, "$commit:REQUIRE") end - reqs = content != nothing ? Reqs.read(split(content, '\n', keep=false)) : Reqs.Line[] + reqs = content !== nothing ? Reqs.read(split(content, '\n', keep=false)) : Reqs.Line[] cd("METADATA") do d = joinpath(pkg,"versions",string(ver)) mkpath(d) diff --git a/base/pkg/github.jl b/base/pkg/github.jl index cef5028f2e569..b4e9bd7bfec0b 100644 --- a/base/pkg/github.jl +++ b/base/pkg/github.jl @@ -151,7 +151,7 @@ end function normalize_url(url::AbstractString) m = match(LibGit2.GITHUB_REGEX,url) - m == nothing ? url : "https://github.com/$(m.captures[1]).git" + m === nothing ? url : "https://github.com/$(m.captures[1]).git" end end # module diff --git a/base/pkg/read.jl b/base/pkg/read.jl index d18eda3d12d1a..19288a7518969 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -215,7 +215,7 @@ end function issue_url(pkg::AbstractString) ispath(pkg,".git") || return "" m = match(LibGit2.GITHUB_REGEX, url(pkg)) - m == nothing && return "" + m === nothing && return "" return "https://github.com/" * m.captures[1] * "/issues" end From 509e4b21722d08ed438414448ed17396a0d2c03c Mon Sep 17 00:00:00 2001 From: wildart Date: Tue, 29 Sep 2015 02:37:37 -0400 Subject: [PATCH 0362/1938] renamed module `GitConst` to `Consts`, added support of ssh-agent --- base/libgit2.jl | 36 ++++++++++---------- base/libgit2/callbacks.jl | 30 +++++++++-------- base/libgit2/commit.jl | 2 +- base/libgit2/{const.jl => consts.jl} | 15 +++++---- base/libgit2/index.jl | 4 +-- base/libgit2/merge.jl | 36 ++++++++++---------- base/libgit2/oid.jl | 2 +- base/libgit2/reference.jl | 8 ++--- base/libgit2/repository.jl | 2 +- base/libgit2/types.jl | 50 ++++++++++++++-------------- base/libgit2/walker.jl | 8 ++--- base/pkg/entry.jl | 4 +-- base/pkg/generate.jl | 2 +- test/libgit2.jl | 8 ++--- 14 files changed, 105 insertions(+), 102 deletions(-) rename base/libgit2/{const.jl => consts.jl} (96%) diff --git a/base/libgit2.jl b/base/libgit2.jl index a27b807a6b9c7..be41143486607 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -9,7 +9,7 @@ export with, GitRepo, GitConfig const GITHUB_REGEX = r"^(?:git@|git://|https://(?:[\w\.\+\-]+@)?)github.com[:/](([^/].+)/(.+?))(?:\.git)?$"i -include("libgit2/const.jl") +include("libgit2/consts.jl") include("libgit2/types.jl") include("libgit2/error.jl") include("libgit2/signature.jl") @@ -70,7 +70,7 @@ function iscommit(id::AbstractString, repo::GitRepo) end """ git diff-index HEAD [-- ]""" -isdirty(repo::GitRepo, paths::AbstractString=""; cached::Bool=false) = isdiff(repo, GitConst.HEAD_FILE, paths, cached=cached) +isdirty(repo::GitRepo, paths::AbstractString=""; cached::Bool=false) = isdiff(repo, Consts.HEAD_FILE, paths, cached=cached) """ git diff-index [-- ]""" function isdiff(repo::GitRepo, treeish::AbstractString, paths::AbstractString=""; cached::Bool=false) @@ -92,7 +92,7 @@ end """ git diff --name-only --diff-filter= """ function diff_files(repo::GitRepo, branch1::AbstractString, branch2::AbstractString; - filter::Set{Cint}=Set([GitConst.DELTA_ADDED, GitConst.DELTA_MODIFIED, GitConst.DELTA_DELETED])) + filter::Set{Cint}=Set([Consts.DELTA_ADDED, Consts.DELTA_MODIFIED, Consts.DELTA_DELETED])) b1_id = revparseid(repo, branch1*"^{tree}") b2_id = revparseid(repo, branch2*"^{tree}") tree1 = get(GitTree, repo, b1_id) @@ -224,7 +224,7 @@ function branch!(repo::GitRepo, branch_name::AbstractString, if !isempty(track) # setup tracking try with(GitConfig, repo) do cfg - set!(cfg, "branch.$branch_name.remote", GitConst.REMOTE_ORIGIN) + set!(cfg, "branch.$branch_name.remote", Consts.REMOTE_ORIGIN) set!(cfg, "branch.$branch_name.merge", name(branch_ref)) end catch @@ -252,12 +252,12 @@ function checkout!(repo::GitRepo, commit::AbstractString = ""; isempty(commit) && return # grab head name - head_name = GitConst.HEAD_FILE + head_name = Consts.HEAD_FILE try with(head(repo)) do head_ref head_name = shortname(head_ref) # if it is HEAD use short OID instead - if head_name == GitConst.HEAD_FILE + if head_name == Consts.HEAD_FILE head_name = string(Oid(head_ref)) end end @@ -267,9 +267,9 @@ function checkout!(repo::GitRepo, commit::AbstractString = ""; obj = get(GitAnyObject, repo, Oid(commit)) obj === nothing && return try - peeled = peel(obj, GitConst.OBJ_COMMIT) + peeled = peel(obj, Consts.OBJ_COMMIT) peeled === nothing && return - opts = force ? CheckoutOptions(checkout_strategy = GitConst.CHECKOUT_FORCE) : + opts = force ? CheckoutOptions(checkout_strategy = Consts.CHECKOUT_FORCE) : CheckoutOptions() try # detach commit @@ -315,7 +315,7 @@ function reset!(repo::GitRepo, committish::AbstractString, pathspecs::AbstractSt end """ git reset [--soft | --mixed | --hard] """ -function reset!(repo::GitRepo, commit::Oid, mode::Cint = GitConst.RESET_MIXED) +function reset!(repo::GitRepo, commit::Oid, mode::Cint = Consts.RESET_MIXED) obj = get(GitAnyObject, repo, commit) obj === nothing && return try @@ -344,10 +344,10 @@ function revcount(repo::GitRepo, fst::AbstractString, snd::AbstractString) snd_id = revparseid(repo, snd) base_id = merge_base(repo, string(fst_id), string(snd_id)) fc = with(GitRevWalker(repo)) do walker - count((i,r)->i!=base_id, walker, oid=fst_id, by=GitConst.SORT_TOPOLOGICAL) + count((i,r)->i!=base_id, walker, oid=fst_id, by=Consts.SORT_TOPOLOGICAL) end sc = with(GitRevWalker(repo)) do walker - count((i,r)->i!=base_id, walker, oid=snd_id, by=GitConst.SORT_TOPOLOGICAL) + count((i,r)->i!=base_id, walker, oid=snd_id, by=Consts.SORT_TOPOLOGICAL) end return (fc-1, sc-1) end @@ -361,7 +361,7 @@ function merge!(repo::GitRepo; # merge into head branch with(head(repo)) do head_ref upst_anns = if !isempty(committish) # merge committish into HEAD - if committish == GitConst.FETCH_HEAD # merge FETCH_HEAD + if committish == Consts.FETCH_HEAD # merge FETCH_HEAD fheads = fetchheads(repo) filter!(fh->fh.ismerge, fheads) length(fheads) == 0 && throw(Error.GitError(Error.Merge, Error.ERROR, "There is no fetch reference for this branch.")) @@ -435,12 +435,12 @@ function authors(repo::GitRepo) map((oid,repo)->with(get(GitCommit, repo, oid)) do cmt author(cmt)::Signature end, - walker) #, by = GitConst.SORT_TIME) + walker) #, by = Consts.SORT_TIME) end end function snapshot(repo::GitRepo) - head = Oid(repo, GitConst.HEAD_FILE) + head = Oid(repo, Consts.HEAD_FILE) index = with(GitIndex, repo) do idx; write_tree!(idx) end work = try with(GitIndex, repo) do idx @@ -465,17 +465,17 @@ function snapshot(repo::GitRepo) end function restore(s::State, repo::GitRepo) - reset!(repo, GitConst.HEAD_FILE, "*") # unstage everything + reset!(repo, Consts.HEAD_FILE, "*") # unstage everything with(GitIndex, repo) do idx read_tree!(idx, s.work) # move work tree to index opts = CheckoutOptions( - checkout_strategy = GitConst.CHECKOUT_FORCE | # check the index out to work - GitConst.CHECKOUT_REMOVE_UNTRACKED) # remove everything else + checkout_strategy = Consts.CHECKOUT_FORCE | # check the index out to work + Consts.CHECKOUT_REMOVE_UNTRACKED) # remove everything else checkout_index(repo, Nullable(idx), options = opts) read_tree!(idx, s.index) # restore index end - reset!(repo, s.head, GitConst.RESET_SOFT) # restore head + reset!(repo, s.head, Consts.RESET_SOFT) # restore head end function transact(f::Function, repo::GitRepo) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index d9896ccbf12d9..fe1899f8a0c11 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -32,19 +32,25 @@ end function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, username_ptr::Cstring, allowed_types::Cuint, payload::Ptr{Void}) + err = 1 url = bytestring(url_ptr) - # for HTTPS use keyboard-interactive prompt - if startswith(url, "https") + if isset(allowed_types, Cuint(Consts.CREDTYPE_USERPASS_PLAINTEXT)) + # use keyboard-interactive prompt username = prompt("Username for '$url'") pass = prompt("Password for '$url'", password=true) err = ccall((:git_cred_userpass_plaintext_new, :libgit2), Cint, (Ptr{Ptr{Void}}, Cstring, Cstring), cred, username, pass) - err != 0 && return Cint(err) - else - # for SSH we need key info, look for environment vars GITHUB_* as well + err == 0 && return Cint(0) + elseif isset(allowed_types, Cuint(Consts.CREDTYPE_SSH_KEY)) && err > 0 + # use ssh-agent + err = ccall((:git_cred_ssh_key_from_agent, :libgit2), Cint, + (Ptr{Ptr{Void}}, Cstring), cred, username_ptr) + err == 0 && return Cint(0) + elseif isset(allowed_types, Cuint(Consts.CREDTYPE_SSH_CUSTOM)) && err > 0 + # for SSH we need key info, look for environment vars SSH_* as well # if username is not provided, then prompt for it username = if username_ptr == Cstring_NULL @@ -53,33 +59,29 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, bytestring(username_ptr) end - publickey = if "GITHUB_PUB_KEY" in keys(ENV) + publickey = if "SSH_PUB_KEY" in keys(ENV) ENV["GITHUB_PUB_KEY"] else keydef = homedir()*"/.ssh/id_rsa.pub" prompt("Public key location", default=keydef) end - privatekey = if "GITHUB_PRV_KEY" in keys(ENV) + privatekey = if "SSH_PRV_KEY" in keys(ENV) ENV["GITHUB_PRV_KEY"] else keydef = homedir()*"/.ssh/id_rsa" prompt("Private key location", default=keydef) end - passphrase= if "GITHUB_PRV_KEY_PASS" in keys(ENV) - ENV["GITHUB_PRV_KEY_PASS"] - else - prompt("Private key passphrase", password=true) - end + passphrase= get(ENV,"SSH_PRV_KEY_PASS","0") == "0" ? "" : prompt("Private key passphrase", password=true) err = ccall((:git_cred_ssh_key_new, :libgit2), Cint, (Ptr{Ptr{Void}}, Cstring, Cstring, Cstring, Cstring), cred, username, publickey, privatekey, passphrase) - err != 0 && return Cint(err) + err == 0 && return Cint(0) end - return Cint(0) + return Cint(err) end function fetchhead_foreach_callback(ref_name::Cstring, remote_url::Cstring, diff --git a/base/libgit2/commit.jl b/base/libgit2/commit.jl index 3c3c66072fd4d..7d84fae580a11 100644 --- a/base/libgit2/commit.jl +++ b/base/libgit2/commit.jl @@ -47,7 +47,7 @@ end """Commit changes to repository""" function commit(repo::GitRepo, msg::AbstractString; - refname::AbstractString=GitConst.HEAD_FILE, + refname::AbstractString=Consts.HEAD_FILE, author::Signature = Signature(repo), committer::Signature = Signature(repo), tree_id::Oid = Oid(), diff --git a/base/libgit2/const.jl b/base/libgit2/consts.jl similarity index 96% rename from base/libgit2/const.jl rename to base/libgit2/consts.jl index 4aeb909108dfb..eac2933d02dac 100644 --- a/base/libgit2/const.jl +++ b/base/libgit2/consts.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -module GitConst +module Consts const HEAD_FILE = "HEAD" const FETCH_HEAD = "FETCH_HEAD" @@ -180,12 +180,6 @@ module GitConst const REBASE_OPERATION_FIXUP = Cint(4) const REBASE_OPERATION_EXEC = Cint(5) - # credentials - const CREDTYPE_USERPASS_PLAINTEXT = Cuint(1 << 0) - const CREDTYPE_SSH_KEY = Cuint(1 << 1) - const CREDTYPE_SSH_CUSTOM = Cuint(1 << 2) - const CREDTYPE_DEFAULT = Cuint(1 << 3) - # fetch_prune const FETCH_PRUNE_UNSPECIFIED = Cint(0) const FETCH_PRUNE = Cint(1) @@ -262,4 +256,11 @@ module GitConst FILEMODE_LINK = 0o120000, FILEMODE_COMMIT = 0o160000) + @enum(GIT_CREDTYPE, CREDTYPE_USERPASS_PLAINTEXT = Cuint(1 << 0), + CREDTYPE_SSH_KEY = Cuint(1 << 1), + CREDTYPE_SSH_CUSTOM = Cuint(1 << 2), + CREDTYPE_DEFAULT = Cuint(1 << 3), + CREDTYPE_SSH_INTERACTIVE = Cuint(1 << 4), + CREDTYPE_USERNAME = Cuint(1 << 5), + CREDTYPE_SSH_MEMORY = Cuint(1 << 6)) end diff --git a/base/libgit2/index.jl b/base/libgit2/index.jl index 3bd1f21c86325..c38d3257e5834 100644 --- a/base/libgit2/index.jl +++ b/base/libgit2/index.jl @@ -42,7 +42,7 @@ function read_tree!(idx::GitIndex, tree_id::Oid) end function add!{T<:AbstractString}(idx::GitIndex, files::T...; - flags::Cuint = GitConst.INDEX_ADD_DEFAULT) + flags::Cuint = Consts.INDEX_ADD_DEFAULT) sa = StrArrayStruct(files...) try @check ccall((:git_index_add_all, :libgit2), Cint, @@ -76,7 +76,7 @@ function remove!{T<:AbstractString}(idx::GitIndex, files::T...) end function add!{T<:AbstractString}(repo::GitRepo, files::T...; - flags::Cuint = GitConst.INDEX_ADD_DEFAULT) + flags::Cuint = Consts.INDEX_ADD_DEFAULT) with(GitIndex, repo) do idx add!(idx, files..., flags = flags) write!(idx) diff --git a/base/libgit2/merge.jl b/base/libgit2/merge.jl index a8b6cc00d5e56..6a3ea80c700de 100644 --- a/base/libgit2/merge.jl +++ b/base/libgit2/merge.jl @@ -27,7 +27,7 @@ end function GitAnnotated(repo::GitRepo, comittish::AbstractString) obj = revparse(repo, comittish) try - cmt = peel(obj, GitConst.OBJ_COMMIT) + cmt = peel(obj, Consts.OBJ_COMMIT) cmt === nothing && return nothing return GitAnnotated(repo, Oid(cmt)) finally @@ -60,7 +60,7 @@ function ffmerge!(repo::GitRepo, ann::GitAnnotated) with(head(repo)) do head_ref cmt_oid = Oid(cmt) msg = "libgit2.merge: fastforward $(string(cmt_oid)) into $(name(head_ref))" - new_head_ref = if reftype(head_ref) == GitConst.REF_OID + new_head_ref = if reftype(head_ref) == Consts.REF_OID target!(head_ref, cmt_oid, msg=msg) else GitReference(repo, cmt_oid, fullname(head_ref), msg=msg) @@ -75,7 +75,7 @@ end """ Merge changes into current head """ function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, merge_opts::MergeOptions, - checkout_opts = CheckoutOptions(checkout_strategy = GitConst.CHECKOUT_SAFE)) + checkout_opts = CheckoutOptions(checkout_strategy = Consts.CHECKOUT_SAFE)) anns_size = Csize_t(length(anns)) @check ccall((:git_merge, :libgit2), Cint, (Ptr{Void}, Ptr{Ptr{Void}}, Csize_t, @@ -91,37 +91,37 @@ Returns `true` if merge was successful, otherwise `false` """ function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, fastforward::Bool, options::MergeOptions) ma, mp = merge_analysis(repo, anns) - if isset(ma, Cint(GitConst.MERGE_ANALYSIS_UP_TO_DATE)) + if isset(ma, Cint(Consts.MERGE_ANALYSIS_UP_TO_DATE)) return true # no merge - everything is up to date end ffPref = if fastforward - GitConst.MERGE_PREFERENCE_FASTFORWARD_ONLY - elseif isset(mp, Cint(GitConst.MERGE_PREFERENCE_NONE)) - GitConst.MERGE_PREFERENCE_NONE - elseif isset(mp, Cint(GitConst.MERGE_PREFERENCE_NO_FASTFORWARD)) - GitConst.MERGE_PREFERENCE_NO_FASTFORWARD - elseif isset(mp, Cint(GitConst.MERGE_PREFERENCE_FASTFORWARD_ONLY)) - GitConst.MERGE_PREFERENCE_FASTFORWARD_ONLY + Consts.MERGE_PREFERENCE_FASTFORWARD_ONLY + elseif isset(mp, Cint(Consts.MERGE_PREFERENCE_NONE)) + Consts.MERGE_PREFERENCE_NONE + elseif isset(mp, Cint(Consts.MERGE_PREFERENCE_NO_FASTFORWARD)) + Consts.MERGE_PREFERENCE_NO_FASTFORWARD + elseif isset(mp, Cint(Consts.MERGE_PREFERENCE_FASTFORWARD_ONLY)) + Consts.MERGE_PREFERENCE_FASTFORWARD_ONLY end if ffPref === nothing warn("Unknown merge preference: $(mp).") return false end - mergeResult = if ffPref == GitConst.MERGE_PREFERENCE_NONE - if isset(ma, Cint(GitConst.MERGE_ANALYSIS_FASTFORWARD)) + mergeResult = if ffPref == Consts.MERGE_PREFERENCE_NONE + if isset(ma, Cint(Consts.MERGE_ANALYSIS_FASTFORWARD)) if length(anns) > 1 warn("Unable to perform Fast-Forward merge with mith multiple merge heads.") false else ffmerge!(repo, anns[1]) end - elseif isset(ma, Cint(GitConst.MERGE_ANALYSIS_NORMAL)) + elseif isset(ma, Cint(Consts.MERGE_ANALYSIS_NORMAL)) merge!(repo, anns, options) end - elseif ffPref == GitConst.MERGE_PREFERENCE_FASTFORWARD_ONLY - if isset(ma, Cint(GitConst.MERGE_ANALYSIS_FASTFORWARD)) + elseif ffPref == Consts.MERGE_PREFERENCE_FASTFORWARD_ONLY + if isset(ma, Cint(Consts.MERGE_ANALYSIS_FASTFORWARD)) if length(anns) > 1 warn("Unable to perform Fast-Forward merge with mith multiple merge heads.") false @@ -132,8 +132,8 @@ function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, fastforward::Bool, op warn("Cannot perform fast-forward merge.") false end - elseif ffPref == GitConst.MERGE_PREFERENCE_NO_FASTFORWARD - if isset(ma, Cint(GitConst.MERGE_ANALYSIS_NORMAL)) + elseif ffPref == Consts.MERGE_PREFERENCE_NO_FASTFORWARD + if isset(ma, Cint(Consts.MERGE_ANALYSIS_NORMAL)) merge!(repo, anns, options) end end diff --git a/base/libgit2/oid.jl b/base/libgit2/oid.jl index 171c29efeefc5..1af95d2edb296 100644 --- a/base/libgit2/oid.jl +++ b/base/libgit2/oid.jl @@ -37,7 +37,7 @@ end function Oid(ref::GitReference) isempty(ref) && return Oid() - reftype(ref) != GitConst.REF_OID && return Oid() + reftype(ref) != Consts.REF_OID && return Oid() oid_ptr = ccall((:git_reference_target, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) oid_ptr == C_NULL && return Oid() return Oid(oid_ptr) diff --git a/base/libgit2/reference.jl b/base/libgit2/reference.jl index 4ffba828a9516..0298a334c0548 100644 --- a/base/libgit2/reference.jl +++ b/base/libgit2/reference.jl @@ -8,7 +8,7 @@ function GitReference(repo::GitRepo, refname::AbstractString) return GitReference(ref_ptr_ptr[]) end -function GitReference(repo::GitRepo, obj_oid::Oid, refname::AbstractString = GitConst.HEAD_FILE; +function GitReference(repo::GitRepo, obj_oid::Oid, refname::AbstractString = Consts.HEAD_FILE; force::Bool=false, msg::AbstractString="") ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_reference_create, :libgit2), Cint, @@ -38,7 +38,7 @@ end function fullname(ref::GitReference) isempty(ref) && return "" - reftype(ref) == GitConst.REF_OID && return "" + reftype(ref) == Consts.REF_OID && return "" rname = ccall((:git_reference_symbolic_target, :libgit2), Ptr{UInt8}, (Ptr{Void},), ref.ptr) rname == C_NULL && return "" return bytestring(rname) @@ -111,7 +111,7 @@ end function lookup_branch(repo::GitRepo, branch_name::AbstractString, remote::Bool=false) ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - branch_type = remote ? GitConst.BRANCH_REMOTE : GitConst.BRANCH_LOCAL + branch_type = remote ? Consts.BRANCH_REMOTE : Consts.BRANCH_LOCAL err = ccall((:git_branch_lookup, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Cint), ref_ptr_ptr, repo.ptr, branch_name, branch_type) @@ -148,7 +148,7 @@ function target!(ref::GitReference, new_oid::Oid; msg::AbstractString="") return GitReference(ref_ptr_ptr[]) end -function GitBranchIter(r::GitRepo, flags::Cint=Cint(GitConst.BRANCH_LOCAL)) +function GitBranchIter(r::GitRepo, flags::Cint=Cint(Consts.BRANCH_LOCAL)) bi_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_branch_iterator_new, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Cint), bi_ptr, r.ptr, flags) diff --git a/base/libgit2/repository.jl b/base/libgit2/repository.jl index 6d25aea5981bc..51929b1e423ac 100644 --- a/base/libgit2/repository.jl +++ b/base/libgit2/repository.jl @@ -13,7 +13,7 @@ function GitRepo(path::AbstractString) return GitRepo(repo_ptr_ptr[]) end -function GitRepoExt(path::AbstractString, flags::Cuint = Cuint(GitConst.REPOSITORY_OPEN_DEFAULT)) +function GitRepoExt(path::AbstractString, flags::Cuint = Cuint(Consts.REPOSITORY_OPEN_DEFAULT)) separator = @unix? ":" : ";" repo_ptr_ptr = Ref{Ptr{Void}}(C_NULL) err = ccall((:git_repository_open_ext, :libgit2), Cint, diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index 8eb0413a9d682..027fb16ba0219 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -80,12 +80,12 @@ immutable CheckoutOptions perfdata_cb::Ptr{Void} perfdata_payload::Ptr{Void} end -CheckoutOptions(; checkout_strategy::Cuint = GitConst.CHECKOUT_SAFE, +CheckoutOptions(; checkout_strategy::Cuint = Consts.CHECKOUT_SAFE, disable_filters::Cint = zero(Cint), dir_mode::Cuint = Cuint(0), # Cuint(0o755), file_mode::Cuint = Cuint(0), #Cuint(0o644), file_open_flags::Cint = zero(Cint), - notify_flags::Cuint = GitConst.CHECKOUT_NOTIFY_NONE, + notify_flags::Cuint = Consts.CHECKOUT_NOTIFY_NONE, notify_cb::Ptr{Void} = Ptr{Void}(0), notify_payload::Ptr{Void} = Ptr{Void}(0), progress_cb::Ptr{Void} = Ptr{Void}(0), @@ -169,9 +169,9 @@ immutable FetchOptions download_tags::Cint end FetchOptions(; callbacks::RemoteCallbacks = RemoteCallbacks(), - prune::Cint = GitConst.FETCH_PRUNE_UNSPECIFIED, + prune::Cint = Consts.FETCH_PRUNE_UNSPECIFIED, update_fetchhead::Cint = one(Cint), - download_tags::Cint = GitConst.REMOTE_DOWNLOAD_TAGS_AUTO + download_tags::Cint = Consts.REMOTE_DOWNLOAD_TAGS_AUTO ) = FetchOptions(one(Cuint), callbacks, prune, @@ -193,7 +193,7 @@ end CloneOptions(; checkout_opts::CheckoutOptions = CheckoutOptions(), fetch_opts::FetchOptions = FetchOptions(), bare::Cint = zero(Cint), - localclone::Cint = GitConst.CLONE_LOCAL_AUTO, + localclone::Cint = Consts.CLONE_LOCAL_AUTO, checkout_branch::Cstring = Cstring_NULL, repository_cb::Ptr{Void} = Ptr{Void}(0), repository_cb_payload::Ptr{Void} = Ptr{Void}(0), @@ -229,8 +229,8 @@ immutable DiffOptionsStruct old_prefix::Cstring new_prefix::Cstring end -DiffOptionsStruct(; flags::UInt32 = GitConst.DIFF_NORMAL, - ignore_submodules::Cint = Cint(GitConst.SUBMODULE_IGNORE_UNSPECIFIED), +DiffOptionsStruct(; flags::UInt32 = Consts.DIFF_NORMAL, + ignore_submodules::Cint = Cint(Consts.SUBMODULE_IGNORE_UNSPECIFIED), pathspec::StrArrayStruct = StrArrayStruct(), notify_cb::Ptr{Void} = C_NULL, notify_payload::Ptr{Void} = C_NULL, @@ -240,7 +240,7 @@ DiffOptionsStruct(; flags::UInt32 = GitConst.DIFF_NORMAL, max_size::Coff_t = Coff_t(512*1024*1024), #zero(Coff_t), #512Mb old_prefix::Cstring = Cstring_NULL, new_prefix::Cstring = Cstring_NULL -)=DiffOptionsStruct(GitConst.DIFF_OPTIONS_VERSION, +)=DiffOptionsStruct(Consts.DIFF_OPTIONS_VERSION, flags, ignore_submodules, pathspec, @@ -286,8 +286,8 @@ MergeOptions(; tree_flags::Cint = Cint(0), rename_threshold::Cuint = Cuint(50), target_limit::Cuint = Cuint(200), metric::Ptr{Void} = C_NULL, - file_favor::Cint = Cint(GitConst.MERGE_FILE_FAVOR_NORMAL), - file_flags::Cuint =Cuint(GitConst.MERGE_FILE_DEFAULT) + file_favor::Cint = Cint(Consts.MERGE_FILE_FAVOR_NORMAL), + file_flags::Cuint =Cuint(Consts.MERGE_FILE_DEFAULT) )=MergeOptions(one(Cuint), tree_flags, rename_threshold, @@ -371,11 +371,11 @@ immutable StatusOptions flags::Cuint pathspec::StrArrayStruct end -StatusOptions(; show::Cint = GitConst.STATUS_SHOW_INDEX_AND_WORKDIR, - flags::Cuint = GitConst.STATUS_OPT_INCLUDE_UNTRACKED | - GitConst.STATUS_OPT_RECURSE_UNTRACKED_DIRS | - GitConst.STATUS_OPT_RENAMES_HEAD_TO_INDEX | - GitConst.STATUS_OPT_SORT_CASE_SENSITIVELY, +StatusOptions(; show::Cint = Consts.STATUS_SHOW_INDEX_AND_WORKDIR, + flags::Cuint = Consts.STATUS_OPT_INCLUDE_UNTRACKED | + Consts.STATUS_OPT_RECURSE_UNTRACKED_DIRS | + Consts.STATUS_OPT_RENAMES_HEAD_TO_INDEX | + Consts.STATUS_OPT_SORT_CASE_SENSITIVELY, pathspec::StrArrayStruct = StrArrayStruct() )=StatusOptions(one(Cuint), show, @@ -482,30 +482,30 @@ end function getobjecttype{T<:GitObject}(::Type{T}) return if T == GitCommit - GitConst.OBJ_COMMIT + Consts.OBJ_COMMIT elseif T == GitTree - GitConst.OBJ_TREE + Consts.OBJ_TREE elseif T == GitBlob - GitConst.OBJ_BLOB + Consts.OBJ_BLOB elseif T == GitTag - GitConst.OBJ_TAG + Consts.OBJ_TAG elseif T == GitAnyObject - GitConst.OBJ_ANY + Consts.OBJ_ANY else throw(GitError(Error.Object, Error.ENOTFOUND, "Type $T is not supported")) end end function getobjecttype(obj_type::Cint) - return if obj_type == GitConst.OBJ_COMMIT + return if obj_type == Consts.OBJ_COMMIT GitCommit - elseif obj_type == GitConst.OBJ_TREE + elseif obj_type == Consts.OBJ_TREE GitTree - elseif obj_type == GitConst.OBJ_BLOB + elseif obj_type == Consts.OBJ_BLOB GitBlob - elseif obj_type == GitConst.OBJ_TAG + elseif obj_type == Consts.OBJ_TAG GitTag - elseif obj_type == GitConst.OBJ_ANY + elseif obj_type == Consts.OBJ_ANY GitAnyObject else throw(GitError(Error.Object, Error.ENOTFOUND, "Object type $obj_type is not supported")) diff --git a/base/libgit2/walker.jl b/base/libgit2/walker.jl index 82918877424e3..45cd8742841ea 100644 --- a/base/libgit2/walker.jl +++ b/base/libgit2/walker.jl @@ -40,8 +40,8 @@ function Base.push!(w::GitRevWalker, range::AbstractString) return w end -function Base.sort!(w::GitRevWalker; by::Cint = GitConst.SORT_NONE, rev::Bool=false) - rev && (by |= GitConst.SORT_REVERSE) +function Base.sort!(w::GitRevWalker; by::Cint = Consts.SORT_NONE, rev::Bool=false) + rev && (by |= Consts.SORT_REVERSE) ccall((:git_revwalk_sorting, :libgit2), Void, (Ptr{Void}, Cint), w.ptr, by) return w end @@ -55,7 +55,7 @@ end function Base.map(f::Function, walker::GitRevWalker; oid::Oid=Oid(), range::AbstractString="", - by::Cint = GitConst.SORT_NONE, + by::Cint = Consts.SORT_NONE, rev::Bool=false, count::Int=0) res = [] @@ -83,7 +83,7 @@ end function Base.count(f::Function, walker::GitRevWalker; oid::Oid=Oid(), - by::Cint = GitConst.SORT_NONE, + by::Cint = Consts.SORT_NONE, rev::Bool=false) c = 0 sort!(walker, by=by, rev=rev) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 53ecfe59fd0bb..04a5c23b7c5c3 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -228,7 +228,7 @@ function checkout(pkg::AbstractString, branch::AbstractString, do_merge::Bool, d with(GitRepo, pkg) do r LibGit2.transact(r) do repo LibGit2.isdirty(repo) && throw(PkgError("$pkg is dirty, bailing")) - LibGit2.branch!(repo, branch, track=LibGit2.GitConst.REMOTE_ORIGIN) + LibGit2.branch!(repo, branch, track=LibGit2.Consts.REMOTE_ORIGIN) do_merge && LibGit2.merge!(repo, fastforward=true) # merge changes if do_pull info("Pulling $pkg latest $branch...") @@ -417,7 +417,7 @@ function publish(branch::AbstractString) ahead_local == 0 && throw(PkgError("There are no METADATA changes to publish")) # get changed files - for path in LibGit2.diff_files(repo, "origin/$branch", LibGit2.GitConst.HEAD_FILE) + for path in LibGit2.diff_files(repo, "origin/$branch", LibGit2.Consts.HEAD_FILE) m = match(r"^(.+?)/versions/([^/]+)/sha1$", path) m !== nothing && ismatch(Base.VERSION_REGEX, m.captures[2]) || continue pkg, ver = m.captures; ver = convert(VersionNumber,ver) diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index e21a0adf476a9..b3b1aee469580 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -74,7 +74,7 @@ function package( Julia Version $VERSION [$(Base.GIT_VERSION_INFO.commit_short)] """ - LibGit2.add!(repo, files..., flags = LibGit2.GitConst.INDEX_ADD_FORCE) + LibGit2.add!(repo, files..., flags = LibGit2.Consts.INDEX_ADD_FORCE) if isnew info("Committing $pkg generated files") LibGit2.commit(repo, msg) diff --git a/test/libgit2.jl b/test/libgit2.jl index 782cea47e3732..ca19cb1f5ff1f 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -2,7 +2,7 @@ # check that libgit2 has been installed correctly -const LIBGIT2_VER = v"0.23.1" +const LIBGIT2_VER = v"0.23.0" function check_version() v = LibGit2.version() @@ -72,7 +72,7 @@ temp_dir() do dir repo = LibGit2.clone(repo_url, repo_path, isbare = true, remote_cb = LibGit2.mirror_cb()) finalize(repo) @test isdir(repo_path) - @test isfile(joinpath(repo_path, LibGit2.GitConst.HEAD_FILE)) + @test isfile(joinpath(repo_path, LibGit2.Consts.HEAD_FILE)) end # clone @@ -173,7 +173,7 @@ temp_dir() do dir_cache credentials!(cfg) end oids = LibGit2.with(LibGit2.GitRevWalker(repo)) do walker - LibGit2.map((oid,repo)->string(oid), walker, by = LibGit2.GitConst.SORT_TIME) + LibGit2.map((oid,repo)->string(oid), walker, by = LibGit2.Consts.SORT_TIME) end @test length(oids) > 0 finalize(repo) @@ -181,7 +181,7 @@ temp_dir() do dir_cache LibGit2.with(LibGit2.GitRepo, path) do repo oid = LibGit2.Oid(oids[end]) oids = LibGit2.with(LibGit2.GitRevWalker(repo)) do walker - LibGit2.map((oid,repo)->(oid,repo), walker, oid=oid, by=LibGit2.GitConst.SORT_TIME) + LibGit2.map((oid,repo)->(oid,repo), walker, oid=oid, by=LibGit2.Consts.SORT_TIME) end @test length(oids) > 0 end From 11f08b5f61d461673e922272340ba0ea8cf4f1b9 Mon Sep 17 00:00:00 2001 From: wildart Date: Tue, 29 Sep 2015 02:47:31 -0400 Subject: [PATCH 0363/1938] returned `Pkg.Git` module --- base/pkg.jl | 4 +- base/pkg/git.jl | 124 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 base/pkg/git.jl diff --git a/base/pkg.jl b/base/pkg.jl index 3cf3872c66969..65312613b2e6e 100644 --- a/base/pkg.jl +++ b/base/pkg.jl @@ -2,7 +2,7 @@ module Pkg -export Dir, GitHub, Types, Reqs, Cache, Read, Query, Resolve, Write, Generate, Entry +export Dir, GitHub, Types, Reqs, Cache, Read, Query, Resolve, Write, Generate, Entry, Git export dir, init, rm, add, available, installed, status, clone, checkout, update, resolve, register, tag, publish, generate, test, build, free, pin, PkgError @@ -14,7 +14,7 @@ type PkgError <: Exception msg::AbstractString end -for file in split("dir github types reqs cache read query resolve write generate entry") +for file in split("dir github types reqs cache read query resolve write generate entry git") include("pkg/$file.jl") end const cd = Dir.cd diff --git a/base/pkg/git.jl b/base/pkg/git.jl new file mode 100644 index 0000000000000..32e492f074dc1 --- /dev/null +++ b/base/pkg/git.jl @@ -0,0 +1,124 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + +module Git +# +# some utility functions for working with git repos +# +import Base: shell_escape + +function dir(d) + g = joinpath(d,".git") + isdir(g) && return g + normpath(d, Base.readchomp(setenv(`git rev-parse --git-dir`, dir=d))) +end + +function git(d) + isempty(d) && return `git` + work_tree = abspath(d) + git_dir = joinpath(work_tree, dir(work_tree)) + normpath(work_tree, ".") == normpath(git_dir, ".") ? # is it a bare repo? + `git --git-dir=$work_tree` : `git --work-tree=$work_tree --git-dir=$git_dir` +end + +cmd(args::Cmd; dir="") = `$(git(dir)) $args` +run(args::Cmd; dir="", out=STDOUT) = Base.run(pipeline(cmd(args,dir=dir), out)) +readall(args::Cmd; dir="") = Base.readall(cmd(args,dir=dir)) +readchomp(args::Cmd; dir="") = Base.readchomp(cmd(args,dir=dir)) + +function success(args::Cmd; dir="") + g = git(dir) + Base.readchomp(`$g rev-parse --is-bare-repository`) == "false" && + Base.run(`$g update-index -q --really-refresh`) + Base.success(`$g $args`) +end + +function version() + vs = split(readchomp(`version`), ' ')[3] + ns = split(vs, '.') + if length(ns) > 3 + VersionNumber(join(ns[1:3], '.')) + else + VersionNumber(join(ns, '.')) + end +end + +modules(args::Cmd; dir="") = readchomp(`config -f .gitmodules $args`, dir=dir) +different(verA::AbstractString, verB::AbstractString, path::AbstractString; dir="") = + !success(`diff-tree --quiet $verA $verB -- $path`, dir=dir) + +dirty(; dir="") = !success(`diff-index --quiet HEAD`, dir=dir) +staged(; dir="") = !success(`diff-index --quiet --cached HEAD`, dir=dir) +unstaged(; dir="") = !success(`diff-files --quiet`, dir=dir) +dirty(paths; dir="") = !success(`diff-index --quiet HEAD -- $paths`, dir=dir) +staged(paths; dir="") = !success(`diff-index --quiet --cached HEAD -- $paths`, dir=dir) +unstaged(paths; dir="") = !success(`diff-files --quiet -- $paths`, dir=dir) + +iscommit(name; dir="") = success(`cat-file commit $name`, dir=dir) +attached(; dir="") = success(`symbolic-ref -q HEAD`, dir=dir) +branch(; dir="") = readchomp(`rev-parse --symbolic-full-name --abbrev-ref HEAD`, dir=dir) +head(; dir="") = readchomp(`rev-parse HEAD`, dir=dir) + +function iscommit(sha1s::Vector; dir="") + indexin(sha1s,split(readchomp(`log --all --format=%H`, dir=dir),"\n")).!=0 +end + +immutable State + head::ASCIIString + index::ASCIIString + work::ASCIIString +end + +function snapshot(; dir="") + head = readchomp(`rev-parse HEAD`, dir=dir) + index = readchomp(`write-tree`, dir=dir) + work = try + if length(readdir(abspath(dir))) > 1 + run(`add --all`, dir=dir) + run(`add .`, dir=dir) + end + readchomp(`write-tree`, dir=dir) + finally + run(`read-tree $index`, dir=dir) # restore index + end + State(head, index, work) +end + +function restore(s::State; dir="") + run(`reset -q --`, dir=dir) # unstage everything + run(`read-tree $(s.work)`, dir=dir) # move work tree to index + run(`checkout-index -fa`, dir=dir) # check the index out to work + run(`clean -qdf`, dir=dir) # remove everything else + run(`read-tree $(s.index)`, dir=dir) # restore index + run(`reset -q --soft $(s.head)`, dir=dir) # restore head +end + +function transact(f::Function; dir="") + state = snapshot(dir=dir) + try f() catch + restore(state, dir=dir) + rethrow() + end +end + +function is_ancestor_of(a::AbstractString, b::AbstractString; dir="") + A = readchomp(`rev-parse $a`, dir=dir) + readchomp(`merge-base $A $b`, dir=dir) == A +end + +const GITHUB_REGEX = + r"^(?:git@|git://|https://(?:[\w\.\+\-]+@)?)github.com[:/](([^/].+)/(.+?))(?:\.git)?$"i + +function set_remote_url(url::AbstractString; remote::AbstractString="origin", dir="") + run(`config remote.$remote.url $url`, dir=dir) + m = match(GITHUB_REGEX,url) + m === nothing && return + push = "git@github.com:$(m.captures[1]).git" + push != url && run(`config remote.$remote.pushurl $push`, dir=dir) +end + +function normalize_url(url::AbstractString) + m = match(GITHUB_REGEX,url) + m === nothing ? url : "git://github.com/$(m.captures[1]).git" +end + +end # module From d18efae0217bc0609ce2a1cd14fdb431982b7775 Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Tue, 29 Sep 2015 15:17:18 -0400 Subject: [PATCH 0364/1938] fix deprecation warning --- base/libgit2/commit.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/libgit2/commit.jl b/base/libgit2/commit.jl index 7d84fae580a11..dfb6c8b77ad9e 100644 --- a/base/libgit2/commit.jl +++ b/base/libgit2/commit.jl @@ -34,9 +34,9 @@ function commit(repo::GitRepo, nparents = length(parents) parentptrs = Ptr{Void}[c.ptr for c in parents] @check ccall((:git_commit_create, :libgit2), Cint, - (Ptr{Oid}, Ptr{Void}, Ptr{Uint8}, + (Ptr{Oid}, Ptr{Void}, Ptr{UInt8}, Ptr{SignatureStruct}, Ptr{SignatureStruct}, - Ptr{Uint8}, Ptr{Uint8}, Ptr{Void}, + Ptr{UInt8}, Ptr{UInt8}, Ptr{Void}, Csize_t, Ptr{Ptr{Void}}), commit_id_ptr, repo.ptr, isempty(refname) ? C_NULL : refname, author.ptr, committer.ptr, From fe5de8ec5548f119d692e0319b258429fcef42cd Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Tue, 29 Sep 2015 09:59:18 -0400 Subject: [PATCH 0365/1938] docs: various ReStructured text formatting fixes --- base/docs/helpdb.jl | 12 ++++++------ base/linalg/arnoldi.jl | 6 +++--- doc/devdocs/stdio.rst | 8 +++----- doc/manual/arrays.rst | 2 +- .../integers-and-floating-point-numbers.rst | 6 +++--- doc/manual/interfaces.rst | 2 +- doc/manual/linear-algebra.rst | 2 +- doc/manual/parallel-computing.rst | 2 +- doc/stdlib/dates.rst | 6 +++--- doc/stdlib/linalg.rst | 18 +++++++++--------- 10 files changed, 31 insertions(+), 33 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index c58df986616c1..a32159d3dae29 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -4081,7 +4081,7 @@ Compute the LU factorization of ``A``. The return type of ``F`` depends on the t ======================= ========================= ======================================== Type of input ``A`` Type of output ``F`` Relationship between ``F`` and ``A`` ------------------------ ------------------------- ---------------------------------------- +======================= ========================= ======================================== :func:`Matrix` ``LU`` ``F[:L]*F[:U] == A[F[:p], :]`` :func:`Tridiagonal` ``LU{T,Tridiagonal{T}}`` ``F[:L]*F[:U] == A[F[:p], :]`` :func:`SparseMatrixCSC` ``UmfpackLU`` ``F[:L]*F[:U] == (F[:Rs] .* A)[F[:p], F[:q]]`` @@ -4091,7 +4091,7 @@ The individual components of the factorization ``F`` can be accessed by indexing =========== ======================================= ====== ======================== ============= Component Description ``LU`` ``LU{T,Tridiagonal{T}}`` ``UmfpackLU`` ------------ --------------------------------------- ------ ------------------------ ------------- +=========== ======================================= ====== ======================== ============= ``F[:L]`` ``L`` (lower triangular) part of ``LU`` ✓ ✓ ✓ ``F[:U]`` ``U`` (upper triangular) part of ``LU`` ✓ ✓ ✓ ``F[:p]`` (right) permutation ``Vector`` ✓ ✓ ✓ @@ -4103,7 +4103,7 @@ Component Description ``LU`` ``LU{T,Tridiagonal{T} ================== ====== ======================== ============= Supported function ``LU`` ``LU{T,Tridiagonal{T}}`` ``UmfpackLU`` ------------------- ------ ------------------------ ------------- +================== ====== ======================== ============= ``/`` ✓ ``\`` ✓ ✓ ✓ ``cond`` ✓ ✓ @@ -9980,7 +9980,7 @@ Computes the QR factorization of ``A``. The return type of ``F`` depends on the ================ ================= ============== ===================================== Return type ``eltype(A)`` ``pivot`` Relationship between ``F`` and ``A`` ----------------- ----------------- -------------- ------------------------------------- +================ ================= ============== ===================================== ``QR`` not ``BlasFloat`` either ``A==F[:Q]*F[:R]`` ``QRCompactWY`` ``BlasFloat`` ``Val{false}`` ``A==F[:Q]*F[:R]`` ``QRPivoted`` ``BlasFloat`` ``Val{true}`` ``A[:,F[:p]]==F[:Q]*F[:R]`` @@ -9992,7 +9992,7 @@ The individual components of the factorization ``F`` can be accessed by indexing =========== ============================================= ================== ===================== ================== Component Description ``QR`` ``QRCompactWY`` ``QRPivoted`` ------------ --------------------------------------------- ------------------ --------------------- ------------------ +=========== ============================================= ================== ===================== ================== ``F[:Q]`` ``Q`` (orthogonal/unitary) part of ``QR`` ✓ (``QRPackedQ``) ✓ (``QRCompactWYQ``) ✓ (``QRPackedQ``) ``F[:R]`` ``R`` (upper right triangular) part of ``QR`` ✓ ✓ ✓ ``F[:p]`` pivot ``Vector`` ✓ @@ -11378,7 +11378,7 @@ the ``format`` string. The following codes can be used for constructing format s =============== ========= =============================================================== Code Matches Comment ---------------- --------- --------------------------------------------------------------- +=============== ========= =============================================================== ``y`` 1996, 96 Returns year of 1996, 0096 ``m`` 1, 01 Matches 1 or 2-digit months ``u`` Jan Matches abbreviated months according to the ``locale`` keyword diff --git a/base/linalg/arnoldi.jl b/base/linalg/arnoldi.jl index 95080b2878944..4d6ca3513a2c0 100644 --- a/base/linalg/arnoldi.jl +++ b/base/linalg/arnoldi.jl @@ -19,7 +19,7 @@ The following keyword arguments are supported: ========= ====================================================================================================================== ``which`` type of eigenvalues - --------- ---------------------------------------------------------------------------------------------------------------------- + ========= ====================================================================================================================== ``:LM`` eigenvalues of largest magnitude (default) ``:SM`` eigenvalues of smallest magnitude ``:LR`` eigenvalues of largest real part @@ -41,7 +41,7 @@ The following keyword arguments are supported: =============== ================================== ================================== ``sigma`` iteration mode ``which`` refers to eigenvalues of - --------------- ---------------------------------- ---------------------------------- + =============== ================================== ================================== ``nothing`` ordinary (forward) :math:`A` real or complex inverse with level shift ``sigma`` :math:`(A - \sigma I )^{-1}` =============== ================================== ================================== @@ -66,7 +66,7 @@ The following keyword arguments are supported: ========= ====================================================================================================================== ``which`` type of eigenvalues - --------- ---------------------------------------------------------------------------------------------------------------------- + ========= ====================================================================================================================== ``:LM`` eigenvalues of largest magnitude (default) ``:SM`` eigenvalues of smallest magnitude ``:LR`` eigenvalues of largest real part diff --git a/doc/devdocs/stdio.rst b/doc/devdocs/stdio.rst index 6e5560f26a21b..63efb74b43283 100644 --- a/doc/devdocs/stdio.rst +++ b/doc/devdocs/stdio.rst @@ -120,8 +120,6 @@ This is needed because :c:func:`jl_printf` caller :c:func:`jl_static_show` is passed an :code:`ios_t` stream by femtolisp's :c:func:`fl_print` function. Julia's :c:func:`jl_write` function has special handling for this:: -```c -if (stream->type > UV_HANDLE_TYPE_MAX) { - return ios_write((ios_t*)stream, str, n); -} -``` + if (stream->type > UV_HANDLE_TYPE_MAX) { + return ios_write((ios_t*)stream, str, n); + } diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index c075d99e12396..653be83e228a0 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -735,7 +735,7 @@ reference. +----------------------------------------+----------------------------------+--------------------------------------------+ | Sparse | Dense | Description | -+----------------------------------------+----------------------------------+--------------------------------------------+ ++========================================+==================================+============================================+ | :func:`spzeros(m,n) ` | :func:`zeros(m,n) ` | Creates a *m*-by-*n* matrix of zeros. | | | | (:func:`spzeros(m,n) ` is empty.) | +----------------------------------------+----------------------------------+--------------------------------------------+ diff --git a/doc/manual/integers-and-floating-point-numbers.rst b/doc/manual/integers-and-floating-point-numbers.rst index 0394600d259cd..9e006c38ae747 100644 --- a/doc/manual/integers-and-floating-point-numbers.rst +++ b/doc/manual/integers-and-floating-point-numbers.rst @@ -30,7 +30,7 @@ The following are Julia's primitive numeric types: ================ ======= ============== ============== ================== Type Signed? Number of bits Smallest value Largest value ----------------- ------- -------------- -------------- ------------------ +================ ======= ============== ============== ================== :class:`Int8` ✓ 8 -2^7 2^7 - 1 :class:`UInt8` 8 0 2^8 - 1 :class:`Int16` ✓ 16 -2^15 2^15 - 1 @@ -48,7 +48,7 @@ Type Signed? Number of bits Smallest value Largest value ================ ========= ============== Type Precision Number of bits ----------------- --------- -------------- +================ ========= ============== :class:`Float16` half_ 16 :class:`Float32` single_ 32 :class:`Float64` double_ 64 @@ -755,7 +755,7 @@ specified type or the type of a given variable. ====================== ===================================================== Function Description ----------------------- ----------------------------------------------------- +====================== ===================================================== :func:`zero(x) ` Literal zero of type ``x`` or type of variable ``x`` :func:`one(x) ` Literal one of type ``x`` or type of variable ``x`` ====================== ===================================================== diff --git a/doc/manual/interfaces.rst b/doc/manual/interfaces.rst index cd5f74e3006d8..0a21a402cd0dd 100644 --- a/doc/manual/interfaces.rst +++ b/doc/manual/interfaces.rst @@ -265,7 +265,7 @@ The result of indexing an ``AbstractArray`` can itself be an array (for instance 1.0 4.0 7.0 2.0 5.0 8.0 -In this example it is accomplished by defining ``Base.similar{T}(A::SparseArray, ::Type{T}, dims::Dims)`` to create the appropriate wrapped array. For this to work it's important that ``SparseArray`` is mutable (supports ``setindex!``). :func:`similar` is also used to allocate result arrays for arithmetic on ``AbstractArray``s, for instance: +In this example it is accomplished by defining ``Base.similar{T}(A::SparseArray, ::Type{T}, dims::Dims)`` to create the appropriate wrapped array. For this to work it's important that ``SparseArray`` is mutable (supports ``setindex!``). :func:`similar` is also used to allocate result arrays for arithmetic on ``AbstractArrays``, for instance: .. doctest:: diff --git a/doc/manual/linear-algebra.rst b/doc/manual/linear-algebra.rst index 9a227b6ebe121..6b4d4dfd136c0 100644 --- a/doc/manual/linear-algebra.rst +++ b/doc/manual/linear-algebra.rst @@ -61,7 +61,7 @@ Elementary operations +-------------------------+-------+-------+-------+-------+--------------------------------+ | Matrix type | ``+`` | ``-`` | ``*`` | ``\`` | Other functions with | | | | | | | optimized methods | -+-------------------------+-------+-------+-------+-------+--------------------------------+ ++=========================+=======+=======+=======+=======+================================+ | :class:`Hermitian` | | | | MV | :func:`inv`, | | | | | | | :func:`sqrtm`, :func:`expm` | +-------------------------+-------+-------+-------+-------+--------------------------------+ diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index 067dfc4480393..89752a1c367be 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -690,7 +690,7 @@ Connections between workers (using the in-built TCP/IP transport) is established - :func:`addprocs` is called on the master process with a :obj:`ClusterManager` object - :func:`addprocs` calls the appropriate :func:`launch` method which spawns -required number of worker processes on appropriate machines + required number of worker processes on appropriate machines - Each worker starts listening on a free port and writes out its host, port information to :const:`STDOUT` - The cluster manager captures the stdout's of each worker and makes it available to the master process - The master process parses this information and sets up TCP/IP connections to each worker diff --git a/doc/stdlib/dates.rst b/doc/stdlib/dates.rst index 01f3ee10f2996..fa2f6a621fc7f 100644 --- a/doc/stdlib/dates.rst +++ b/doc/stdlib/dates.rst @@ -86,7 +86,7 @@ alternatively, you could call ``using Dates`` to bring all exported functions in =============== ========= =============================================================== Code Matches Comment - --------------- --------- --------------------------------------------------------------- + =============== ========= =============================================================== ``y`` 1996, 96 Returns year of 1996, 0096 ``m`` 1, 01 Matches 1 or 2-digit months ``u`` Jan Matches abbreviated months according to the ``locale`` keyword @@ -476,7 +476,7 @@ Days of the Week: =============== ========= ============= Variable Abbr. Value (Int) ---------------- --------- ------------- +=============== ========= ============= ``Monday`` ``Mon`` 1 ``Tuesday`` ``Tue`` 2 ``Wednesday`` ``Wed`` 3 @@ -490,7 +490,7 @@ Months of the Year: =============== ========= ============= Variable Abbr. Value (Int) ---------------- --------- ------------- +=============== ========= ============= ``January`` ``Jan`` 1 ``February`` ``Feb`` 2 ``March`` ``Mar`` 3 diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 17f6d99b9e2c8..bb84859ef6670 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -73,7 +73,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f ======================= ========================= ======================================== Type of input ``A`` Type of output ``F`` Relationship between ``F`` and ``A`` - ----------------------- ------------------------- ---------------------------------------- + ======================= ========================= ======================================== :func:`Matrix` ``LU`` ``F[:L]*F[:U] == A[F[:p], :]`` :func:`Tridiagonal` ``LU{T,Tridiagonal{T}}`` ``F[:L]*F[:U] == A[F[:p], :]`` :func:`SparseMatrixCSC` ``UmfpackLU`` ``F[:L]*F[:U] == (F[:Rs] .* A)[F[:p], F[:q]]`` @@ -83,7 +83,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f =========== ======================================= ====== ======================== ============= Component Description ``LU`` ``LU{T,Tridiagonal{T}}`` ``UmfpackLU`` - ----------- --------------------------------------- ------ ------------------------ ------------- + =========== ======================================= ====== ======================== ============= ``F[:L]`` ``L`` (lower triangular) part of ``LU`` ✓ ✓ ✓ ``F[:U]`` ``U`` (upper triangular) part of ``LU`` ✓ ✓ ✓ ``F[:p]`` (right) permutation ``Vector`` ✓ ✓ ✓ @@ -95,7 +95,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f ================== ====== ======================== ============= Supported function ``LU`` ``LU{T,Tridiagonal{T}}`` ``UmfpackLU`` - ------------------ ------ ------------------------ ------------- + ================== ====== ======================== ============= ``/`` ✓ ``\`` ✓ ✓ ✓ ``cond`` ✓ ✓ @@ -175,7 +175,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f ================ ================= ============== ===================================== Return type ``eltype(A)`` ``pivot`` Relationship between ``F`` and ``A`` - ---------------- ----------------- -------------- ------------------------------------- + ================ ================= ============== ===================================== ``QR`` not ``BlasFloat`` either ``A==F[:Q]*F[:R]`` ``QRCompactWY`` ``BlasFloat`` ``Val{false}`` ``A==F[:Q]*F[:R]`` ``QRPivoted`` ``BlasFloat`` ``Val{true}`` ``A[:,F[:p]]==F[:Q]*F[:R]`` @@ -187,7 +187,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f =========== ============================================= ================== ===================== ================== Component Description ``QR`` ``QRCompactWY`` ``QRPivoted`` - ----------- --------------------------------------------- ------------------ --------------------- ------------------ + =========== ============================================= ================== ===================== ================== ``F[:Q]`` ``Q`` (orthogonal/unitary) part of ``QR`` ✓ (``QRPackedQ``) ✓ (``QRCompactWYQ``) ✓ (``QRPackedQ``) ``F[:R]`` ``R`` (upper right triangular) part of ``QR`` ✓ ✓ ✓ ``F[:p]`` pivot ``Vector`` ✓ @@ -934,7 +934,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f ========= ====================================================================================================================== ``which`` type of eigenvalues - --------- ---------------------------------------------------------------------------------------------------------------------- + ========= ====================================================================================================================== ``:LM`` eigenvalues of largest magnitude (default) ``:SM`` eigenvalues of smallest magnitude ``:LR`` eigenvalues of largest real part @@ -956,7 +956,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f =============== ================================== ================================== ``sigma`` iteration mode ``which`` refers to eigenvalues of - --------------- ---------------------------------- ---------------------------------- + =============== ================================== ================================== ``nothing`` ordinary (forward) :math:`A` real or complex inverse with level shift ``sigma`` :math:`(A - \sigma I )^{-1}` =============== ================================== ================================== @@ -977,7 +977,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f ========= ====================================================================================================================== ``which`` type of eigenvalues - --------- ---------------------------------------------------------------------------------------------------------------------- + ========= ====================================================================================================================== ``:LM`` eigenvalues of largest magnitude (default) ``:SM`` eigenvalues of smallest magnitude ``:LR`` eigenvalues of largest real part @@ -1310,7 +1310,7 @@ Usually a function has 4 methods defined, one each for ``Float64``, An object of type ``UniformScaling``, representing an identity matrix of any size. LAPACK Functions --------------- +---------------- .. module:: Base.LinAlg.LAPACK From 2eb5a50bfecc44beb61b7ce52d9ef59f84afda39 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Tue, 29 Sep 2015 15:52:53 -0400 Subject: [PATCH 0366/1938] docs: fix another broken RST table --- base/linalg/arnoldi.jl | 2 +- doc/stdlib/linalg.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/linalg/arnoldi.jl b/base/linalg/arnoldi.jl index 4d6ca3513a2c0..0806a6c60aaf8 100644 --- a/base/linalg/arnoldi.jl +++ b/base/linalg/arnoldi.jl @@ -88,7 +88,7 @@ The following keyword arguments are supported: =============== ================================== ================================== ``sigma`` iteration mode ``which`` refers to the problem - --------------- ---------------------------------- ---------------------------------- + =============== ================================== ================================== ``nothing`` ordinary (forward) :math:`Av = Bv\lambda` real or complex inverse with level shift ``sigma`` :math:`(A - \sigma B )^{-1}B = v\nu` =============== ================================== ================================== diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index bb84859ef6670..591222d739e54 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -999,7 +999,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f =============== ================================== ================================== ``sigma`` iteration mode ``which`` refers to the problem - --------------- ---------------------------------- ---------------------------------- + =============== ================================== ================================== ``nothing`` ordinary (forward) :math:`Av = Bv\lambda` real or complex inverse with level shift ``sigma`` :math:`(A - \sigma B )^{-1}B = v\nu` =============== ================================== ================================== From 35cd85f327217813f850d2b7fc5bbe66f6b5bc8c Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Tue, 29 Sep 2015 15:29:04 -0400 Subject: [PATCH 0367/1938] add error branch test for package availablity --- test/pkg.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/pkg.jl b/test/pkg.jl index 784e4e7ae295d..6c6deea4ec916 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -41,6 +41,7 @@ temp_pkg_dir() do Pkg.rm("Example") @test isempty(Pkg.installed()) @test !isempty(Pkg.available("Example")) + @test Pkg.available("IDoNotExist") === nothing Pkg.clone("https://github.com/JuliaLang/Example.jl.git") @test [keys(Pkg.installed())...] == ["Example"] Pkg.status("Example", iob) From 2305d49f74d21dcc776aa8b5a8e2d5490739b80a Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Tue, 29 Sep 2015 11:08:23 -0400 Subject: [PATCH 0368/1938] Support `@doc "" -> f()` syntax early in bootstrap and move windows only doc string to julia. --- base/docs/bootstrap.jl | 15 +++++++++++-- base/libc.jl | 49 ++++++++++++++++++++++++++---------------- doc/stdlib/libc.rst | 4 ++++ 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/base/docs/bootstrap.jl b/base/docs/bootstrap.jl index 8e045262723ad..9fbe4b4bfafe9 100644 --- a/base/docs/bootstrap.jl +++ b/base/docs/bootstrap.jl @@ -19,11 +19,22 @@ _expand_ = nothing setexpand!(f) = global _expand_ = f -setexpand!() do str, obj +function __bootexpand(str, obj) global docs = List((ccall(:jl_get_current_module, Any, ()), str, obj), docs) - (isa(obj, Expr) && obj.head == :call) ? nothing : esc(Expr(:toplevel, obj)) + (isa(obj, Expr) && obj.head == :call) && return nothing + (isa(obj, Expr) && obj.head == :module) && return esc(Expr(:toplevel, obj)) + esc(obj) end +function __bootexpand(expr::Expr) + if expr.head !== :-> + throw(ArgumentError("Wrong argument to @doc")) + end + __bootexpand(expr.args...) +end + +setexpand!(__bootexpand) + """ DocBootstrap :: Module diff --git a/base/libc.jl b/base/libc.jl index fe0a2fe3b0467..a6ab779aa2192 100644 --- a/base/libc.jl +++ b/base/libc.jl @@ -167,25 +167,36 @@ strerror(e::Integer) = bytestring(ccall(:strerror, Ptr{UInt8}, (Int32,), e)) strerror() = strerror(errno()) @windows_only begin -GetLastError() = ccall(:GetLastError,stdcall,UInt32,()) -function FormatMessage(e=GetLastError()) - const FORMAT_MESSAGE_ALLOCATE_BUFFER = UInt32(0x100) - const FORMAT_MESSAGE_FROM_SYSTEM = UInt32(0x1000) - const FORMAT_MESSAGE_IGNORE_INSERTS = UInt32(0x200) - const FORMAT_MESSAGE_MAX_WIDTH_MASK = UInt32(0xFF) - lpMsgBuf = Array(Ptr{UInt16}) - lpMsgBuf[1] = 0 - len = ccall(:FormatMessageW,stdcall,UInt32,(Cint, Ptr{Void}, Cint, Cint, Ptr{Ptr{UInt16}}, Cint, Ptr{Void}), - FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_MAX_WIDTH_MASK, - C_NULL, e, 0, lpMsgBuf, 0, C_NULL) - p = lpMsgBuf[1] - len == 0 && return utf8("") - len = len + 1 - buf = Array(UInt16, len) - unsafe_copy!(pointer(buf), p, len) - ccall(:LocalFree,stdcall,Ptr{Void},(Ptr{Void},),p) - return utf8(UTF16String(buf)) -end + @doc """ + GetLastError() + + Call the Win32 `GetLastError` function [only available on Windows]. + """ -> + GetLastError() = ccall(:GetLastError,stdcall,UInt32,()) + + @doc """ + FormatMessage(n=GetLastError()) + + Convert a Win32 system call error code to a descriptive string [only available on Windows]. + """ -> + function FormatMessage(e=GetLastError()) + const FORMAT_MESSAGE_ALLOCATE_BUFFER = UInt32(0x100) + const FORMAT_MESSAGE_FROM_SYSTEM = UInt32(0x1000) + const FORMAT_MESSAGE_IGNORE_INSERTS = UInt32(0x200) + const FORMAT_MESSAGE_MAX_WIDTH_MASK = UInt32(0xFF) + lpMsgBuf = Array(Ptr{UInt16}) + lpMsgBuf[1] = 0 + len = ccall(:FormatMessageW,stdcall,UInt32,(Cint, Ptr{Void}, Cint, Cint, Ptr{Ptr{UInt16}}, Cint, Ptr{Void}), + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_MAX_WIDTH_MASK, + C_NULL, e, 0, lpMsgBuf, 0, C_NULL) + p = lpMsgBuf[1] + len == 0 && return utf8("") + len = len + 1 + buf = Array(UInt16, len) + unsafe_copy!(pointer(buf), p, len) + ccall(:LocalFree,stdcall,Ptr{Void},(Ptr{Void},),p) + return utf8(UTF16String(buf)) + end end ## Memory related ## diff --git a/doc/stdlib/libc.rst b/doc/stdlib/libc.rst index ba36122b74065..4510a8baeafef 100644 --- a/doc/stdlib/libc.rst +++ b/doc/stdlib/libc.rst @@ -46,10 +46,14 @@ .. function:: GetLastError() + .. Docstring generated from Julia source + Call the Win32 ``GetLastError`` function [only available on Windows]. .. function:: FormatMessage(n=GetLastError()) + .. Docstring generated from Julia source + Convert a Win32 system call error code to a descriptive string [only available on Windows]. .. function:: time(t::TmStruct) From ed6a1b5c5d873c82b59d797c93d7d969486b8fb6 Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Tue, 29 Sep 2015 22:57:34 -0400 Subject: [PATCH 0369/1938] reexport old Git module for now --- base/exports.jl | 1 + base/pkg.jl | 2 +- base/sysimg.jl | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/base/exports.jl b/base/exports.jl index 5a36c596c25dc..4884bb8bcd361 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -7,6 +7,7 @@ export Meta, Operators, Pkg, + Git, LibGit2, Profile, Dates, diff --git a/base/pkg.jl b/base/pkg.jl index 65312613b2e6e..5c49c5ff26860 100644 --- a/base/pkg.jl +++ b/base/pkg.jl @@ -2,7 +2,7 @@ module Pkg -export Dir, GitHub, Types, Reqs, Cache, Read, Query, Resolve, Write, Generate, Entry, Git +export Git, Dir, GitHub, Types, Reqs, Cache, Read, Query, Resolve, Write, Generate, Entry, Git export dir, init, rm, add, available, installed, status, clone, checkout, update, resolve, register, tag, publish, generate, test, build, free, pin, PkgError diff --git a/base/sysimg.jl b/base/sysimg.jl index 3d0b4da5e26fe..c555a41cf6d6c 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -278,6 +278,7 @@ include("libgit2.jl") # package manager include("pkg.jl") +const Git = Pkg.Git # profiler include("profile.jl") From d709e11d43d6d8542f55549c890efa5bef6a90ac Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Wed, 30 Sep 2015 00:09:56 -0400 Subject: [PATCH 0370/1938] properly finalize the GitRepo when an exception is thrown --- base/pkg/read.jl | 63 ++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/base/pkg/read.jl b/base/pkg/read.jl index 19288a7518969..df00d661ec88d 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -77,23 +77,26 @@ function isfixed(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::Dict=availa else false end - res = true - for (ver,info) in avail - if cache_has_head && LibGit2.iscommit(info.sha1, crepo) - if LibGit2.is_ancestor_of(head, info.sha1, crepo) - res = false - break - end - elseif LibGit2.iscommit(info.sha1, prepo) - if LibGit2.is_ancestor_of(head, info.sha1, prepo) - res = false - break + try + res = true + for (ver,info) in avail + if cache_has_head && LibGit2.iscommit(info.sha1, crepo) + if LibGit2.is_ancestor_of(head, info.sha1, crepo) + res = false + break + end + elseif LibGit2.iscommit(info.sha1, prepo) + if LibGit2.is_ancestor_of(head, info.sha1, prepo) + res = false + break + end + else + Base.warn_once("unknown $pkg commit $(info.sha1[1:8]), metadata may be ahead of package cache") end - else - Base.warn_once("unknown $pkg commit $(info.sha1[1:8]), metadata may be ahead of package cache") end + finally + cache_has_head && LibGit2.finalize(crepo) end - cache_has_head && LibGit2.finalize(crepo) return res end @@ -116,21 +119,23 @@ function installed_version(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::D end ancestors = VersionNumber[] descendants = VersionNumber[] - for (ver,info) in avail - sha1 = info.sha1 - base = if cache_has_head && LibGit2.iscommit(sha1, crepo) - LibGit2.merge_base(crepo, head, sha1) - elseif LibGit2.iscommit(sha1, prepo) - LibGit2.merge_base(prepo, head, sha1) - else - Base.warn_once("unknown $pkg commit $(sha1[1:8]), metadata may be ahead of package cache") - continue + try + for (ver,info) in avail + sha1 = info.sha1 + base = if cache_has_head && LibGit2.iscommit(sha1, crepo) + LibGit2.merge_base(crepo, head, sha1) + elseif LibGit2.iscommit(sha1, prepo) + LibGit2.merge_base(prepo, head, sha1) + else + Base.warn_once("unknown $pkg commit $(sha1[1:8]), metadata may be ahead of package cache") + continue + end + base == sha1 && push!(ancestors,ver) + base == head && push!(descendants,ver) end - base == sha1 && push!(ancestors,ver) - base == head && push!(descendants,ver) + finally + cache_has_head && LibGit2.finalize(crepo) end - cache_has_head && LibGit2.finalize(crepo) - both = sort!(intersect(ancestors,descendants)) isempty(both) || warn("$pkg: some versions are both ancestors and descendants of head: $both") if !isempty(descendants) @@ -173,8 +178,8 @@ function installed(avail::Dict=available()) for pkg in readdir() isinstalled(pkg) || continue ap = get(avail,pkg,Dict{VersionNumber,Available}()) + prepo = LibGit2.GitRepo(pkg) try - prepo = LibGit2.GitRepo(pkg) try ver = installed_version(pkg, prepo, ap) fixed = isfixed(pkg, prepo, ap) @@ -182,7 +187,7 @@ function installed(avail::Dict=available()) catch e pkgs[pkg] = (typemin(VersionNumber), true) finally - finalize(prepo) + LibGit2.finalize(prepo) end catch pkgs[pkg] = (typemin(VersionNumber), true) From 4deb73f9d95a55387afbe4076ab1513d599b6383 Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Wed, 30 Sep 2015 00:22:06 -0400 Subject: [PATCH 0371/1938] fix bug where `LibGit2.Oid` was not geting converted to string --- base/pkg/entry.jl | 2 +- test/pkg.jl | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 04a5c23b7c5c3..49a759403068a 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -655,7 +655,7 @@ function tag(pkg::AbstractString, ver::Union{Symbol,VersionNumber}, force::Bool= filter!(tag->ismatch(Base.VERSION_REGEX,tag), tags) existing = VersionNumber[tags...] filter!(tags) do tag - sha1 = LibGit2.revparseid(repo, "$tag^{commit}") + sha1 = string(LibGit2.revparseid(repo, "$tag^{commit}")) LibGit2.is_ancestor_of(sha1, commit, repo) end ancestors = VersionNumber[tags...] diff --git a/test/pkg.jl b/test/pkg.jl index 6c6deea4ec916..794442d4e9e3c 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -15,6 +15,7 @@ function temp_pkg_dir(fn::Function) end end + # Test basic operations: adding or removing a package, status, free #Also test for the existence of REQUIRE and META_Branch temp_pkg_dir() do @@ -163,3 +164,10 @@ end""" end end end + +# issue #13373 +temp_pkg_dir() do + Pkg.generate("Foo", "MIT") + Pkg.tag("Foo") + Pkg.tag("Foo") +end From 0a44ce8177def25b3003490171a1c19a49156640 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sun, 20 Sep 2015 13:39:07 +0200 Subject: [PATCH 0372/1938] move some sparse docs from helpdb.jl --- base/docs/helpdb.jl | 147 ---------------------------------- base/reduce.jl | 6 ++ base/sparse/abstractsparse.jl | 5 ++ base/sparse/csparse.jl | 17 ++++ base/sparse/sparsematrix.jl | 122 ++++++++++++++++++++++++++++ 5 files changed, 150 insertions(+), 147 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index a32159d3dae29..7b955106ee92a 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -875,13 +875,6 @@ Calling `Ref(array[, index])` is generally preferable to this function. """ pointer -doc""" - countnz(A) - -Counts the number of nonzero values in array `A` (dense or sparse). Note that this is not a constant-time operation. For sparse matrices, one should usually use `nnz`, which returns the number of stored values. -""" -countnz - doc""" isnan(f) -> Bool @@ -1593,25 +1586,6 @@ Compute the inverse cosine of `x`, where the output is in radians """ acos -doc""" - nzrange(A, col) - -Return the range of indices to the structural nonzero values of a sparse matrix column. In conjunction with `nonzeros(A)` and `rowvals(A)`, this allows for convenient iterating over a sparse matrix : - - A = sparse(I,J,V) - rows = rowvals(A) - vals = nonzeros(A) - m, n = size(A) - for i = 1:n - for j in nzrange(A, i) - row = rows[j] - val = vals[j] - # perform sparse wizardry... - end - end -""" -nzrange - doc""" ispath(path) -> Bool @@ -1693,13 +1667,6 @@ Scaled Bessel function of the third kind of order `nu`, $H^{(1)}_\nu(x) e^{-x i} """ hankelh1x -doc""" - blkdiag(A...) - -Concatenate matrices block-diagonally. Currently only implemented for sparse matrices. -""" -blkdiag - doc""" replace(string, pat, r[, n]) @@ -2100,13 +2067,6 @@ Transform the given dimensions of array `A` using function `f`. `f` is called on """ mapslices -doc""" - spdiagm(B, d[, m, n]) - -Construct a sparse diagonal matrix. `B` is a tuple of vectors containing the diagonals and `d` is a tuple containing the positions of the diagonals. In the case the input contains only one diagonaly, `B` can be a vector (instead of a tuple) and `d` can be the diagonal position (instead of a tuple), defaulting to 0 (diagonal). Optionally, `m` and `n` specify the size of the resulting sparse matrix. -""" -spdiagm - doc""" svdvals(A) @@ -2601,13 +2561,6 @@ From an array view `A`, returns the corresponding indexes in the parent """ parentindexes -doc""" - spones(S) - -Create a sparse matrix with the same structure as that of `S`, but with every nonzero element having the value `1.0`. -""" -spones - doc""" display(x) display(d::Display, x) @@ -2907,13 +2860,6 @@ Get a hexadecimal string of the binary representation of a floating point number """ num2hex -doc""" - speye(type,m[,n]) - -Create a sparse identity matrix of specified type of size `m x m`. In case `n` is supplied, create a sparse identity matrix of size `m x n`. -""" -speye - doc""" count_ones(x::Integer) -> Integer @@ -3252,20 +3198,6 @@ For matrices or vectors $A$ and $B$, calculates $A / Bᴴ$ """ A_rdiv_Bc -doc""" - sparse(I,J,V,[m,n,combine]) - -Create a sparse matrix `S` of dimensions `m x n` such that `S[I[k], J[k]] = V[k]`. The `combine` function is used to combine duplicates. If `m` and `n` are not specified, they are set to `maximum(I)` and `maximum(J)` respectively. If the `combine` function is not supplied, duplicates are added by default. All elements of `I` must satisfy `1 <= I[k] <= m`, and all elements of `J` must satisfy `1 <= J[k] <= n`. -""" -sparse(I, J, V, m=?, n=?, combine=?) - -doc""" - sparse(A) - -Convert an AbstractMatrix `A` into a sparse matrix. -""" -sparse(A) - doc""" ```rst .. round([T,] x, [digits, [base]], [r::RoundingMode]) @@ -4793,13 +4725,6 @@ For a given iterable object and iteration state, return the current item and the """ next -doc""" - nnz(A) - -Returns the number of stored (filled) elements in a sparse matrix. -""" -nnz - doc""" unshift!(collection, items...) -> collection @@ -4834,13 +4759,6 @@ Construct a real symmetric tridiagonal matrix from the diagonal and upper diagon """ SymTridiagonal -doc""" - spzeros(m,n) - -Create a sparse matrix of size `m x n`. This sparse matrix will not contain any nonzero values. No storage will be allocated for nonzero values during construction. -""" -spzeros - doc""" colon(start, [step], stop) @@ -5725,13 +5643,6 @@ julia> trailing_zeros(2) """ trailing_zeros -doc""" - etree(A[, post]) - -Compute the elimination tree of a symmetric sparse matrix `A` from `triu(A)` and, optionally, its post-ordering permutation. -""" -etree - doc""" isalnum(c::Union{Char,AbstractString}) -> Bool @@ -5893,27 +5804,6 @@ Remainder after division, returning in the range (0,m\] """ rem1 -doc""" - sparsevec(I, V, [m, combine]) - -Create a sparse matrix `S` of size `m x 1` such that `S[I[k]] = V[k]`. Duplicates are combined using the `combine` function, which defaults to `+` if it is not provided. In julia, sparse vectors are really just sparse matrices with one column. Given Julia's Compressed Sparse Columns (CSC) storage format, a sparse column matrix with one column is sparse, whereas a sparse row matrix with one row ends up being dense. -""" -sparsevec(I, V) - -doc""" - sparsevec(D::Dict, [m]) - -Create a sparse matrix of size `m x 1` where the row values are keys from the dictionary, and the nonzero values are the values from the dictionary. -""" -sparsevec(D::Dict) - -doc""" - sparsevec(A) - -Convert a dense vector `A` into a sparse matrix of size `m x 1`. In julia, sparse vectors are really just sparse matrices with one column. -""" -sparsevec(A) - doc""" isalpha(c::Union{Char,AbstractString}) -> Bool @@ -7094,13 +6984,6 @@ Get the file name part of a path. """ basename -doc""" - issparse(S) - -Returns `true` if `S` is sparse, and `false` otherwise. -""" -issparse - doc""" ArgumentError(msg) @@ -8867,13 +8750,6 @@ Compute the inverse hyperbolic secant of `x` """ asech -doc""" - sprandn(m,n,p) - -Create a random `m` by `n` sparse matrix with the specified (independent) probability `p` of any entry being nonzero, where nonzero values are sampled from the normal distribution. -""" -sprandn - doc""" ```rst .. ismarked(s) @@ -10094,13 +9970,6 @@ Rounds (in the sense of `round`) `x` so that there are `digits` significant digi """ signif -doc""" - sprandbool(m,n,p) - -Create a random `m` by `n` sparse boolean matrix with the specified (independent) probability `p` of any entry being `true`. -""" -sprandbool - doc""" nextpow2(n) @@ -10420,15 +10289,6 @@ the Riemann zeta function $\zeta(s)$ for the case of `z=1`.) """ zeta(s,z) -doc""" -```rst -.. sprand([rng,] m,n,p [,rfn]) - -Create a random ``m`` by ``n`` sparse matrix, in which the probability of any element being nonzero is independently given by ``p`` (and hence the mean density of nonzeros is also exactly ``p``). Nonzero values are sampled from the distribution specified by ``rfn``. The uniform distribution is used in case ``rfn`` is not specified. The optional ``rng`` argument specifies a random number generator, see :ref:`Random Numbers `. -``` -""" -sprand - doc""" A_mul_Bt(A, B) @@ -10893,13 +10753,6 @@ Quit (or control-D at the prompt). The default exit code is zero, indicating tha """ exit -doc""" - nonzeros(A) - -Return a vector of the structural nonzero values in sparse matrix `A`. This includes zeros that are explicitly stored in the sparse matrix. The returned vector points directly to the internal nonzero storage of `A`, and any modifications to the returned vector will mutate `A` as well. See `rowvals(A)` and `nzrange(A, col)`. -""" -nonzeros - doc""" istext(m::MIME) diff --git a/base/reduce.jl b/base/reduce.jl index 09099a954e670..7dbf314e7b601 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -410,4 +410,10 @@ end immutable NotEqZero <: Func{1} end call(::NotEqZero, x) = x != 0 +""" + countnz(A) + +Counts the number of nonzero values in array `A` (dense or sparse). Note that this is not a constant-time operation. +For sparse matrices, one should usually use `nnz`, which returns the number of stored values. +""" countnz(a) = count(NotEqZero(), a) diff --git a/base/sparse/abstractsparse.jl b/base/sparse/abstractsparse.jl index 41255cfaef42b..b19a54f4910af 100644 --- a/base/sparse/abstractsparse.jl +++ b/base/sparse/abstractsparse.jl @@ -5,6 +5,11 @@ abstract AbstractSparseArray{Tv,Ti,N} <: AbstractArray{Tv,N} typealias AbstractSparseVector{Tv,Ti} AbstractSparseArray{Tv,Ti,1} typealias AbstractSparseMatrix{Tv,Ti} AbstractSparseArray{Tv,Ti,2} +""" + issparse(S) + +Returns `true` if `S` is sparse, and `false` otherwise. +""" issparse(A::AbstractArray) = false issparse(S::AbstractSparseArray) = true diff --git a/base/sparse/csparse.jl b/base/sparse/csparse.jl index 4b8afc0f48aaa..695ad08236213 100644 --- a/base/sparse/csparse.jl +++ b/base/sparse/csparse.jl @@ -12,6 +12,16 @@ # Based on Direct Methods for Sparse Linear Systems, T. A. Davis, SIAM, Philadelphia, Sept. 2006. # Section 2.4: Triplet form # http://www.cise.ufl.edu/research/sparse/CSparse/ + +""" + sparse(I,J,V,[m,n,combine]) + +Create a sparse matrix `S` of dimensions `m x n` such that `S[I[k], J[k]] = V[k]`. +The `combine` function is used to combine duplicates. If `m` and `n` are not +specified, they are set to `maximum(I)` and `maximum(J)` respectively. If the +`combine` function is not supplied, duplicates are added by default. All elements +of `I` must satisfy `1 <= I[k] <= m`, and all elements of `J` must satisfy `1 <= J[k] <= n`. +""" function sparse{Tv,Ti<:Integer}(I::AbstractVector{Ti}, J::AbstractVector{Ti}, V::AbstractVector{Tv}, @@ -195,6 +205,13 @@ end # A root node is indicated by 0. This tree may actually be a forest in that # there may be more than one root, indicating complete separability. # A trivial example is speye(n, n) in which every node is a root. + +""" + etree(A[, post]) + +Compute the elimination tree of a symmetric sparse matrix `A` from `triu(A)` +and, optionally, its post-ordering permutation. +""" function etree{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, postorder::Bool) m,n = size(A) Ap = A.colptr diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index e63d3e1bd5149..ce5bf77c30852 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -25,11 +25,46 @@ function SparseMatrixCSC(m::Integer, n::Integer, colptr::Vector, rowval::Vector, end size(S::SparseMatrixCSC) = (S.m, S.n) + +""" + nnz(A) + +Returns the number of stored (filled) elements in a sparse matrix. +""" nnz(S::SparseMatrixCSC) = Int(S.colptr[end]-1) countnz(S::SparseMatrixCSC) = countnz(S.nzval) +""" + nonzeros(A) + +Return a vector of the structural nonzero values in sparse matrix `A`. This +includes zeros that are explicitly stored in the sparse matrix. The returned +vector points directly to the internal nonzero storage of `A`, and any +modifications to the returned vector will mutate `A` as well. See `rowvals(A)` +and `nzrange(A, col)`. +""" nonzeros(S::SparseMatrixCSC) = S.nzval rowvals(S::SparseMatrixCSC) = S.rowval + +""" + nzrange(A, col) + +Return the range of indices to the structural nonzero values of a sparse matrix +column. In conjunction with `nonzeros(A)` and `rowvals(A)`, this allows for +convenient iterating over a sparse matrix : + + A = sparse(I,J,V) + rows = rowvals(A) + vals = nonzeros(A) + m, n = size(A) + for i = 1:n + for j in nzrange(A, i) + row = rows[j] + val = vals[j] + # perform sparse wizardry... + end + end +""" nzrange(S::SparseMatrixCSC, col::Integer) = S.colptr[col]:(S.colptr[col+1]-1) function Base.showarray(io::IO, S::SparseMatrixCSC; @@ -212,9 +247,20 @@ function vec{Tv,Ti}(S::SparseMatrixCSC{Tv,Ti}) SparseMatrixCSC(lS, 1, colptr, rowval, copy(S.nzval)) end +""" + sparse(A) + +Convert an AbstractMatrix `A` into a sparse matrix. +""" sparsevec(A::AbstractMatrix) = reshape(sparse(A), (length(A),1)) sparsevec(S::SparseMatrixCSC) = vec(S) +""" + sparsevec(D::Dict, [m]) + +Create a sparse matrix of size `m x 1` where the row values are keys from +the dictionary, and the nonzero values are the values from the dictionary. +""" sparsevec{K<:Integer,V}(d::Dict{K,V}, len::Int) = sparsevec(collect(keys(d)), collect(values(d)), len) sparsevec{K<:Integer,V}(d::Dict{K,V}) = sparsevec(collect(keys(d)), collect(values(d))) @@ -223,6 +269,16 @@ sparsevec(I::AbstractVector, V, m::Integer) = sparsevec(I, V, m, AddFun()) sparsevec(I::AbstractVector, V) = sparsevec(I, V, maximum(I), AddFun()) +""" + sparsevec(I, V, [m, combine]) + +Create a sparse matrix `S` of size `m x 1` such that `S[I[k]] = V[k]`. +Duplicates are combined using the `combine` function, which defaults to +`+` if it is not provided. In julia, sparse vectors are really just sparse +matrices with one column. Given Julia's Compressed Sparse Columns (CSC) +storage format, a sparse column matrix with one column is sparse, whereas +a sparse row matrix with one row ends up being dense. +""" function sparsevec(I::AbstractVector, V, m::Integer, combine::Union{Function,Base.Func}) nI = length(I) if isa(V, Number) @@ -245,6 +301,12 @@ function sparsevec(I::AbstractVector, V, m::Integer, combine::Union{Function,Bas sparse_IJ_sorted!(I, ones(eltype(I), nI), V, m, 1, combine) end +""" + sparsevec(A) + +Convert a dense vector `A` into a sparse matrix of size `m x 1`. In julia, +sparse vectors are really just sparse matrices with one column. +""" function sparsevec(a::Vector) n = length(a) I = find(a) @@ -458,18 +520,57 @@ function sprand{T}(m::Integer, n::Integer, density::AbstractFloat, sparse_IJ_sorted!(I, J, rfn(length(I)), m, n, AddFun()) # it will never need to combine end +""" +```rst +.. sprand([rng,] m,n,p [,rfn]) + +Create a random ``m`` by ``n`` sparse matrix, in which the probability of any +element being nonzero is independently given by ``p`` (and hence the mean +density of nonzeros is also exactly ``p``). Nonzero values are sampled from +the distribution specified by ``rfn``. The uniform distribution is used in +case ``rfn`` is not specified. The optional ``rng`` argument specifies a +random number generator, see :ref:`Random Numbers `. +``` +""" sprand(r::AbstractRNG, m::Integer, n::Integer, density::AbstractFloat) = sprand(r,m,n,density,rand,Float64) sprand(m::Integer, n::Integer, density::AbstractFloat) = sprand(GLOBAL_RNG,m,n,density) sprandn(r::AbstractRNG, m::Integer, n::Integer, density::AbstractFloat) = sprand(r,m,n,density,randn,Float64) + +""" + sprandn(m,n,p) + +Create a random `m` by `n` sparse matrix with the specified (independent) +probability `p` of any entry being nonzero, where nonzero values are +sampled from the normal distribution. +""" sprandn( m::Integer, n::Integer, density::AbstractFloat) = sprandn(GLOBAL_RNG,m,n,density) truebools(r::AbstractRNG, n::Integer) = ones(Bool, n) sprandbool(r::AbstractRNG, m::Integer, n::Integer, density::AbstractFloat) = sprand(r,m,n,density,truebools,Bool) + +""" + sprandbool(m,n,p) + +Create a random `m` by `n` sparse boolean matrix with the specified +(independent) probability `p` of any entry being `true`. +""" sprandbool(m::Integer, n::Integer, density::AbstractFloat) = sprandbool(GLOBAL_RNG,m,n,density) +""" + spones(S) + +Create a sparse matrix with the same structure as that of `S`, but with every nonzero +element having the value `1.0`. +""" spones{T}(S::SparseMatrixCSC{T}) = SparseMatrixCSC(S.m, S.n, copy(S.colptr), copy(S.rowval), ones(T, S.colptr[end]-1)) +""" + spzeros(m,n) + +Create a sparse matrix of size `m x n`. This sparse matrix will not contain any +nonzero values. No storage will be allocated for nonzero values during construction. +""" spzeros(m::Integer, n::Integer) = spzeros(Float64, m, n) spzeros(Tv::Type, m::Integer, n::Integer) = spzeros(Tv, Int, m, n) function spzeros(Tv::Type, Ti::Type, m::Integer, n::Integer) @@ -477,12 +578,19 @@ function spzeros(Tv::Type, Ti::Type, m::Integer, n::Integer) SparseMatrixCSC(m, n, ones(Ti, n+1), Array(Ti, 0), Array(Tv, 0)) end + speye(n::Integer) = speye(Float64, n) speye(T::Type, n::Integer) = speye(T, n, n) speye(m::Integer, n::Integer) = speye(Float64, m, n) speye{T}(S::SparseMatrixCSC{T}) = speye(T, size(S, 1), size(S, 2)) eye(S::SparseMatrixCSC) = speye(S) +""" + speye(type,m[,n]) + +Create a sparse identity matrix of specified type of size `m x m`. In case `n` is supplied, +create a sparse identity matrix of size `m x n`. +""" function speye(T::Type, m::Integer, n::Integer) ((m < 0) || (n < 0)) && throw(ArgumentError("invalid Array dimensions")) x = min(m,n) @@ -2416,6 +2524,11 @@ function hvcat(rows::Tuple{Vararg{Int}}, X::SparseMatrixCSC...) vcat(tmp_rows...) end +""" + blkdiag(A...) + +Concatenate matrices block-diagonally. Currently only implemented for sparse matrices. +""" function blkdiag(X::SparseMatrixCSC...) num = length(X) mX = [ size(x, 1) for x in X ] @@ -2565,6 +2678,15 @@ function spdiagm_internal(B, d) return (I,J,V) end +""" + spdiagm(B, d[, m, n]) + +Construct a sparse diagonal matrix. `B` is a tuple of vectors containing the diagonals and +`d` is a tuple containing the positions of the diagonals. In the case the input contains only +one diagonaly, `B` can be a vector (instead of a tuple) and `d` can be the diagonal position +(instead of a tuple), defaulting to 0 (diagonal). Optionally, `m` and `n` specify the size +of the resulting sparse matrix. +""" function spdiagm(B, d, m::Integer, n::Integer) (I,J,V) = spdiagm_internal(B, d) return sparse(I,J,V,m,n) From a4897f6a2db1e317cef476ecd8106b479fb3178d Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 30 Sep 2015 09:57:48 +0200 Subject: [PATCH 0373/1938] rerun genstdlib.jl --- doc/stdlib/arrays.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 3f855a4301564..611e59dc097e0 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -963,7 +963,12 @@ Sparse matrices support much of the same set of operations as dense matrices. Th .. Docstring generated from Julia source - Create a random ``m`` by ``n`` sparse matrix, in which the probability of any element being nonzero is independently given by ``p`` (and hence the mean density of nonzeros is also exactly ``p``). Nonzero values are sampled from the distribution specified by ``rfn``. The uniform distribution is used in case ``rfn`` is not specified. The optional ``rng`` argument specifies a random number generator, see :ref:`Random Numbers `. + Create a random ``m`` by ``n`` sparse matrix, in which the probability of any + element being nonzero is independently given by ``p`` (and hence the mean + density of nonzeros is also exactly ``p``). Nonzero values are sampled from + the distribution specified by ``rfn``. The uniform distribution is used in + case ``rfn`` is not specified. The optional ``rng`` argument specifies a + random number generator, see :ref:`Random Numbers `. .. function:: sprandn(m,n,p) From ae419ce10478f37b1ae39a72fa0eb50693b28343 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 30 Sep 2015 10:59:55 +0200 Subject: [PATCH 0374/1938] fixups of sparse docs --- base/sparse/sparsematrix.jl | 36 +++++++++++++++++------------------- doc/stdlib/arrays.rst | 2 +- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index ce5bf77c30852..124a368d751cc 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -247,11 +247,6 @@ function vec{Tv,Ti}(S::SparseMatrixCSC{Tv,Ti}) SparseMatrixCSC(lS, 1, colptr, rowval, copy(S.nzval)) end -""" - sparse(A) - -Convert an AbstractMatrix `A` into a sparse matrix. -""" sparsevec(A::AbstractMatrix) = reshape(sparse(A), (length(A),1)) sparsevec(S::SparseMatrixCSC) = vec(S) @@ -317,8 +312,11 @@ end sparse(a::Vector) = sparsevec(a) -## Construct a sparse matrix +""" + sparse(A) +Convert an AbstractMatrix `A` into a sparse matrix. +""" sparse{Tv}(A::AbstractMatrix{Tv}) = convert(SparseMatrixCSC{Tv,Int}, A) sparse(S::SparseMatrixCSC) = copy(S) @@ -500,6 +498,18 @@ function sprand_IJ(r::AbstractRNG, m::Integer, n::Integer, density::AbstractFloa I, J end +""" +```rst +.. sprand([rng,] m,n,p [,rfn]) + +Create a random ``m`` by ``n`` sparse matrix, in which the probability of any +element being nonzero is independently given by ``p`` (and hence the mean +density of nonzeros is also exactly ``p``). Nonzero values are sampled from +the distribution specified by ``rfn``. The uniform distribution is used in +case ``rfn`` is not specified. The optional ``rng`` argument specifies a +random number generator, see :ref:`Random Numbers `. +``` +""" function sprand{T}(r::AbstractRNG, m::Integer, n::Integer, density::AbstractFloat, rfn::Function, ::Type{T}=eltype(rfn(r,1))) N = m*n @@ -520,18 +530,6 @@ function sprand{T}(m::Integer, n::Integer, density::AbstractFloat, sparse_IJ_sorted!(I, J, rfn(length(I)), m, n, AddFun()) # it will never need to combine end -""" -```rst -.. sprand([rng,] m,n,p [,rfn]) - -Create a random ``m`` by ``n`` sparse matrix, in which the probability of any -element being nonzero is independently given by ``p`` (and hence the mean -density of nonzeros is also exactly ``p``). Nonzero values are sampled from -the distribution specified by ``rfn``. The uniform distribution is used in -case ``rfn`` is not specified. The optional ``rng`` argument specifies a -random number generator, see :ref:`Random Numbers `. -``` -""" sprand(r::AbstractRNG, m::Integer, n::Integer, density::AbstractFloat) = sprand(r,m,n,density,rand,Float64) sprand(m::Integer, n::Integer, density::AbstractFloat) = sprand(GLOBAL_RNG,m,n,density) sprandn(r::AbstractRNG, m::Integer, n::Integer, density::AbstractFloat) = sprand(r,m,n,density,randn,Float64) @@ -2683,7 +2681,7 @@ end Construct a sparse diagonal matrix. `B` is a tuple of vectors containing the diagonals and `d` is a tuple containing the positions of the diagonals. In the case the input contains only -one diagonaly, `B` can be a vector (instead of a tuple) and `d` can be the diagonal position +one diagonal, `B` can be a vector (instead of a tuple) and `d` can be the diagonal position (instead of a tuple), defaulting to 0 (diagonal). Optionally, `m` and `n` specify the size of the resulting sparse matrix. """ diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 611e59dc097e0..fd6e168492e3e 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -957,7 +957,7 @@ Sparse matrices support much of the same set of operations as dense matrices. Th .. Docstring generated from Julia source - Construct a sparse diagonal matrix. ``B`` is a tuple of vectors containing the diagonals and ``d`` is a tuple containing the positions of the diagonals. In the case the input contains only one diagonaly, ``B`` can be a vector (instead of a tuple) and ``d`` can be the diagonal position (instead of a tuple), defaulting to 0 (diagonal). Optionally, ``m`` and ``n`` specify the size of the resulting sparse matrix. + Construct a sparse diagonal matrix. ``B`` is a tuple of vectors containing the diagonals and ``d`` is a tuple containing the positions of the diagonals. In the case the input contains only one diagonal, ``B`` can be a vector (instead of a tuple) and ``d`` can be the diagonal position (instead of a tuple), defaulting to 0 (diagonal). Optionally, ``m`` and ``n`` specify the size of the resulting sparse matrix. .. function:: sprand([rng,] m,n,p [,rfn]) From b31925cda1b25fbe486a07dd5b404e36fb1c5d11 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Wed, 30 Sep 2015 10:19:45 +0100 Subject: [PATCH 0375/1938] add trunc methods to BigInt, fixes #13367 --- base/gmp.jl | 16 +++++++++++++--- test/bigint.jl | 16 ++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/base/gmp.jl b/base/gmp.jl index b6f67fec91b0c..de70240fa21ac 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -8,7 +8,7 @@ import Base: *, +, -, /, <, <<, >>, >>>, <=, ==, >, >=, ^, (~), (&), (|), ($), binomial, cmp, convert, div, divrem, factorial, fld, gcd, gcdx, lcm, mod, ndigits, promote_rule, rem, show, isqrt, string, isprime, powermod, sum, trailing_zeros, trailing_ones, count_ones, base, tryparse_internal, - bin, oct, dec, hex, isequal, invmod, prevpow2, nextpow2, ndigits0z, widen, signed + bin, oct, dec, hex, isequal, invmod, prevpow2, nextpow2, ndigits0z, widen, signed, unsafe_trunc, trunc if Clong == Int32 typealias ClongMax Union{Int8, Int16, Int32} @@ -120,13 +120,23 @@ end convert(::Type{BigInt}, x::Bool) = BigInt(UInt(x)) -function convert(::Type{BigInt}, x::Float64) - !isinteger(x) && throw(InexactError()) + +function unsafe_trunc(::Type{BigInt}, x::CdoubleMax) z = BigInt() ccall((:__gmpz_set_d, :libgmp), Void, (Ptr{BigInt}, Cdouble), &z, x) return z end +function convert(::Type{BigInt}, x::CdoubleMax) + isinteger(x) || throw(InexactError()) + unsafe_trunc(BigInt,x) +end + +function trunc(::Type{BigInt}, x::CdoubleMax) + isfinite(x) || throw(InexactError()) + unsafe_trunc(BigInt,x) +end + convert(::Type{BigInt}, x::Float16) = BigInt(Float64(x)) convert(::Type{BigInt}, x::Float32) = BigInt(Float64(x)) diff --git a/test/bigint.jl b/test/bigint.jl index 50bb8bc3137e6..c2a11443c2d78 100644 --- a/test/bigint.jl +++ b/test/bigint.jl @@ -280,3 +280,19 @@ ndigits_mismatch(n) = ndigits(n) != ndigits(BigInt(n)) @test BigInt(2.0) == BigInt(2.0f0) == BigInt(big(2.0)) == 2 @test_throws InexactError convert(BigInt, 2.1) @test_throws InexactError convert(BigInt, big(2.1)) + +# issue #13367 +@test trunc(BigInt,2.1) == 2 +@test round(BigInt,2.1) == 2 +@test floor(BigInt,2.1) == 2 +@test ceil(BigInt,2.1) == 3 + +@test trunc(BigInt,2.1f0) == 2 +@test round(BigInt,2.1f0) == 2 +@test floor(BigInt,2.1f0) == 2 +@test ceil(BigInt,2.1f0) == 3 + +@test_throws InexactError trunc(BigInt,Inf) +@test_throws InexactError round(BigInt,Inf) +@test_throws InexactError floor(BigInt,Inf) +@test_throws InexactError ceil(BigInt,Inf) From 307953375055b7d6a95b25bc3016928fa4b47eaf Mon Sep 17 00:00:00 2001 From: Spencer Lyon Date: Wed, 30 Sep 2015 10:03:44 -0400 Subject: [PATCH 0376/1938] strip all MD markup when calling docsearch on `Markdown.MD` object --- base/docs/utils.jl | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/base/docs/utils.jl b/base/docs/utils.jl index 407de84d0bd7c..3df0009b0564c 100644 --- a/base/docs/utils.jl +++ b/base/docs/utils.jl @@ -337,21 +337,34 @@ function docsearch(haystack::FuncDoc, needle) false end -## Recursive Markdown search -docsearch(haystack::Markdown.BlockQuote, needle) = docsearch(haystack.content, needle) -docsearch(haystack::Markdown.Bold, needle) = docsearch(haystack.text, needle) -docsearch(haystack::Markdown.Code, needle) = docsearch(haystack.code, needle) -docsearch(haystack::Markdown.Header, needle) = docsearch(haystack.text, needle) -docsearch(haystack::Markdown.HorizontalRule, needle) = false -docsearch(haystack::Markdown.Image, needle) = docsearch(haystack.alt, needle) -docsearch(haystack::Markdown.Italic, needle) = docsearch(haystack.text, needle) -docsearch(haystack::Markdown.LaTeX, needle) = docsearch(haystack.formula, needle) -docsearch(haystack::Markdown.LineBreak, needle) = false -docsearch(haystack::Markdown.Link, needle) = docsearch(haystack.text, needle) # URL too? -docsearch(haystack::Markdown.List, needle) = docsearch(haystack.items, needle) -docsearch(haystack::Markdown.MD, needle) = docsearch(haystack.content, needle) -docsearch(haystack::Markdown.Paragraph, needle) = docsearch(haystack.content, needle) -docsearch(haystack::Markdown.Table, needle) = docsearch(haystack.rows, needle) +## Markdown search simply strips all markup and searches plain text version +docsearch(haystack::Markdown.MD, needle) = + docsearch(stripmd(haystack.content), needle) + +""" + stripmd(x) + +Strip all Markdown markup from x, leaving the result in plain text. Used +internally by apropos to make docstrings containing more than one markdown +element searchable. +""" +stripmd(x::AbstractString) = x # base case +stripmd(x::Vector) = string(map(stripmd, x)...) +stripmd(x::Markdown.BlockQuote) = "$(stripmd(x.content))" +stripmd(x::Markdown.Bold) = "$(stripmd(x.text))" +stripmd(x::Markdown.Code) = "$(stripmd(x.code))" +stripmd{N}(x::Markdown.Header{N}) = stripmd(x.text) +stripmd(x::Markdown.HorizontalRule) = " " +stripmd(x::Markdown.Image) = "$(stripmd(x.alt)) $(x.url)" +stripmd(x::Markdown.Italic) = "$(stripmd(x.text))" +stripmd(x::Markdown.LaTeX) = "$(x.formula)" +stripmd(x::Markdown.LineBreak) = " " +stripmd(x::Markdown.Link) = "$(stripmd(x.text)) $(x.url)" +stripmd(x::Markdown.List) = join(map(stripmd, x.items), " ") +stripmd(x::Markdown.MD) = join(map(stripmd, x.content), " ") +stripmd(x::Markdown.Paragraph) = stripmd(x.content) +stripmd(x::Markdown.Table) = + join([join(map(stripmd, r), " ") for r in x.rows], " ") # Apropos searches through all available documentation for some string or regex """ From 92670a905bacdf0510000f91358be1bd28a05668 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Wed, 30 Sep 2015 11:34:09 -0400 Subject: [PATCH 0377/1938] NEWS updates for 0.5 --- NEWS.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index ecf85b6cdba40..0cae4b2c39771 100644 --- a/NEWS.md +++ b/NEWS.md @@ -7,7 +7,7 @@ New language features Language changes ---------------- -Command line option changes +Command-line option changes --------------------------- Compiler/Runtime improvements @@ -16,11 +16,16 @@ Compiler/Runtime improvements Library improvements -------------------- -Deprecated or removed ---------------------- + * The package system (`Pkg`) is now based on the libgit2 library, rather than running the `git` program, increasing performance (especially on Windows) ([#11196]). + + * The `Base.Test` module now has a `@testset` feature to bundle + tests together and delay throwing an error until the end ([#13062]). * The function `remotecall`, `remotecall_fetch`, and `remotecall_wait` now have the - the function argument as the first argument to allow for do-block syntax. [#13338] + the function argument as the first argument to allow for do-block syntax ([#13338]). + +Deprecated or removed +--------------------- Julia v0.4.0 Release Notes ========================== @@ -1628,6 +1633,7 @@ Too numerous to mention. [#11105]: https://github.com/JuliaLang/julia/issues/11105 [#11145]: https://github.com/JuliaLang/julia/issues/11145 [#11171]: https://github.com/JuliaLang/julia/issues/11171 +[#11196]: https://github.com/JuliaLang/julia/issues/11196 [#11241]: https://github.com/JuliaLang/julia/issues/11241 [#11279]: https://github.com/JuliaLang/julia/issues/11279 [#11347]: https://github.com/JuliaLang/julia/issues/11347 @@ -1639,6 +1645,7 @@ Too numerous to mention. [#11849]: https://github.com/JuliaLang/julia/issues/11849 [#11891]: https://github.com/JuliaLang/julia/issues/11891 [#11922]: https://github.com/JuliaLang/julia/issues/11922 +[#11947]: https://github.com/JuliaLang/julia/issues/11947 [#11985]: https://github.com/JuliaLang/julia/issues/11985 [#12025]: https://github.com/JuliaLang/julia/issues/12025 [#12031]: https://github.com/JuliaLang/julia/issues/12031 @@ -1655,3 +1662,5 @@ Too numerous to mention. [#12576]: https://github.com/JuliaLang/julia/issues/12576 [#12727]: https://github.com/JuliaLang/julia/issues/12727 [#12739]: https://github.com/JuliaLang/julia/issues/12739 +[#13062]: https://github.com/JuliaLang/julia/issues/13062 +[#13338]: https://github.com/JuliaLang/julia/issues/13338 From 28e854bbc54978b810b91a7647b1fec1c6a36a9a Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Wed, 30 Sep 2015 13:37:06 -0400 Subject: [PATCH 0378/1938] give context to GitErrors when showing error --- base/libgit2/error.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/libgit2/error.jl b/base/libgit2/error.jl index 193849f9ac622..3fc226215c66e 100644 --- a/base/libgit2/error.jl +++ b/base/libgit2/error.jl @@ -65,7 +65,7 @@ immutable GitError <: Exception code::Code msg::AbstractString end -Base.show(io::IO, err::GitError) = print(io, "[Code:$(err.code), Class:$(err.class)]: $(err.msg)") +Base.show(io::IO, err::GitError) = print(io, "GitError(Code:$(err.code), Class:$(err.class), $(err.msg))") function last_error() err = ccall((:giterr_last, :libgit2), Ptr{ErrorStruct}, ()) From a507bcffac4a3d213a31b6dd6d9b65750f3d0140 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Wed, 30 Sep 2015 19:05:54 +0100 Subject: [PATCH 0379/1938] avoid overflow in histrange, fixes #13326 --- base/statistics.jl | 4 ++-- test/statistics.jl | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/base/statistics.jl b/base/statistics.jl index 5440db590d49a..3f031075fb7c7 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -607,8 +607,8 @@ function histrange{T<:Integer,N}(v::AbstractArray{T,N}, n::Integer) if hi == lo step = 1 else - bw = (hi - lo) / n - e = 10^max(0,floor(Int,log10(bw))) + bw = (Float64(hi) - Float64(lo)) / n + e = 10.0^max(0,floor(log10(bw))) r = bw / e if r <= 1 step = e diff --git a/test/statistics.jl b/test/statistics.jl index af65a4cb15eb7..122dfd15d0061 100644 --- a/test/statistics.jl +++ b/test/statistics.jl @@ -307,6 +307,11 @@ end @test histrange([1, 600], 4) == 0.0:200.0:600.0 @test histrange([1, -1000], 4) == -1500.0:500.0:500.0 +# issue #13326 +l,h = extrema(histrange([typemin(Int),typemax(Int)], 10)) +@test l <= typemin(Int) +@test h >= typemax(Int) + @test_throws ArgumentError histrange([1, 10], 0) @test_throws ArgumentError histrange([1, 10], -1) @test_throws ArgumentError histrange(Float64[],-1) From 5947f3cf074d136d2ade2c3f456a5b356aaee4b0 Mon Sep 17 00:00:00 2001 From: Michael Hatherly Date: Wed, 30 Sep 2015 21:26:00 +0200 Subject: [PATCH 0380/1938] Fix error in type summaries Types with fields containing unicode characters would throw an error since the initial type of the `parts` vector would be ASCII instead of utf8. --- base/docs/Docs.jl | 2 +- test/docs.jl | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index 63672a90941ad..9988ed016d7ff 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -326,7 +326,7 @@ end doc(f::Base.Callable, args::Any...) = doc(f, Tuple{args...}) function typesummary(T::DataType) - parts = [ + parts = UTF8String[ """ **Summary:** ```julia diff --git a/test/docs.jl b/test/docs.jl index 52b0b84ced72a..5bd3c47073d8d 100644 --- a/test/docs.jl +++ b/test/docs.jl @@ -318,6 +318,9 @@ end @test isdefined(MacroGenerated, :_g) +# Issue #13385. +@test @doc(I) !== nothing + # Issue #12700. @test @doc(DocsTest.@m) == doc"Inner.@m" From b3a8f8e84cef6f034bf6faa03bec8c10ca426009 Mon Sep 17 00:00:00 2001 From: wildart Date: Wed, 30 Sep 2015 18:22:32 -0400 Subject: [PATCH 0381/1938] fixed #13379: `transact` slowdown due to unnecessary file addition --- base/libgit2.jl | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/base/libgit2.jl b/base/libgit2.jl index be41143486607..1fa5ceffecfa6 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -444,12 +444,8 @@ function snapshot(repo::GitRepo) index = with(GitIndex, repo) do idx; write_tree!(idx) end work = try with(GitIndex, repo) do idx - content = readdir(path(repo)) - if length(content) > 1 - files = [utf8(bytestring(c))::UTF8String for c in content] - push!(files, utf8(".")) - - add!(idx, files...) + if length(readdir(path(repo))) > 1 + add!(idx, utf8(".")) write!(idx) end write_tree!(idx) From 72c88f66347c09a6f9b53692dd538698b1a9ae33 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sat, 26 Sep 2015 12:08:40 -0400 Subject: [PATCH 0382/1938] Disable FastISel on ARM. Fix #13321 --- src/codegen.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 747d7c46db633..af1a59ee66a00 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5998,7 +5998,8 @@ extern "C" void jl_init_codegen(void) ); delete targetMachine; assert(jl_TargetMachine); -#ifdef USE_MCJIT +#if defined(USE_MCJIT) && !defined(_CPU_ARM_) + // FastISel seems to be buggy for ARM. Ref #13321 jl_TargetMachine->setFastISel(true); #endif #if defined(LLVM38) From e95968d85a1a062711ca29a0e21af6ab35372a78 Mon Sep 17 00:00:00 2001 From: wildart Date: Wed, 30 Sep 2015 18:43:06 -0400 Subject: [PATCH 0383/1938] checks if folder was created before deleting that should solve exception inhibition caused by `rm` exception [#13390] --- base/pkg/entry.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 49a759403068a..b9de6dd87269a 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -197,7 +197,7 @@ function clone(url::AbstractString, pkg::AbstractString) LibGit2.set_remote_url(repo, url) end catch - Base.rm(pkg, recursive=true) + isdir(pkg) && Base.rm(pkg, recursive=true) rethrow() end info("Computing changes...") From f7b2f0a6cc38cecd8932e3580879ef5abeec3d9e Mon Sep 17 00:00:00 2001 From: wildart Date: Wed, 30 Sep 2015 21:18:04 -0400 Subject: [PATCH 0384/1938] remove unused var [fix #13394] --- base/libgit2/walker.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/libgit2/walker.jl b/base/libgit2/walker.jl index 45cd8742841ea..9bf2fa93529df 100644 --- a/base/libgit2/walker.jl +++ b/base/libgit2/walker.jl @@ -95,7 +95,7 @@ function Base.count(f::Function, walker::GitRevWalker; s = start(walker) repo = repository(walker) - while !done(walker, s) && val + while !done(walker, s) val = f(s[1], repo) _, s = next(walker, s) c += (val == true) From 1b443ff05c3a20200762c4905fbcd2fe39699c23 Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Wed, 30 Sep 2015 22:21:20 -0400 Subject: [PATCH 0385/1938] Cleanup triangular multiplication and solves and remove redundant copies --- base/linalg/triangular.jl | 52 ++++++++------------------------------- 1 file changed, 10 insertions(+), 42 deletions(-) diff --git a/base/linalg/triangular.jl b/base/linalg/triangular.jl index 507cc0a1bffb8..fe0a34cf95722 100644 --- a/base/linalg/triangular.jl +++ b/base/linalg/triangular.jl @@ -979,41 +979,25 @@ for (f, g) in ((:*, :A_mul_B!), (:Ac_mul_B, :Ac_mul_B!), (:At_mul_B, :At_mul_B!) @eval begin function ($f){TA,TB}(A::AbstractTriangular{TA}, B::StridedVecOrMat{TB}) TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + ($g)(convert(AbstractArray{TAB}, A), copy_oftype(B, TAB)) end end end ### Left division with triangle to the left hence rhs cannot be transposed. No quotients. for (f, g) in ((:\, :A_ldiv_B!), (:Ac_ldiv_B, :Ac_ldiv_B!), (:At_ldiv_B, :At_ldiv_B!)) @eval begin - function ($f){TA,TB,S}(A::UnitUpperTriangular{TA,S}, B::StridedVecOrMat{TB}) + function ($f){TA,TB,S}(A::AbstractTriangular{TA,S}, B::StridedVecOrMat{TB}) TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) - end - end -end -for (f, g) in ((:\, :A_ldiv_B!), (:Ac_ldiv_B, :Ac_ldiv_B!), (:At_ldiv_B, :At_ldiv_B!)) - @eval begin - function ($f){TA,TB,S}(A::UnitLowerTriangular{TA,S}, B::StridedVecOrMat{TB}) - TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + ($g)(convert(AbstractArray{TAB}, A), copy_oftype(B, TAB)) end end end ### Left division with triangle to the left hence rhs cannot be transposed. Quotients. for (f, g) in ((:\, :A_ldiv_B!), (:Ac_ldiv_B, :Ac_ldiv_B!), (:At_ldiv_B, :At_ldiv_B!)) @eval begin - function ($f){TA,TB,S}(A::UpperTriangular{TA,S}, B::StridedVecOrMat{TB}) - TAB = typeof((zero(TA)*zero(TB) + zero(TA)*zero(TB))/one(TA)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) - end - end -end -for (f, g) in ((:\, :A_ldiv_B!), (:Ac_ldiv_B, :Ac_ldiv_B!), (:At_ldiv_B, :At_ldiv_B!)) - @eval begin - function ($f){TA,TB,S}(A::LowerTriangular{TA,S}, B::StridedVecOrMat{TB}) + function ($f){TA,TB,S}(A::AbstractTriangular{TA,S}, B::StridedVecOrMat{TB}) TAB = typeof((zero(TA)*zero(TB) + zero(TA)*zero(TB))/one(TA)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + ($g)(convert(AbstractArray{TAB}, A), copy_oftype(B, TAB)) end end end @@ -1022,41 +1006,25 @@ for (f, g) in ((:*, :A_mul_B!), (:A_mul_Bc, :A_mul_Bc!), (:A_mul_Bt, :A_mul_Bt!) @eval begin function ($f){TA,TB}(A::StridedVecOrMat{TA}, B::AbstractTriangular{TB}) TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + ($g)(copy_oftype(A, TAB), convert(AbstractArray{TAB}, B)) end end end ### Right division with triangle to the right hence lhs cannot be transposed. No quotients. for (f, g) in ((:/, :A_rdiv_B!), (:A_rdiv_Bc, :A_rdiv_Bc!), (:A_rdiv_Bt, :A_rdiv_Bt!)) @eval begin - function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::UnitUpperTriangular{TB,S}) + function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::AbstractTriangular{TB,S}) TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) - end - end -end -for (f, g) in ((:/, :A_rdiv_B!), (:A_rdiv_Bc, :A_rdiv_Bc!), (:A_rdiv_Bt, :A_rdiv_Bt!)) - @eval begin - function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::UnitLowerTriangular{TB,S}) - TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + ($g)(copy_oftype(A, TAB), convert(AbstractArray{TAB}, B)) end end end ### Right division with triangle to the right hence lhs cannot be transposed. Quotients. for (f, g) in ((:/, :A_rdiv_B!), (:A_rdiv_Bc, :A_rdiv_Bc!), (:A_rdiv_Bt, :A_rdiv_Bt!)) @eval begin - function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::UpperTriangular{TB,S}) - TAB = typeof((zero(TA)*zero(TB) + zero(TA)*zero(TB))/one(TA)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) - end - end -end -for (f, g) in ((:/, :A_rdiv_B!), (:A_rdiv_Bc, :A_rdiv_Bc!), (:A_rdiv_Bt, :A_rdiv_Bt!)) - @eval begin - function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::LowerTriangular{TB,S}) + function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::AbstractTriangular{TB,S}) TAB = typeof((zero(TA)*zero(TB) + zero(TA)*zero(TB))/one(TA)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + ($g)(copy_oftype(A, TAB), convert(AbstractArray{TAB}, B)) end end end From 418a2626cfb466be0ed7a53d793bb4520b66f6ed Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Thu, 1 Oct 2015 12:28:00 +0530 Subject: [PATCH 0386/1938] Add links to the discussion on heuristics for sparse indexing. [ci skip] --- base/sparse/sparsematrix.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 8c3fb71ed5c38..e97da259bbc60 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1497,7 +1497,9 @@ function getindex_I_sorted{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, I::AbstractVector, nI = length(I) nzA = nnz(A) avgM = div(nzA,n) - # heuristics based on experiments + # Heuristics based on experiments discussed in: + # https://github.com/JuliaLang/julia/issues/12860 + # https://github.com/JuliaLang/julia/pull/12934 alg = ((m > nzA) && (m > nI)) ? 0 : ((nI - avgM) > 2^8) ? 1 : ((avgM - nI) > 2^10) ? 0 : 2 From 3f8d5a9aabfc76e445ce5d58875b4327c525454e Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Thu, 1 Oct 2015 13:21:14 +0530 Subject: [PATCH 0387/1938] Code organization cleanup. Sparse linear getindex code was in the wrong place, grouped with logical indexing. Now clubbed together in the right place. --- base/sparse/sparsematrix.jl | 85 ++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 43 deletions(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index e97da259bbc60..c29100678abd3 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1759,6 +1759,48 @@ function getindex{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, I::AbstractVector, J::Abstra end end +function getindex{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractArray) + szA = size(A) + nA = szA[1]*szA[2] + colptrA = A.colptr + rowvalA = A.rowval + nzvalA = A.nzval + + n = length(I) + outm = size(I,1) + outn = size(I,2) + szB = (outm, outn) + colptrB = zeros(Int, outn+1) + rowvalB = Array(Int, n) + nzvalB = Array(Tv, n) + + colB = 1 + rowB = 1 + colptrB[colB] = 1 + idxB = 1 + + for i in 1:n + ((I[i] < 1) | (I[i] > nA)) && throw(BoundsError()) + row,col = ind2sub(szA, I[i]) + for r in colptrA[col]:(colptrA[col+1]-1) + @inbounds if rowvalA[r] == row + rowB,colB = ind2sub(szB, i) + colptrB[colB+1] += 1 + rowvalB[idxB] = rowB + nzvalB[idxB] = nzvalA[r] + idxB += 1 + break + end + end + end + colptrB = cumsum(colptrB) + if n > (idxB-1) + deleteat!(nzvalB, idxB:n) + deleteat!(rowvalB, idxB:n) + end + SparseMatrixCSC(outm, outn, colptrB, rowvalB, nzvalB) +end + # logical getindex getindex{Tv,Ti<:Integer}(A::SparseMatrixCSC{Tv,Ti}, I::Range{Bool}, J::AbstractVector{Bool}) = error("Cannot index with Range{Bool}") getindex{Tv,Ti<:Integer,T<:Integer}(A::SparseMatrixCSC{Tv,Ti}, I::Range{Bool}, J::AbstractVector{T}) = error("Cannot index with Range{Bool}") @@ -1810,49 +1852,6 @@ function getindex{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractArray{Bool}) SparseMatrixCSC(n, 1, colptrB, rowvalB, nzvalB) end -function getindex{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractArray) - szA = size(A) - nA = szA[1]*szA[2] - colptrA = A.colptr - rowvalA = A.rowval - nzvalA = A.nzval - - n = length(I) - outm = size(I,1) - outn = size(I,2) - szB = (outm, outn) - colptrB = zeros(Int, outn+1) - rowvalB = Array(Int, n) - nzvalB = Array(Tv, n) - - colB = 1 - rowB = 1 - colptrB[colB] = 1 - idxB = 1 - - for i in 1:n - ((I[i] < 1) | (I[i] > nA)) && throw(BoundsError()) - row,col = ind2sub(szA, I[i]) - for r in colptrA[col]:(colptrA[col+1]-1) - @inbounds if rowvalA[r] == row - rowB,colB = ind2sub(szB, i) - colptrB[colB+1] += 1 - rowvalB[idxB] = rowB - nzvalB[idxB] = nzvalA[r] - idxB += 1 - break - end - end - end - colptrB = cumsum(colptrB) - if n > (idxB-1) - deleteat!(nzvalB, idxB:n) - deleteat!(rowvalB, idxB:n) - end - SparseMatrixCSC(outm, outn, colptrB, rowvalB, nzvalB) -end - - ## setindex! function setindex!{T,Ti}(A::SparseMatrixCSC{T,Ti}, v, i0::Integer, i1::Integer) i0 = convert(Ti, i0) From db9141d52a2fe8fb8da0dc9714cf2467a46c69b7 Mon Sep 17 00:00:00 2001 From: Scott Lundberg Date: Thu, 1 Oct 2015 07:13:13 -0700 Subject: [PATCH 0388/1938] GitHub no longer stores OAuth tokens Fixes Pkg.submit by removing code looking to fetch an existing token from GitHub. --- base/pkg/github.jl | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/base/pkg/github.jl b/base/pkg/github.jl index b4e9bd7bfec0b..69a58624b21e4 100644 --- a/base/pkg/github.jl +++ b/base/pkg/github.jl @@ -66,7 +66,8 @@ function token(user::AbstractString=user()) tok = strip(readchomp(tokfile)) !isempty(tok) && return tok end - status, header, content = curl("https://api.github.com/authorizations",AUTH_DATA,`-u $user`) + params = merge(AUTH_DATA, ["fingerprint" => randstring(40)]) + status, header, content = curl("https://api.github.com/authorizations",params,`-u $user`) tfa = false # Check for two-factor authentication @@ -75,33 +76,12 @@ function token(user::AbstractString=user()) info("Two-factor authentication in use. Enter auth code. (You may have to re-enter your password.)") print(STDERR, "Authentication code: ") code = readline(STDIN) |> chomp - status, header, content = curl("https://api.github.com/authorizations",AUTH_DATA,`-H "X-GitHub-OTP: $code" -u $user`) + status, header, content = curl("https://api.github.com/authorizations",params,`-H "X-GitHub-OTP: $code" -u $user`) end if status == 422 error_code = json().parse(content)["errors"][1]["code"] - if error_code == "already_exists" - if tfa - info("Retrieving existing GitHub token. (You may have to re-enter your password twice more.)") - status, header, content = curl("https://api.github.com/authorizations",AUTH_DATA,`-u $user`) - status != 401 && throw(PkgError("$status: $(json().parse(content)["message"])")) - print(STDERR, "New authentication code: ") - code = readline(STDIN) |> chomp - status, header, content = curl("https://api.github.com/authorizations",`-H "X-GitHub-OTP: $code" -u $user`) - else - info("Retrieving existing GitHub token. (You may have to re-enter your password.)") - status, header, content = curl("https://api.github.com/authorizations", `-u $user`) - end - (status >= 400) && throw(PkgError("$status: $(json().parse(content)["message"])")) - for entry in json().parse(content) - if entry["note"] == AUTH_NOTE - tok = entry["token"] - break - end - end - else - throw(PkgError("GitHub returned validation error (422): $error_code: $(json().parse(content)["message"])")) - end + throw(PkgError("GitHub returned validation error (422): $error_code: $(json().parse(content)["message"])")) else (status != 401 && status != 403) || throw(PkgError("$status: $(json().parse(content)["message"])")) tok = json().parse(content)["token"] From 66bf7be3942468a428ba3cfbea9ae16aba0cee49 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 1 Oct 2015 13:13:59 -0400 Subject: [PATCH 0389/1938] fix whitespace script to not get confused by glob expansion, and whitespace errors --- base/atomics.jl | 6 +++--- base/locks.jl | 6 +++--- base/threadingconstructs.jl | 6 +++--- base/threads.jl | 6 +++--- contrib/README.md | 2 +- contrib/check-whitespace.sh | 1 + 6 files changed, 14 insertions(+), 13 deletions(-) diff --git a/base/atomics.jl b/base/atomics.jl index 916326f407ee0..ce6abb4cd1449 100644 --- a/base/atomics.jl +++ b/base/atomics.jl @@ -1,8 +1,8 @@ # Copyright (c) 2015, Intel Corporation -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: -# +# # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright @@ -11,7 +11,7 @@ # * Neither the name of Intel Corporation nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE diff --git a/base/locks.jl b/base/locks.jl index 367bc14a0afe0..3765a4ebf3264 100644 --- a/base/locks.jl +++ b/base/locks.jl @@ -1,8 +1,8 @@ # Copyright (c) 2015, Intel Corporation -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: -# +# # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright @@ -11,7 +11,7 @@ # * Neither the name of Intel Corporation nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE diff --git a/base/threadingconstructs.jl b/base/threadingconstructs.jl index db6bb4fdc6221..3be1eea112d2a 100644 --- a/base/threadingconstructs.jl +++ b/base/threadingconstructs.jl @@ -1,8 +1,8 @@ # Copyright (c) 2015, Intel Corporation -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: -# +# # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright @@ -11,7 +11,7 @@ # * Neither the name of Intel Corporation nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE diff --git a/base/threads.jl b/base/threads.jl index 78f7f2a2095e4..d0a9e37b1c083 100644 --- a/base/threads.jl +++ b/base/threads.jl @@ -1,8 +1,8 @@ # Copyright (c) 2015, Intel Corporation -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: -# +# # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright @@ -11,7 +11,7 @@ # * Neither the name of Intel Corporation nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE diff --git a/contrib/README.md b/contrib/README.md index e6ebe13554168..912610b128332 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -5,7 +5,7 @@ Installation | ---------------------------- | --------------------------------------------------------- | |[ build_executable.jl ](https://github.com/JuliaLang/julia/blob/master/contrib/build_executable.jl) | Support building standalone executables | |[ build_sysimg.jl ](https://github.com/JuliaLang/julia/blob/master/contrib/build_sysimg.jl) | Build a system image binary | -|[ check-whitespace.sh ](https://github.com/JuliaLang/julia/blob/master/contrib/check-whitespace.sh) | Check for trailing white space | +|[ check-whitespace.sh ](https://github.com/JuliaLang/julia/blob/master/contrib/check-whitespace.sh) | Check for trailing white space | |[ fixup-libgfortran.sh ](https://github.com/JuliaLang/julia/blob/master/contrib/fixup-libgfortran.sh) | Include libgfortran and libquadmath for installations | |[ fixup-libstdc++.sh ](https://github.com/JuliaLang/julia/blob/master/contrib/fixup-libstdc++.sh) | Include libstdc++ for installations | |[ install.sh ](https://github.com/JuliaLang/julia/blob/master/contrib/install.sh) | installation script with different permissions | diff --git a/contrib/check-whitespace.sh b/contrib/check-whitespace.sh index ddbd3dda16bc8..cdb843c6fa876 100755 --- a/contrib/check-whitespace.sh +++ b/contrib/check-whitespace.sh @@ -5,6 +5,7 @@ # report an error if so # Files to check: +set -f # disable glob expansion in this script file_patterns=' *.1 *.c From 0921b942a60b97946f8782884a0f667efbbb1c80 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 1 Oct 2015 14:54:33 -0400 Subject: [PATCH 0390/1938] set a min target architecture of pentium4 to require support for sse2 and _mm_pause --- .travis.yml | 1 + Make.inc | 22 ++++++++++++++++------ Make.user | 6 ------ contrib/windows/msys_build.sh | 3 +-- src/codegen.cpp | 2 +- 5 files changed, 19 insertions(+), 15 deletions(-) delete mode 100644 Make.user diff --git a/.travis.yml b/.travis.yml index f1137de6d3c50..20c1212e75fd5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,7 @@ before_install: sudo add-apt-repository ppa:staticfloat/julia-deps -y; sudo apt-get update -qq -y; if [ "$ARCH" = "i686" ]; then + export BUILDOPTS="$BUILDOPTS MARCH=pentium4"; sudo apt-get remove libblas3gf liblapack3gf libarmadillo2 -y; sudo apt-get install binutils:i386 -y; sudo apt-get install gcc:i386 g++:i386 make:i386 cpp:i386 g++-4.6:i386 gcc-4.6:i386 libssl-dev:i386 patchelf:i386 gfortran:i386 llvm-3.3-dev:i386 libsuitesparse-dev:i386 libopenblas-dev:i386 libopenblas-base:i386 libblas-dev:i386 liblapack-dev:i386 liblapack3:i386 libarpack2-dev:i386 libarpack2:i386 libfftw3-dev:i386 libgmp-dev:i386 libpcre3-dev:i386 libunwind7-dev:i386 libopenlibm-dev:i386 libmpfr-dev:i386 -y; diff --git a/Make.inc b/Make.inc index f5451f042544e..262a92d9fedb9 100644 --- a/Make.inc +++ b/Make.inc @@ -378,6 +378,8 @@ DEBUGFLAGS := -O0 -g -DJL_DEBUG_BUILD -fstack-protector-all SHIPFLAGS := -O3 -g -falign-functions endif +JCPPFLAGS += -ftls-model=global-dynamic + ifeq ($(USECCACHE), 1) # expand CC and CXX at declaration time because we will redefine them CC_ARG := $(CC) # Expand CC and CXX here already because we want @@ -492,6 +494,8 @@ else ifneq ($(XC_HOST),) XC_HOST := $(ARCH)$(shell echo $(XC_HOST) | sed "s/[^-]*\(.*\)$$/\1/") MARCH := $(subst _,-,$(ARCH)) +else # insert ARCH into HOST +XC_HOST := $(ARCH)$(shell echo $(BUILD_MACHINE) | sed "s/[^-]*\(.*\)$$/\1/") endif endif @@ -523,6 +527,18 @@ ARCH := $(BUILD_OS) endif endif +# Detect common pre-SSE2 JULIA_CPU_TARGET values known not to work (#7185) +ifeq ($(MARCH),) +ifneq ($(findstring $(ARCH),i386 i486 i586 i686),) +MARCH := pentium4 +endif +endif + +ifneq ($(findstring $(MARCH),i386 i486 i586 i686 pentium pentium2 pentium3),) +$(error Pre-SSE2 CPU targets not supported. To create a generic 32-bit x86 binary, \ +pass 'MARCH=pentium4'.) +endif + ifneq ($(MARCH),) CC += -march=$(MARCH) CXX += -march=$(MARCH) @@ -546,12 +562,6 @@ endif JULIA_CPU_TARGET ?= native -# Detect common pre-SSE2 JULIA_CPU_TARGET values known not to work (#7185) -ifneq ($(findstring $(JULIA_CPU_TARGET),i386 i486 i586 i686 pentium pentium2 pentium3),) -$(error Pre-SSE2 CPU targets not supported. To create a generic 32-bit x86 binary, \ -pass 'MARCH=i686 JULIA_CPU_TARGET=pentium4', or 'MARCH=pentium4' if not building any dependencies.) -endif - # We map amd64 to x86_64 for compatibility with systems that identify 64-bit systems as such ifeq ($(ARCH),amd64) override ARCH := x86_64 diff --git a/Make.user b/Make.user deleted file mode 100644 index 861c40993919d..0000000000000 --- a/Make.user +++ /dev/null @@ -1,6 +0,0 @@ -JCPPFLAGS = -ftls-model=global-dynamic -LLVM_VER = svn -LLVM_GIT_URL_LLVM = https://github.com/JuliaLang/llvm.git -b jn/tlsrebaseagainagain - -LLVM_DEBUG = 0 -LLVM_ASSERTIONS = 1 diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index 27caae65a75a7..2ced571769363 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -60,8 +60,7 @@ else bits=32 archsuffix=86 exc=sjlj - echo "override MARCH = i686" >> Make.user - echo "override JULIA_CPU_TARGET = pentium4" >> Make.user + echo "override MARCH = pentium4" >> Make.user fi # Set XC_HOST if in Cygwin or Linux diff --git a/src/codegen.cpp b/src/codegen.cpp index ca57e27f76091..035417cae69c6 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5963,7 +5963,7 @@ extern "C" void jl_init_codegen(void) .setMCJITMemoryManager(std::move(std::unique_ptr{new SectionMemoryManager()})) #endif .setTargetOptions(options) - .setRelocationModel(Reloc::PIC_) + .setRelocationModel(Reloc::Default) .setCodeModel(CodeModel::JITDefault) #ifdef DISABLE_OPT .setOptLevel(CodeGenOpt::None) From f68130cbadf59505efcc38f65fef928939f99206 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 1 Oct 2015 15:04:05 -0400 Subject: [PATCH 0391/1938] Uint => UInt deprecation --- base/atomics.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/base/atomics.jl b/base/atomics.jl index ce6abb4cd1449..d09d74b28e05c 100644 --- a/base/atomics.jl +++ b/base/atomics.jl @@ -43,11 +43,11 @@ end Atomic() = Atomic{Int}() -atomicintsmap = Dict(Int8 => "i8", Uint8 => "i8", - Int16 => "i16", Uint16 => "i16", - Int32 => "i32", Uint32 => "i32", - Int64 => "i64", Uint64 => "i64", - Int128 => "i128", Uint128 => "i128") +atomicintsmap = Dict(Int8 => "i8", UInt8 => "i8", + Int16 => "i16", UInt16 => "i16", + Int32 => "i32", UInt32 => "i32", + Int64 => "i64", UInt64 => "i64", + Int128 => "i128", UInt128 => "i128") unsafe_convert{T}(::Type{Ptr{T}}, x::Atomic{T}) = convert(Ptr{T}, pointer_from_objref(x)) setindex!{T}(x::Atomic{T}, v) = setindex!(x, convert(T, v)) From 5203429a29be968993a1abd9226f942fbb84af7d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 1 Oct 2015 15:28:14 -0400 Subject: [PATCH 0392/1938] get threads test passing without threading support --- src/threading.c | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/src/threading.c b/src/threading.c index d44bde638d6ad..0578bc4463dd2 100644 --- a/src/threading.c +++ b/src/threading.c @@ -86,6 +86,19 @@ void ti_initthread(int16_t tid) jl_all_task_states[tid].ptask_arg_in_transit = &jl_task_arg_in_transit; } +// all threads call this function to run user code +static jl_value_t *ti_run_fun(jl_function_t *f, jl_svec_t *args) +{ + JL_TRY { + jl_apply(f, jl_svec_data(args), jl_svec_len(args)); + } + JL_CATCH { + return jl_exception_in_transit; + } + return jl_nothing; +} + + #ifdef JULIA_ENABLE_THREADING // lock for code generation @@ -142,18 +155,6 @@ void ti_threadsetaffinity(uint64_t thread_id, int proc_num) #endif } -// all threads call this function to run user code -static jl_value_t *ti_run_fun(jl_function_t *f, jl_svec_t *args) -{ - JL_TRY { - jl_apply(f, jl_svec_data(args), jl_svec_len(args)); - } - JL_CATCH { - return jl_exception_in_transit; - } - return jl_nothing; -} - // thread function: used by all except the main thread void ti_threadfun(void *arg) { @@ -347,7 +348,9 @@ DLLEXPORT jl_value_t *jl_threading_run(jl_function_t *f, jl_svec_t *args) jl_function_t *fun = NULL; if ((jl_value_t*)args == jl_emptytuple) args = jl_emptysvec; - assert(jl_is_svec(args)); + JL_TYPECHK(jl_threading_run, function, (jl_value_t*)f); + JL_TYPECHK(jl_threading_run, svec, (jl_value_t*)args); + JL_GC_PUSH2(&argtypes, &fun); if (jl_svec_len(args) == 0) argtypes = (jl_tupletype_t*)jl_typeof(jl_emptytuple); @@ -454,6 +457,15 @@ void jl_threading_profile() #else // !JULIA_ENABLE_THREADING +DLLEXPORT jl_value_t *jl_threading_run(jl_function_t *f, jl_svec_t *args) +{ + if ((jl_value_t*)args == jl_emptytuple) + args = jl_emptysvec; + JL_TYPECHK(jl_threading_run, function, (jl_value_t*)f); + JL_TYPECHK(jl_threading_run, simplevector, (jl_value_t*)args); + return ti_run_fun(f, args); +} + void jl_init_threading(void) { static jl_thread_task_state_t _jl_all_task_states; From b0bc951f301ebeaeede9faf6b7199ffdfcb09bc2 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Thu, 1 Oct 2015 15:53:18 -0400 Subject: [PATCH 0393/1938] as discussed in #8734, since we rename the openblas symbols we also need to rename the library to avoid conflicts --- Make.inc | 4 ++++ deps/Makefile | 12 ++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Make.inc b/Make.inc index c571d0d61641a..4bfdfbc2484f6 100644 --- a/Make.inc +++ b/Make.inc @@ -8,6 +8,7 @@ # OPENBLAS build options OPENBLAS_TARGET_ARCH:= OPENBLAS_SYMBOLSUFFIX:= +OPENBLAS_LIBNAMESUFFIX:= # If OPENBLAS_TARGET_ARCH is set, we default to disabling OPENBLAS_DYNAMIC_ARCH ifneq ($(OPENBLAS_TARGET_ARCH),) @@ -914,6 +915,9 @@ endif ifeq ($(USE_SYSTEM_BLAS), 0) ifeq ($(USE_BLAS64), 1) OPENBLAS_SYMBOLSUFFIX := 64_ +OPENBLAS_LIBNAMESUFFIX := 64_ +LIBBLASNAME := $(LIBBLASNAME)$(OPENBLAS_LIBNAMESUFFIX) +LIBLAPACKNAME := $(LIBBLASNAME) endif endif diff --git a/deps/Makefile b/deps/Makefile index 319de7803d3de..c00fedf06576f 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -1059,10 +1059,10 @@ install-objconv: $(OBJCONV_TARGET) # LAPACK is built into OpenBLAS by default OPENBLAS_GIT_URL := git://github.com/xianyi/OpenBLAS.git OPENBLAS_TAR_URL = https://api.github.com/repos/xianyi/OpenBLAS/tarball/$1 -$(eval $(call git-external,openblas,OPENBLAS,Makefile,libopenblas.$(SHLIB_EXT),$(BUILDDIR))) +$(eval $(call git-external,openblas,OPENBLAS,Makefile,$(LIBBLASNAME).$(SHLIB_EXT),$(BUILDDIR))) -OPENBLAS_OBJ_SOURCE := $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/libopenblas.$(SHLIB_EXT) -OPENBLAS_OBJ_TARGET := $(build_shlibdir)/libopenblas.$(SHLIB_EXT) +OPENBLAS_OBJ_SOURCE := $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/$(LIBBLASNAME).$(SHLIB_EXT) +OPENBLAS_OBJ_TARGET := $(build_shlibdir)/$(LIBBLASNAME).$(SHLIB_EXT) OPENBLAS_BUILD_OPTS := CC="$(CC)" FC="$(FC)" RANLIB="$(RANLIB)" FFLAGS="$(FFLAGS) $(JFFLAGS)" TARGET=$(OPENBLAS_TARGET_ARCH) BINARY=$(BINARY) # Thread support @@ -1098,7 +1098,7 @@ endif # 64-bit BLAS interface ifeq ($(USE_BLAS64), 1) -OPENBLAS_BUILD_OPTS += INTERFACE64=1 SYMBOLSUFFIX="$(OPENBLAS_SYMBOLSUFFIX)" +OPENBLAS_BUILD_OPTS += INTERFACE64=1 SYMBOLSUFFIX="$(OPENBLAS_SYMBOLSUFFIX)" LIBPREFIX="$(LIBBLASNAME)" ifeq ($(OS), Darwin) OPENBLAS_BUILD_OPTS += OBJCONV=$(abspath $(BUILDDIR)/objconv/objconv) $(OPENBLAS_OBJ_SOURCE): $(OBJCONV_SOURCE) @@ -1139,9 +1139,9 @@ $(OPENBLAS_OBJ_TARGET): $(OPENBLAS_OBJ_SOURCE) | $(build_shlibdir) cp -f $< $@ ifeq ($(OS), Linux) cd $(dir $@) && \ - ln -sf libopenblas.$(SHLIB_EXT) libopenblas.$(SHLIB_EXT).0 + ln -sf $(LIBBLASNAME).$(SHLIB_EXT) $(LIBBLASNAME).$(SHLIB_EXT).0 endif - $(INSTALL_NAME_CMD)libopenblas.$(SHLIB_EXT) $@ + $(INSTALL_NAME_CMD)$(LIBBLASNAME).$(SHLIB_EXT) $@ clean-openblas: -$(MAKE) -C $(BUILDDIR)/$(OPENBLAS_SRC_DIR) clean From 74f906f2b064ad07f333e4611d6a9d75886e09c3 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 1 Oct 2015 17:53:23 -0400 Subject: [PATCH 0394/1938] add support for llvm < v3.6 to atomics --- base/atomics.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/atomics.jl b/base/atomics.jl index d09d74b28e05c..78b3095d89b9b 100644 --- a/base/atomics.jl +++ b/base/atomics.jl @@ -53,9 +53,10 @@ unsafe_convert{T}(::Type{Ptr{T}}, x::Atomic{T}) = convert(Ptr{T}, pointer_from_o setindex!{T}(x::Atomic{T}, v) = setindex!(x, convert(T, v)) for (typ, lt) in atomicintsmap + rt = VersionNumber(Base.libllvm_version) >= v"3.6" ? "$lt, $lt*" : "$lt*" @eval getindex(x::Atomic{$typ}) = llvmcall($""" - %rv = load atomic volatile $lt, $lt* %0 monotonic, align $WORD_SIZE + %rv = load atomic volatile $rt %0 monotonic, align $WORD_SIZE ret $lt %rv """, $typ, Tuple{Ptr{$typ}}, unsafe_convert(Ptr{$typ}, x)) @eval setindex!(x::Atomic{$typ}, v::$typ) = From 49d582b5eefcbc770d834617a9c10183ecab3e33 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 1 Oct 2015 18:26:11 -0400 Subject: [PATCH 0395/1938] correct size of uv_mutex on windows (& everywhere) --- base/locks.jl | 7 ++----- src/sys.c | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/base/locks.jl b/base/locks.jl index 3765a4ebf3264..5f0d8dec168fa 100644 --- a/base/locks.jl +++ b/base/locks.jl @@ -23,8 +23,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -include("uv_constants.jl") - export SpinLock, Mutex, init_lock!, destroy_lock!, lock!, trylock!, unlock! abstract Lock @@ -121,8 +119,7 @@ end # TODO: how defensive to get, and how to turn it off? # TODO: how to catch an abort? -# TODO: this size is tested correct for pthreads on Linux and Darwin only -const UV_MUTEX_SIZE = 48 +const UV_MUTEX_SIZE = ccall(:jl_sizeof_uv_mutex, Cint, ()) type Mutex <: Lock ownertid::Int16 @@ -155,7 +152,7 @@ end function unlock!(m::Mutex) if m.ownertid != threadid() - return UV_EPERM + return Base.UV_EPERM end m.ownertid = 0 ccall(:uv_mutex_unlock, Void, (Ptr{Void},), m.handle) diff --git a/src/sys.c b/src/sys.c index 4d88301b8aed3..1aa0cd1725611 100644 --- a/src/sys.c +++ b/src/sys.c @@ -66,6 +66,7 @@ DLLEXPORT uint32_t jl_getutf8(ios_t *s) return wc; } +DLLEXPORT int jl_sizeof_uv_mutex(void) { return sizeof(uv_mutex_t); } DLLEXPORT int jl_sizeof_off_t(void) { return sizeof(off_t); } #ifndef _OS_WINDOWS_ DLLEXPORT off_t jl_lseek(int fd, off_t offset, int whence) { return lseek(fd, offset, whence); } From 26b58c6138b3e8e704ba35762e79b982b4e69b33 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Thu, 1 Oct 2015 18:22:38 -0700 Subject: [PATCH 0396/1938] Fix linking to renamed openblas, ref #13407 --- Make.inc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Make.inc b/Make.inc index 4bfdfbc2484f6..610ec5d91df3e 100644 --- a/Make.inc +++ b/Make.inc @@ -916,6 +916,8 @@ ifeq ($(USE_SYSTEM_BLAS), 0) ifeq ($(USE_BLAS64), 1) OPENBLAS_SYMBOLSUFFIX := 64_ OPENBLAS_LIBNAMESUFFIX := 64_ +LIBBLAS := -L$(build_shlibdir) -lopenblas$(OPENBLAS_LIBNAMESUFFIX) +LIBLAPACK := $(LIBBLAS) LIBBLASNAME := $(LIBBLASNAME)$(OPENBLAS_LIBNAMESUFFIX) LIBLAPACKNAME := $(LIBBLASNAME) endif From b9848ddefe56c0659838f6632b8c85fcd62ba605 Mon Sep 17 00:00:00 2001 From: ggggggggg Date: Thu, 1 Oct 2015 16:50:42 -0600 Subject: [PATCH 0397/1938] document 2nd argument of hash is UInt --- base/docs/helpdb.jl | 2 +- doc/stdlib/base.rst | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 7b955106ee92a..aecf883f77206 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -6603,7 +6603,7 @@ Compute the cosecant of `x`, where `x` is in radians csc doc""" - hash(x[, h]) + hash(x[, h::UInt]) Compute an integer hash code such that `isequal(x,y)` implies `hash(x)==hash(y)`. The optional second argument `h` is a hash code to be mixed with the result. diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index ae5bb7e30b031..31529bcda680d 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -306,7 +306,7 @@ All Objects Get a unique integer id for ``x``\ . ``object_id(x)==object_id(y)`` if and only if ``is(x,y)``\ . -.. function:: hash(x[, h]) +.. function:: hash(x[, h::UInt]) .. Docstring generated from Julia source @@ -1384,4 +1384,3 @@ Internals .. Docstring generated from Julia source Compile the given function ``f`` for the argument tuple (of types) ``args``\ , but do not execute it. - From 0e5cc1659a8cbf5d76f26f0d50a12b5428a667fd Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Thu, 1 Oct 2015 23:35:18 -0400 Subject: [PATCH 0398/1938] Make methods(::Symbol) an error, fixes #13398 --- base/reflection.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/reflection.jl b/base/reflection.jl index 26a0deb1a6239..feda3d695bdf3 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -182,6 +182,7 @@ function methods(f::Function) f.env end +methods(x::Symbol) = error("Symbol (:$x) is not a function or type") methods(x::ANY) = methods(call, Tuple{isa(x,Type) ? Type{x} : typeof(x), Vararg{Any}}) function length(mt::MethodTable) From ceb3b9d6b8cd0205202bb74cf32fe405a399f89a Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Thu, 1 Oct 2015 23:41:17 -0400 Subject: [PATCH 0399/1938] Improve error message for undef ccall return type --- src/ccall.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/ccall.cpp b/src/ccall.cpp index da7d4a86418c7..4866639609b44 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -998,6 +998,14 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) static_rt = true; } } + if (jl_exception_in_transit && jl_typeis(jl_exception_in_transit, + jl_undefvarerror_type)) { + std::string msg = "ccall return type undefined: " + + std::string(((jl_sym_t*)args[2])->name ); + emit_error(msg.c_str(), ctx); + JL_GC_POP(); + return jl_cgval_t(); + } if (rt == NULL) { emit_error("error interpreting ccall return type", ctx); JL_GC_POP(); From 52d940e24b7268d3114d15cbc4e4320fc3201c9e Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Fri, 2 Oct 2015 01:48:42 -0400 Subject: [PATCH 0400/1938] Fix indentation in pkg and replcompletions tests --- test/pkg.jl | 240 ++++++++++++++++++++-------------------- test/replcompletions.jl | 20 ++-- 2 files changed, 130 insertions(+), 130 deletions(-) diff --git a/test/pkg.jl b/test/pkg.jl index 794442d4e9e3c..e2f74d8ca86d4 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -1,128 +1,128 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license function temp_pkg_dir(fn::Function) - # Used in tests below to setup and teardown a sandboxed package directory - const tmpdir = ENV["JULIA_PKGDIR"] = joinpath(tempdir(),randstring()) - @test !isdir(Pkg.dir()) - try - Pkg.init() - @test isdir(Pkg.dir()) - Pkg.resolve() - - fn() - finally - rm(tmpdir, recursive=true) - end + # Used in tests below to setup and teardown a sandboxed package directory + const tmpdir = ENV["JULIA_PKGDIR"] = joinpath(tempdir(),randstring()) + @test !isdir(Pkg.dir()) + try + Pkg.init() + @test isdir(Pkg.dir()) + Pkg.resolve() + + fn() + finally + rm(tmpdir, recursive=true) + end end # Test basic operations: adding or removing a package, status, free #Also test for the existence of REQUIRE and META_Branch temp_pkg_dir() do - @test isfile(joinpath(Pkg.dir(),"REQUIRE")) - @test isfile(joinpath(Pkg.dir(),"META_BRANCH")) - @test isempty(Pkg.installed()) - Pkg.add("Example") - @test [keys(Pkg.installed())...] == ["Example"] - iob = IOBuffer() - Pkg.checkout("Example") - Pkg.status("Example", iob) - str = chomp(takebuf_string(iob)) - @test startswith(str, " - Example") - @test endswith(str, "master") - Pkg.free("Example") - Pkg.status("Example", iob) - str = chomp(takebuf_string(iob)) - @test endswith(str, string(Pkg.installed("Example"))) - Pkg.checkout("Example") - Pkg.free(("Example",)) - Pkg.status("Example", iob) - str = chomp(takebuf_string(iob)) - @test endswith(str, string(Pkg.installed("Example"))) - Pkg.rm("Example") - @test isempty(Pkg.installed()) - @test !isempty(Pkg.available("Example")) - @test Pkg.available("IDoNotExist") === nothing - Pkg.clone("https://github.com/JuliaLang/Example.jl.git") - @test [keys(Pkg.installed())...] == ["Example"] - Pkg.status("Example", iob) - str = chomp(takebuf_string(iob)) - @test startswith(str, " - Example") - @test endswith(str, "master") - Pkg.free("Example") - Pkg.status("Example", iob) - str = chomp(takebuf_string(iob)) - @test endswith(str, string(Pkg.installed("Example"))) - Pkg.checkout("Example") - Pkg.free(("Example",)) - Pkg.status("Example", iob) - str = chomp(takebuf_string(iob)) - @test endswith(str, string(Pkg.installed("Example"))) - Pkg.rm("Example") - @test isempty(Pkg.installed()) + @test isfile(joinpath(Pkg.dir(),"REQUIRE")) + @test isfile(joinpath(Pkg.dir(),"META_BRANCH")) + @test isempty(Pkg.installed()) + Pkg.add("Example") + @test [keys(Pkg.installed())...] == ["Example"] + iob = IOBuffer() + Pkg.checkout("Example") + Pkg.status("Example", iob) + str = chomp(takebuf_string(iob)) + @test startswith(str, " - Example") + @test endswith(str, "master") + Pkg.free("Example") + Pkg.status("Example", iob) + str = chomp(takebuf_string(iob)) + @test endswith(str, string(Pkg.installed("Example"))) + Pkg.checkout("Example") + Pkg.free(("Example",)) + Pkg.status("Example", iob) + str = chomp(takebuf_string(iob)) + @test endswith(str, string(Pkg.installed("Example"))) + Pkg.rm("Example") + @test isempty(Pkg.installed()) + @test !isempty(Pkg.available("Example")) + @test Pkg.available("IDoNotExist") === nothing + Pkg.clone("https://github.com/JuliaLang/Example.jl.git") + @test [keys(Pkg.installed())...] == ["Example"] + Pkg.status("Example", iob) + str = chomp(takebuf_string(iob)) + @test startswith(str, " - Example") + @test endswith(str, "master") + Pkg.free("Example") + Pkg.status("Example", iob) + str = chomp(takebuf_string(iob)) + @test endswith(str, string(Pkg.installed("Example"))) + Pkg.checkout("Example") + Pkg.free(("Example",)) + Pkg.status("Example", iob) + str = chomp(takebuf_string(iob)) + @test endswith(str, string(Pkg.installed("Example"))) + Pkg.rm("Example") + @test isempty(Pkg.installed()) end # testing a package with test dependencies causes them to be installed for the duration of the test temp_pkg_dir() do - Pkg.generate("PackageWithTestDependencies", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) - @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] - @test readall(Pkg.dir("PackageWithTestDependencies","REQUIRE")) == "julia $(Pkg.Generate.versionfloor(VERSION))\n" + Pkg.generate("PackageWithTestDependencies", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) + @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] + @test readall(Pkg.dir("PackageWithTestDependencies","REQUIRE")) == "julia $(Pkg.Generate.versionfloor(VERSION))\n" - isdir(Pkg.dir("PackageWithTestDependencies","test")) || mkdir(Pkg.dir("PackageWithTestDependencies","test")) - open(Pkg.dir("PackageWithTestDependencies","test","REQUIRE"),"w") do f - println(f,"Example") - end + isdir(Pkg.dir("PackageWithTestDependencies","test")) || mkdir(Pkg.dir("PackageWithTestDependencies","test")) + open(Pkg.dir("PackageWithTestDependencies","test","REQUIRE"),"w") do f + println(f,"Example") + end - open(Pkg.dir("PackageWithTestDependencies","test","runtests.jl"),"w") do f - println(f,"using Base.Test") - println(f,"@test haskey(Pkg.installed(), \"Example\")") - end + open(Pkg.dir("PackageWithTestDependencies","test","runtests.jl"),"w") do f + println(f,"using Base.Test") + println(f,"@test haskey(Pkg.installed(), \"Example\")") + end - Pkg.resolve() - @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] + Pkg.resolve() + @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] - Pkg.test("PackageWithTestDependencies") + Pkg.test("PackageWithTestDependencies") - @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] + @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] end # testing a package with no runtests.jl errors temp_pkg_dir() do - Pkg.generate("PackageWithNoTests", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) + Pkg.generate("PackageWithNoTests", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) - if isfile(Pkg.dir("PackageWithNoTests", "test", "runtests.jl")) - rm(Pkg.dir("PackageWithNoTests", "test", "runtests.jl")) - end + if isfile(Pkg.dir("PackageWithNoTests", "test", "runtests.jl")) + rm(Pkg.dir("PackageWithNoTests", "test", "runtests.jl")) + end - try - Pkg.test("PackageWithNoTests") - catch err - @test err.msg == "PackageWithNoTests did not provide a test/runtests.jl file" - end + try + Pkg.test("PackageWithNoTests") + catch err + @test err.msg == "PackageWithNoTests did not provide a test/runtests.jl file" + end end # testing a package with failing tests errors temp_pkg_dir() do - Pkg.generate("PackageWithFailingTests", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) - - isdir(Pkg.dir("PackageWithFailingTests","test")) || mkdir(Pkg.dir("PackageWithFailingTests","test")) - open(Pkg.dir("PackageWithFailingTests", "test", "runtests.jl"),"w") do f - println(f,"using Base.Test") - println(f,"@test false") - end - - try - Pkg.test("PackageWithFailingTests") - catch err - @test err.msg == "PackageWithFailingTests had test errors" - end + Pkg.generate("PackageWithFailingTests", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) + + isdir(Pkg.dir("PackageWithFailingTests","test")) || mkdir(Pkg.dir("PackageWithFailingTests","test")) + open(Pkg.dir("PackageWithFailingTests", "test", "runtests.jl"),"w") do f + println(f,"using Base.Test") + println(f,"@test false") + end + + try + Pkg.test("PackageWithFailingTests") + catch err + @test err.msg == "PackageWithFailingTests had test errors" + end end # Testing with code-coverage temp_pkg_dir() do - Pkg.generate("PackageWithCodeCoverage", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) + Pkg.generate("PackageWithCodeCoverage", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) - src = """ + src = """ module PackageWithCodeCoverage export f1, f2, f3, untested @@ -135,34 +135,34 @@ end untested(x) = 7 end""" - linetested = [false, false, false, false, true, true, false, true, false, false] - open(Pkg.dir("PackageWithCodeCoverage", "src", "PackageWithCodeCoverage.jl"), "w") do f - println(f, src) - end - isdir(Pkg.dir("PackageWithCodeCoverage","test")) || mkdir(Pkg.dir("PackageWithCodeCoverage","test")) - open(Pkg.dir("PackageWithCodeCoverage", "test", "runtests.jl"),"w") do f - println(f,"using PackageWithCodeCoverage, Base.Test") - println(f,"@test f2(2) == 4") - println(f,"@test f3(5) == 15") - end - - Pkg.test("PackageWithCodeCoverage") - covdir = Pkg.dir("PackageWithCodeCoverage","src") - covfiles = filter!(x -> contains(x, "PackageWithCodeCoverage.jl") && contains(x,".cov"), readdir(covdir)) - @test isempty(covfiles) - Pkg.test("PackageWithCodeCoverage", coverage=true) - covfiles = filter!(x -> contains(x, "PackageWithCodeCoverage.jl") && contains(x,".cov"), readdir(covdir)) - @test !isempty(covfiles) - for file in covfiles - @test isfile(joinpath(covdir,file)) - covstr = readall(joinpath(covdir,file)) - srclines = split(src, '\n') - covlines = split(covstr, '\n') - for i = 1:length(linetested) - covline = (linetested[i] ? " 1 " : " - ")*srclines[i] - @test covlines[i] == covline - end - end + linetested = [false, false, false, false, true, true, false, true, false, false] + open(Pkg.dir("PackageWithCodeCoverage", "src", "PackageWithCodeCoverage.jl"), "w") do f + println(f, src) + end + isdir(Pkg.dir("PackageWithCodeCoverage","test")) || mkdir(Pkg.dir("PackageWithCodeCoverage","test")) + open(Pkg.dir("PackageWithCodeCoverage", "test", "runtests.jl"),"w") do f + println(f,"using PackageWithCodeCoverage, Base.Test") + println(f,"@test f2(2) == 4") + println(f,"@test f3(5) == 15") + end + + Pkg.test("PackageWithCodeCoverage") + covdir = Pkg.dir("PackageWithCodeCoverage","src") + covfiles = filter!(x -> contains(x, "PackageWithCodeCoverage.jl") && contains(x,".cov"), readdir(covdir)) + @test isempty(covfiles) + Pkg.test("PackageWithCodeCoverage", coverage=true) + covfiles = filter!(x -> contains(x, "PackageWithCodeCoverage.jl") && contains(x,".cov"), readdir(covdir)) + @test !isempty(covfiles) + for file in covfiles + @test isfile(joinpath(covdir,file)) + covstr = readall(joinpath(covdir,file)) + srclines = split(src, '\n') + covlines = split(covstr, '\n') + for i = 1:length(linetested) + covline = (linetested[i] ? " 1 " : " - ")*srclines[i] + @test covlines[i] == covline + end + end end # issue #13373 diff --git a/test/replcompletions.jl b/test/replcompletions.jl index 4a0e2bb060526..7e9f55e7ae305 100644 --- a/test/replcompletions.jl +++ b/test/replcompletions.jl @@ -37,16 +37,16 @@ module CompletionFoo end function temp_pkg_dir(fn::Function) - # Used in tests below to setup and teardown a sandboxed package directory - const tmpdir = ENV["JULIA_PKGDIR"] = joinpath(tempdir(),randstring()) - @test !isdir(Pkg.dir()) - try - mkpath(Pkg.dir()) - @test isdir(Pkg.dir()) - fn() - finally - rm(tmpdir, recursive=true) - end + # Used in tests below to setup and teardown a sandboxed package directory + const tmpdir = ENV["JULIA_PKGDIR"] = joinpath(tempdir(),randstring()) + @test !isdir(Pkg.dir()) + try + mkpath(Pkg.dir()) + @test isdir(Pkg.dir()) + fn() + finally + rm(tmpdir, recursive=true) + end end test_complete(s) = completions(s,endof(s)) From bcd744dc7aa1be3f1c59ffe44727b3715034fa88 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Fri, 2 Oct 2015 02:38:04 -0400 Subject: [PATCH 0401/1938] Add error("unexpected") in try-catches that should fail --- test/backtrace.jl | 2 ++ test/misc.jl | 1 + test/parallel.jl | 2 ++ test/unicode/utf32.jl | 2 +- 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test/backtrace.jl b/test/backtrace.jl index c33bcd65ad8d3..19c0f4f17eeeb 100644 --- a/test/backtrace.jl +++ b/test/backtrace.jl @@ -39,6 +39,7 @@ eval(Expr(:function, Expr(:call, :test_inline_2), try eval(:(test_inline_1())) + error("unexpected") catch err lkup = get_bt_frame(:test_inline_1, catch_backtrace()) if is(lkup, nothing) @@ -51,6 +52,7 @@ catch err end try eval(:(test_inline_2())) + error("unexpected") catch err lkup = get_bt_frame(:test_inline_2, catch_backtrace()) if is(lkup, nothing) diff --git a/test/misc.jl b/test/misc.jl index b605fd6f2f1cd..2c76e387ace96 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -94,6 +94,7 @@ let deepthought(x, y) = 42 try @assert 1 == 2 string("the answer to the ultimate question: ", deepthought(6, 9)) + error("unexpected") catch ex @test isa(ex, AssertionError) @test ex.msg == "the answer to the ultimate question: 42" diff --git a/test/parallel.jl b/test/parallel.jl index 8d5d14cfa9b1a..dc4a01a8e58c8 100644 --- a/test/parallel.jl +++ b/test/parallel.jl @@ -383,6 +383,7 @@ try @async error(i) end end + error("unexpected") catch ex @test typeof(ex) == CompositeException @test length(ex) == 5 @@ -394,6 +395,7 @@ end try remotecall_fetch(()->throw(ErrorException("foobar")), id_other) + error("unexpected") catch ex @test typeof(ex) == RemoteException @test typeof(ex.captured) == CapturedException diff --git a/test/unicode/utf32.jl b/test/unicode/utf32.jl index 2b9488a024e07..53e7de500ed3e 100644 --- a/test/unicode/utf32.jl +++ b/test/unicode/utf32.jl @@ -163,7 +163,7 @@ for T in (UTF8String, UTF16String, UTF32String) # Lead followed by non-continuation character > 0xbf @test_throws UnicodeError convert(T, strval(T, UInt8[byt,0x80,0x80,0xc0])) end - catch exp ; + catch exp println("Error checking $T: $byt") throw(exp) end From 9d368d4fdb3b5634a638ed388ebea982d371dd92 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Fri, 2 Oct 2015 02:38:04 -0400 Subject: [PATCH 0402/1938] Actually throw PkgErrors, add Pkg error tests Fix MethodError from giving too many inputs to PkgError Add an unsatisfiable REQUIRES test Add tests for Pkg.pin error cases Adjust comment for issue number in a test --- base/pkg/dir.jl | 10 +--------- base/pkg/entry.jl | 2 +- test/pkg.jl | 41 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/base/pkg/dir.jl b/base/pkg/dir.jl index 4925eee06640f..7566f6ec2c9ad 100644 --- a/base/pkg/dir.jl +++ b/base/pkg/dir.jl @@ -28,15 +28,7 @@ function cd(f::Function, args...; kws...) !haskey(ENV,"JULIA_PKGDIR") ? init() : throw(PkgError("Package metadata directory $metadata_dir doesn't exist; run Pkg.init() to initialize it.")) end - try - Base.cd(()->f(args...; kws...), dir) - catch err - if isa(err, PkgError) - print_with_color(:red, "ERROR: $(err.msg)") - else - throw(err) - end - end + Base.cd(()->f(args...; kws...), dir) end function init(meta::AbstractString=DEFAULT_META, branch::AbstractString=META_BRANCH) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index b9de6dd87269a..0e767fd0aa4a2 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -475,7 +475,7 @@ function resolve( for pkg in keys(reqs) if !haskey(deps,pkg) if "julia" in conflicts[pkg] - throw(PkgError("$pkg can't be installed because it has no versions that support ", VERSION, " of julia. " * + throw(PkgError("$pkg can't be installed because it has no versions that support $VERSION of julia. " * "You may need to update METADATA by running `Pkg.update()`")) else sconflicts = join(conflicts[pkg], ", ", " and ") diff --git a/test/pkg.jl b/test/pkg.jl index e2f74d8ca86d4..d60c7bfd776ab 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -1,5 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +import Base.Pkg.PkgError + function temp_pkg_dir(fn::Function) # Used in tests below to setup and teardown a sandboxed package directory const tmpdir = ENV["JULIA_PKGDIR"] = joinpath(tempdir(),randstring()) @@ -42,7 +44,6 @@ temp_pkg_dir() do Pkg.rm("Example") @test isempty(Pkg.installed()) @test !isempty(Pkg.available("Example")) - @test Pkg.available("IDoNotExist") === nothing Pkg.clone("https://github.com/JuliaLang/Example.jl.git") @test [keys(Pkg.installed())...] == ["Example"] Pkg.status("Example", iob) @@ -60,6 +61,40 @@ temp_pkg_dir() do @test endswith(str, string(Pkg.installed("Example"))) Pkg.rm("Example") @test isempty(Pkg.installed()) + + # adding a package with unsatisfiable julia version requirements (REPL.jl) errors + try + Pkg.add("REPL") + error("unexpected") + catch err + @test isa(err.exceptions[1].ex, PkgError) + @test err.exceptions[1].ex.msg == "REPL can't be installed because " * + "it has no versions that support $VERSION of julia. You may " * + "need to update METADATA by running `Pkg.update()`" + end + + # trying to add, check availability, or pin a nonexistent package errors + try + Pkg.add("NonexistentPackage") + error("unexpected") + catch err + @test isa(err.exceptions[1].ex, PkgError) + @test err.exceptions[1].ex.msg == "unknown package NonexistentPackage" + end + try + Pkg.available("NonexistentPackage") + error("unexpected") + catch err + @test isa(err, PkgError) + @test err.msg == "NonexistentPackage is not a package (not registered or installed)" + end + try + Pkg.pin("NonexistentPackage", v"1.0.0") + error("unexpected") + catch err + @test isa(err, PkgError) + @test err.msg == "NonexistentPackage is not a git repo" + end end # testing a package with test dependencies causes them to be installed for the duration of the test @@ -96,6 +131,7 @@ temp_pkg_dir() do try Pkg.test("PackageWithNoTests") + error("unexpected") catch err @test err.msg == "PackageWithNoTests did not provide a test/runtests.jl file" end @@ -113,6 +149,7 @@ temp_pkg_dir() do try Pkg.test("PackageWithFailingTests") + error("unexpected") catch err @test err.msg == "PackageWithFailingTests had test errors" end @@ -165,7 +202,7 @@ end""" end end -# issue #13373 +# issue #13374 temp_pkg_dir() do Pkg.generate("Foo", "MIT") Pkg.tag("Foo") From 3a57ccbc21e3543379055573b5e711630f83bff0 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Fri, 2 Oct 2015 04:09:42 -0400 Subject: [PATCH 0403/1938] Remove extraneous .tmp in Pkg.pin error paths and add corresponding tests --- base/pkg/entry.jl | 4 ++-- test/pkg.jl | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 0e767fd0aa4a2..ef6d561b99d56 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -308,9 +308,9 @@ pin(pkg::AbstractString) = pin(pkg, "") function pin(pkg::AbstractString, ver::VersionNumber) ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) - Read.isinstalled(pkg) || throw(PkgError("$pkg cannot be pinned – not an installed package".tmp)) + Read.isinstalled(pkg) || throw(PkgError("$pkg cannot be pinned – not an installed package")) avail = Read.available(pkg) - isempty(avail) && throw(PkgError("$pkg cannot be pinned – not a registered package".tmp)) + isempty(avail) && throw(PkgError("$pkg cannot be pinned – not a registered package")) haskey(avail,ver) || throw(PkgError("$pkg – $ver is not a registered version")) pin(pkg, avail[ver].sha1) end diff --git a/test/pkg.jl b/test/pkg.jl index d60c7bfd776ab..a40fc31dd315f 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -59,8 +59,6 @@ temp_pkg_dir() do Pkg.status("Example", iob) str = chomp(takebuf_string(iob)) @test endswith(str, string(Pkg.installed("Example"))) - Pkg.rm("Example") - @test isempty(Pkg.installed()) # adding a package with unsatisfiable julia version requirements (REPL.jl) errors try @@ -95,6 +93,27 @@ temp_pkg_dir() do @test isa(err, PkgError) @test err.msg == "NonexistentPackage is not a git repo" end + + # trying to pin a git repo under Pkg.dir that is not an installed package errors + try + Pkg.pin("METADATA", v"1.0.0") + error("unexpected") + catch err + @test isa(err, PkgError) + @test err.msg == "METADATA cannot be pinned – not an installed package" + end + + # trying to pin an installed, registered package to an unregistered version errors + try + Pkg.pin("Example", v"2147483647.0.0") + error("unexpected") + catch err + @test isa(err, PkgError) + @test err.msg == "Example – 2147483647.0.0 is not a registered version" + end + + Pkg.rm("Example") + @test isempty(Pkg.installed()) end # testing a package with test dependencies causes them to be installed for the duration of the test @@ -119,6 +138,15 @@ temp_pkg_dir() do Pkg.test("PackageWithTestDependencies") @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] + + # trying to pin an unregistered package errors + try + Pkg.pin("PackageWithTestDependencies", v"1.0.0") + error("unexpected") + catch err + @test isa(err, PkgError) + @test err.msg == "PackageWithTestDependencies cannot be pinned – not a registered package" + end end # testing a package with no runtests.jl errors From e5b32a920396c83d4a3b7b0e17804b22b9985c31 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Thu, 1 Oct 2015 21:22:23 +0100 Subject: [PATCH 0404/1938] fix ambiguity warnings trunc(BigInt,::Float16) --- base/gmp.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/gmp.jl b/base/gmp.jl index de70240fa21ac..7e965fc986061 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -121,18 +121,18 @@ end convert(::Type{BigInt}, x::Bool) = BigInt(UInt(x)) -function unsafe_trunc(::Type{BigInt}, x::CdoubleMax) +function unsafe_trunc(::Type{BigInt}, x::Union{Float32,Float64}) z = BigInt() ccall((:__gmpz_set_d, :libgmp), Void, (Ptr{BigInt}, Cdouble), &z, x) return z end -function convert(::Type{BigInt}, x::CdoubleMax) +function convert(::Type{BigInt}, x::Union{Float32,Float64}) isinteger(x) || throw(InexactError()) unsafe_trunc(BigInt,x) end -function trunc(::Type{BigInt}, x::CdoubleMax) +function trunc(::Type{BigInt}, x::Union{Float32,Float64}) isfinite(x) || throw(InexactError()) unsafe_trunc(BigInt,x) end From 691f888a4a9f5210b55ccabfa01c55e4c5d95433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1ll=20Haraldsson?= Date: Fri, 2 Oct 2015 10:04:54 +0000 Subject: [PATCH 0405/1938] Better link Link not specific to Wily, so not saying that. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3d0cefa896c52..c6eadbc08ce91 100644 --- a/README.md +++ b/README.md @@ -399,7 +399,7 @@ The following distributions include julia, but the versions may be out of date d * Git package for openSUSE: [OBS page](https://build.opensuse.org/package/show/science/julia-unstable), [1 Click Install](http://software.opensuse.org/download.html?project=science&package=julia-unstable) * [NixOS](https://github.com/NixOS/nixpkgs/blob/master/pkgs/development/compilers/julia) * Ubuntu - * [Ubuntu 15.10 (Wily Werewolf)](http://packages.ubuntu.com/wily/julia) + * [Ubuntu](http://packages.ubuntu.com/search?keywords=julia) * [Nightly builds PPA](https://launchpad.net/~staticfloat/+archive/julianightlies) (depends on the [julia-deps PPA](https://launchpad.net/~staticfloat/+archive/julia-deps/)) * [OS X Homebrew Tap](https://github.com/staticfloat/homebrew-julia/) From ff19f891807bbb453e2c161b9522c5de2e2cacf9 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 2 Oct 2015 11:07:57 -0400 Subject: [PATCH 0406/1938] fix a codegen assertion failure --- src/codegen.cpp | 3 ++- test/core.jl | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index af1a59ee66a00..1d6e59b9140a6 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2239,7 +2239,8 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, if (!jl_is_tuple_type(tp0) && jl_is_leaf_type(tp0)) { *ret = emit_expr(args[1], ctx); emit_typecheck(*ret, tp0, "typeassert", ctx); - *ret = remark_julia_type(*ret, expr_type(expr, ctx)); + if (ret->isboxed) + *ret = remark_julia_type(*ret, expr_type(expr, ctx)); JL_GC_POP(); return true; } diff --git a/test/core.jl b/test/core.jl index 0f898a9504273..6fabf8c54076a 100644 --- a/test/core.jl +++ b/test/core.jl @@ -1594,6 +1594,12 @@ end h5142(true) @test_throws TypeError h5142(1) h5142(2) +f5142() = h5142(1) +try + # try running this code in a different context that triggers the codegen + # assertion `assert(isboxed || v.typ == typ)`. + f5142() +end bitstype 8 Int5142b function h5142b(a::Int) From 2bbdc30448ac3ded06f41a5c9d2ce54bdf2b5c67 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 18 Sep 2015 18:54:14 -0400 Subject: [PATCH 0407/1938] eliminate codegen dependence on jl_pvalue_llvmt as a unique type --- src/ccall.cpp | 93 ++++---- src/cgutils.cpp | 97 +++++---- src/codegen.cpp | 513 +++++++++++++++++++++++++-------------------- src/intrinsics.cpp | 73 ++++--- 4 files changed, 429 insertions(+), 347 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 4866639609b44..b66b236ac8dbd 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -314,13 +314,13 @@ Value *llvm_type_rewrite(Value *v, Type *from_type, Type *target_type, // to = desired LLVM type // jlto = Julia type of formal argument // jvinfo = value of actual argument -static Value *julia_to_native(Type *to, jl_value_t *jlto, const jl_cgval_t &jvinfo, +static Value *julia_to_native(Type *to, bool toboxed, jl_value_t *jlto, const jl_cgval_t &jvinfo, bool addressOf, bool byRef, bool inReg, bool needCopy, bool tojulia, int argn, jl_codectx_t *ctx, bool *needStackRestore) { // We're passing Any - if (to == jl_pvalue_llvmt) { + if (toboxed) { assert(!addressOf && !byRef); // don't expect any ABI to pass pointers by pointer return boxed(jvinfo, ctx); } @@ -586,7 +586,7 @@ static jl_cgval_t emit_cglobal(jl_value_t **args, size_t nargs, jl_codectx_t *ct } JL_GC_POP(); - return mark_julia_type(res, rt); + return mark_julia_type(res, false, rt); } // llvmcall(ir, (rettypes...), (argtypes...), args...) @@ -663,7 +663,8 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c */ for (size_t i = 0; i < nargt; ++i) { jl_value_t *tti = jl_svecref(tt,i); - Type *t = julia_type_to_llvm(tti); + bool toboxed; + Type *t = julia_type_to_llvm(tti, &toboxed); argtypes.push_back(t); if (4+i > nargs) { jl_error("Missing arguments to llvmcall!"); @@ -671,19 +672,20 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c jl_value_t *argi = args[4+i]; jl_cgval_t arg; bool needroot = false; - if (t == jl_pvalue_llvmt || !jl_isbits(tti)) { + if (toboxed || !jl_isbits(tti)) { arg = emit_expr(argi, ctx, true); - if (t == jl_pvalue_llvmt && !arg.isboxed) { + if (toboxed && !arg.isboxed) { needroot = true; } } else { arg = emit_unboxed(argi, ctx); + toboxed = false; } - Value *v = julia_to_native(t, tti, arg, false, false, false, false, false, i, ctx, NULL); + Value *v = julia_to_native(t, toboxed, tti, arg, false, false, false, false, false, i, ctx, NULL); // make sure args are rooted - if (t == jl_pvalue_llvmt && (needroot || might_need_root(argi))) { + if (toboxed && (needroot || might_need_root(argi))) { make_gcroot(v, ctx); } bool issigned = jl_signed_type && jl_subtype(tti, (jl_value_t*)jl_signed_type, 0); @@ -691,7 +693,8 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c } Function *f; - Type *rettype = julia_type_to_llvm(rtt); + bool retboxed; + Type *rettype = julia_type_to_llvm(rtt, &retboxed); if (isString) { // Make sure to find a unique name std::string ir_name; @@ -779,7 +782,7 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c // the actual call assert(f->getParent() == jl_Module); // no prepare_call(f) is needed below, since this was just emitted into the same module - CallInst *inst = builder.CreateCall(f,ArrayRef(&argvals[0],nargt)); + CallInst *inst = builder.CreateCall(f, ArrayRef(&argvals[0], nargt)); ctx->to_inline.push_back(inst); JL_GC_POP(); @@ -788,12 +791,12 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c jl_error("Return type of llvmcall'ed function does not match declared return type"); } - return mark_julia_type(inst, rtt); + return mark_julia_type(inst, retboxed, rtt); } // --- code generator for ccall itself --- -static jl_cgval_t mark_or_box_ccall_result(Value *result, jl_value_t *rt_expr, jl_value_t *rt, bool static_rt, jl_codectx_t *ctx) +static jl_cgval_t mark_or_box_ccall_result(Value *result, bool isboxed, jl_value_t *rt_expr, jl_value_t *rt, bool static_rt, jl_codectx_t *ctx) { if (!static_rt) { // box if concrete type was not statically known @@ -802,9 +805,10 @@ static jl_cgval_t mark_or_box_ccall_result(Value *result, jl_value_t *rt_expr, j int nb = sizeof(void*); return mark_julia_type( init_bits_value(emit_allocobj(nb), runtime_bt, result), + true, (jl_value_t*)jl_pointer_type); } - return mark_julia_type(result, rt); + return mark_julia_type(result, isboxed, rt); } typedef AttributeSet attr_type; @@ -814,6 +818,7 @@ static std::string generate_func_sig( Type **prt, // out parameter of the llvm return type for the function signature int &sret, // out parameter for indicating whether return value has been moved to the first argument position std::vector &fargt, // vector of llvm output types (julia_struct_to_llvm) for arguments (vararg is the last item, if applicable) + std::vector &fargt_isboxed, // vector of whether the llvm output types is boxed for each argument (vararg is the last item, if applicable) std::vector &fargt_sig, // vector of ABI coercion types for call signature Type *&fargt_vasig, // ABI coercion type for vararg list std::vector &inRegList, // vector of "inreg" parameters (vararg is the last item, if applicable) @@ -859,12 +864,14 @@ static std::string generate_func_sig( tti = jl_tparam0(tti); } Type *t = NULL; + bool isboxed; Attribute::AttrKind av = Attribute::None; if (jl_is_abstract_ref_type(tti)) { if (jl_is_typevar(jl_tparam0(tti))) jl_error("ccall: argument type Ref should have an element type, not Ref{T}"); tti = (jl_value_t*)jl_voidpointer_type; t = T_pint8; + isboxed = false; } else { if (jl_is_cpointer_type(tti) && jl_is_typevar(jl_tparam0(tti))) @@ -881,7 +888,7 @@ static std::string generate_func_sig( } } - t = julia_struct_to_llvm(tti); + t = julia_struct_to_llvm(tti, &isboxed); if (t == NULL || t == T_void) { std::stringstream msg; msg << "ccall: the type of argument "; @@ -915,6 +922,7 @@ static std::string generate_func_sig( byRefList.push_back(byRef); inRegList.push_back(inReg); fargt.push_back(t); + fargt_isboxed.push_back(isboxed); if (!current_isVa) fargt_sig.push_back(pat); else @@ -1034,7 +1042,8 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) } JL_TYPECHK(ccall, type, rt); - Type *lrt = julia_struct_to_llvm(rt); + bool retboxed; + Type *lrt = julia_struct_to_llvm(rt, &retboxed); if (lrt == NULL) { emit_error("ccall: return type doesn't correspond to a C type", ctx); JL_GC_POP(); @@ -1111,8 +1120,8 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) assert(!(jl_is_expr(argi) && ((jl_expr_t*)argi)->head == amp_sym)); jl_cgval_t ary = emit_expr(argi, ctx); JL_GC_POP(); - return mark_or_box_ccall_result(builder.CreateBitCast(emit_arrayptr(boxed(ary, ctx)),lrt), - args[2], rt, static_rt, ctx); + return mark_or_box_ccall_result(builder.CreateBitCast(emit_arrayptr(boxed(ary, ctx)), lrt), + retboxed, args[2], rt, static_rt, ctx); } if (fptr == (void *) &jl_value_ptr || ((f_lib==NULL || (intptr_t)f_lib==2) @@ -1132,11 +1141,15 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) } Value *ary; Type *largty; - if (addressOf) - largty = jl_pvalue_llvmt; - else - largty = julia_struct_to_llvm(tti); - if (largty == jl_pvalue_llvmt) { + bool isboxed; + if (addressOf) { + largty = T_pjlvalue; + isboxed = true; + } + else { + largty = julia_struct_to_llvm(tti, &isboxed); + } + if (isboxed) { ary = boxed(emit_expr(argi, ctx),ctx); } else { @@ -1145,7 +1158,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) } JL_GC_POP(); return mark_or_box_ccall_result(builder.CreateBitCast(ary, lrt), - args[2], rt, static_rt, ctx); + retboxed, args[2], rt, static_rt, ctx); } if (fptr == (void *) &jl_is_leaf_type || ((f_lib==NULL || (intptr_t)f_lib==2) @@ -1157,7 +1170,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) int isleaf = jl_is_leaf_type(jl_tparam0(ty)); JL_GC_POP(); return mark_or_box_ccall_result(ConstantInt::get(T_int32, isleaf), - args[2], rt, static_rt, ctx); + false, args[2], rt, static_rt, ctx); } } if (fptr == (void*)&jl_function_ptr || @@ -1191,7 +1204,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) emit_expr(args[8], ctx); JL_GC_POP(); return mark_or_box_ccall_result(builder.CreateBitCast(llvmf, lrt), - args[2], rt, static_rt, ctx); + retboxed, args[2], rt, static_rt, ctx); } JL_CATCH { } @@ -1215,13 +1228,14 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) std::vector fargt(0); std::vector fargt_sig(0); + std::vector fargt_isboxed(0); Type *fargt_vasig = NULL; std::vector inRegList(0); std::vector byRefList(0); attr_type attrs; Type *prt = NULL; int sret = 0; - std::string err_msg = generate_func_sig(&lrt, &prt, sret, fargt, fargt_sig, fargt_vasig, + std::string err_msg = generate_func_sig(&lrt, &prt, sret, fargt, fargt_isboxed, fargt_sig, fargt_vasig, inRegList, byRefList, attrs, rt, tt, (nargs - 3)/2); if (!err_msg.empty()) { JL_GC_POP(); @@ -1236,6 +1250,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) // First, if the ABI requires us to provide the space for the return // argument, allocate the box and store that as the first argument type + bool sretboxed; if (sret) { jl_cgval_t sret_val = emit_new_struct(rt,1,NULL,ctx); // TODO: is it valid to be creating an incomplete type this way? assert(sret_val.typ != NULL && "Type was not concrete"); @@ -1246,10 +1261,11 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) argvals[0] = result; } else { - // XXX: result needs a GC root here if result->getType() == jl_pvalue_llvmt + // XXX: result needs a GC root here if result->getType() == T_pjlvalue result = sret_val.V; argvals[0] = builder.CreateBitCast(result, fargt_sig.at(0)); } + sretboxed = sret_val.isboxed; } // save argument depth until after we're done emitting arguments @@ -1271,16 +1287,19 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) } Type *largty; // LLVM type of the current parameter + bool toboxed; jl_value_t *jargty; // Julia type of the current parameter bool byRef, inReg; // Argument attributes if (isVa && ai >= nargt - 1) { largty = fargt.at(nargt - 1); + toboxed = fargt_isboxed.at(nargt - 1); jargty = jl_tparam0(jl_svecref(tt, nargt - 1)); byRef = byRefList.at(nargt - 1); inReg = inRegList.at(nargt - 1); } else { largty = fargt.at(ai); + toboxed = fargt_isboxed.at(ai); jargty = jl_svecref(tt, ai); byRef = byRefList.at(ai); inReg = inRegList.at(ai); @@ -1302,9 +1321,9 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) } jargty = (jl_value_t*)jl_voidpointer_type; } - else if (largty == jl_pvalue_llvmt || largty->isStructTy()) { + else if (toboxed || largty->isStructTy()) { arg = emit_expr(argi, ctx, true); - if (largty == jl_pvalue_llvmt && !arg.isboxed) { + if (toboxed && !arg.isboxed) { needroot = true; } } @@ -1312,10 +1331,10 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) arg = emit_unboxed(argi, ctx); } - Value *v = julia_to_native(largty, jargty, arg, addressOf, byRef, inReg, + Value *v = julia_to_native(largty, toboxed, jargty, arg, addressOf, byRef, inReg, need_private_copy(jargty, byRef), false, ai + 1, ctx, &needStackRestore); // make sure args are rooted - if (largty == jl_pvalue_llvmt && (needroot || might_need_root(argi))) { + if (toboxed && (needroot || might_need_root(argi))) { make_gcroot(v, ctx); } bool issigned = jl_signed_type && jl_subtype(jargty, (jl_value_t*)jl_signed_type, 0); @@ -1411,18 +1430,17 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) // type because the ABI required us to pass a pointer (sret), // then we do not need to do this. if (!sret) { - Type *jlrt = julia_type_to_llvm(rt); + Type *jlrt = julia_type_to_llvm(rt, &retboxed); // compute the real "julian" return type and update retboxed if (type_is_ghost(jlrt)) { return ghostValue(rt); } - else if (lrt->isStructTy() && jlrt == jl_pvalue_llvmt) { + else if (lrt->isStructTy() && retboxed) { assert(jl_is_structtype(rt)); jl_cgval_t newst = emit_new_struct(rt, 1, NULL, ctx); // emit a new, empty struct assert(newst.typ != NULL && "Type was not concrete"); assert(newst.isboxed); // copy the data from the return value to the new struct - // julia gc is aligned 16, otherwise use default alignment for alloca pointers - builder.CreateAlignedStore(result, builder.CreateBitCast(newst.V, prt->getPointerTo()), newst.V->getType() == jl_pvalue_llvmt ? 16 : 0); + builder.CreateAlignedStore(result, builder.CreateBitCast(newst.V, prt->getPointerTo()), 16); // julia gc is aligned 16 return newst; } else if (jlrt != prt) { @@ -1431,9 +1449,10 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) } } else { - if (result->getType() != jl_pvalue_llvmt) + retboxed = sretboxed; + if (!retboxed) result = builder.CreateLoad(result); // something alloca'd above } - return mark_or_box_ccall_result(result, args[2], rt, static_rt, ctx); + return mark_or_box_ccall_result(result, retboxed, args[2], rt, static_rt, ctx); } diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 2ff4fad667ae3..8deefc2cb9fef 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -541,9 +541,9 @@ static Value *julia_gv(const char *cname, void *addr) std::stringstream gvname; gvname << cname << globalUnique++; // no existing GlobalVariable, create one and store it - GlobalVariable *gv = new GlobalVariable(*jl_Module, jl_pvalue_llvmt, + GlobalVariable *gv = new GlobalVariable(*jl_Module, T_pjlvalue, false, imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - ConstantPointerNull::get((PointerType*)jl_pvalue_llvmt), gvname.str()); + ConstantPointerNull::get((PointerType*)T_pjlvalue), gvname.str()); addComdat(gv); // make the pointer valid for this session @@ -595,7 +595,7 @@ static Value *literal_pointer_val(jl_value_t *p) // emit a pointer to any jl_value_t which will be valid across reloading code // also, try to give it a nice name for gdb, for easy identification if (p == NULL) - return ConstantPointerNull::get((PointerType*)jl_pvalue_llvmt); + return ConstantPointerNull::get((PointerType*)T_pjlvalue); // some common constant values if (p == jl_false) return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlfalse_var))); @@ -604,7 +604,7 @@ static Value *literal_pointer_val(jl_value_t *p) if (p == (jl_value_t*)jl_emptysvec) return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlemptysvec_var))); if (!imaging_mode) - return literal_static_pointer_val(p, jl_pvalue_llvmt); + return literal_static_pointer_val(p, T_pjlvalue); if (jl_is_datatype(p)) { jl_datatype_t *addr = (jl_datatype_t*)p; // DataTypes are prefixed with a + @@ -640,9 +640,9 @@ static Value *literal_pointer_val(jl_binding_t *p) { // emit a pointer to any jl_value_t which will be valid across reloading code if (p == NULL) - return ConstantPointerNull::get((PointerType*)jl_pvalue_llvmt); + return ConstantPointerNull::get((PointerType*)T_pjlvalue); if (!imaging_mode) - return literal_static_pointer_val(p, jl_pvalue_llvmt); + return literal_static_pointer_val(p, T_pjlvalue); // bindings are prefixed with jl_bnd# return julia_gv("jl_bnd#", p->name, p->owner, p); } @@ -659,23 +659,26 @@ static Value *julia_binding_gv(jl_binding_t *b) // emit a literal_pointer_val to the value field of a jl_binding_t // binding->value are prefixed with * Value *bv = imaging_mode ? - builder.CreateBitCast(julia_gv("*", b->name, b->owner, b), jl_ppvalue_llvmt) : - literal_static_pointer_val(b,jl_ppvalue_llvmt); + builder.CreateBitCast(julia_gv("*", b->name, b->owner, b), T_ppjlvalue) : + literal_static_pointer_val(b,T_ppjlvalue); return julia_binding_gv(bv); } // --- mapping between julia and llvm types --- -static Type *julia_struct_to_llvm(jl_value_t *jt); +static Type *julia_struct_to_llvm(jl_value_t *jt, bool *isboxed); extern "C" { -DLLEXPORT Type *julia_type_to_llvm(jl_value_t *jt) +DLLEXPORT Type *julia_type_to_llvm(jl_value_t *jt, bool *isboxed) { // this function converts a Julia Type into the equivalent LLVM type + if (isboxed) *isboxed = false; if (jt == (jl_value_t*)jl_bool_type) return T_int1; if (jt == (jl_value_t*)jl_bottom_type) return T_void; - if (!jl_is_leaf_type(jt)) - return jl_pvalue_llvmt; + if (!jl_is_leaf_type(jt)) { + if (isboxed) *isboxed = true; + return T_pjlvalue; + } if (jl_is_cpointer_type(jt)) { Type *lt = julia_type_to_llvm(jl_tparam0(jt)); if (lt == NULL) @@ -705,18 +708,20 @@ DLLEXPORT Type *julia_type_to_llvm(jl_value_t *jt) if (((jl_datatype_t*)jt)->size == 0) { return T_void; } - return julia_struct_to_llvm(jt); + return julia_struct_to_llvm(jt, isboxed); } - return jl_pvalue_llvmt; + if (isboxed) *isboxed = true; + return T_pjlvalue; } } -static Type *julia_struct_to_llvm(jl_value_t *jt) +static Type *julia_struct_to_llvm(jl_value_t *jt, bool *isboxed) { // this function converts a Julia Type into the equivalent LLVM struct // use this where C-compatible (unboxed) structs are desired // use julia_type_to_llvm directly when you want to preserve Julia's type semantics bool isTuple = jl_is_tuple_type(jt); + if (isboxed) *isboxed = false; if ((isTuple || jl_is_structtype(jt)) && !jl_is_array_type(jt)) { if (!jl_is_leaf_type(jt)) return NULL; @@ -738,7 +743,7 @@ static Type *julia_struct_to_llvm(jl_value_t *jt) jl_value_t *ty = jl_svecref(jst->types, i); Type *lty; if (jl_field_isptr(jst, i)) - lty = jl_pvalue_llvmt; + lty = T_pjlvalue; else lty = ty==(jl_value_t*)jl_bool_type ? T_int8 : julia_type_to_llvm(ty); if (lasttype != NULL && lasttype != lty) @@ -767,7 +772,7 @@ static Type *julia_struct_to_llvm(jl_value_t *jt) } return (Type*)jst->struct_decl; } - return julia_type_to_llvm(jt); + return julia_type_to_llvm(jt, isboxed); } static bool is_datatype_all_pointers(jl_datatype_t *dt) @@ -806,13 +811,13 @@ static bool deserves_sret(jl_value_t *dt, Type *T) static Value *emit_nthptr_addr(Value *v, ssize_t n) { - return builder.CreateGEP(builder.CreateBitCast(v, jl_ppvalue_llvmt), + return builder.CreateGEP(builder.CreateBitCast(v, T_ppjlvalue), ConstantInt::get(T_size, (ssize_t)n)); } static Value *emit_nthptr_addr(Value *v, Value *idx) { - return builder.CreateGEP(builder.CreateBitCast(v, jl_ppvalue_llvmt), idx); + return builder.CreateGEP(builder.CreateBitCast(v, T_ppjlvalue), idx); } static Value *emit_nthptr(Value *v, ssize_t n, MDNode *tbaa) @@ -845,12 +850,12 @@ static Value *emit_typeptr_addr(Value *p) static Value *emit_typeof(Value *tt) { // given p, a jl_value_t*, compute its type tag - assert(tt->getType() == jl_pvalue_llvmt); + assert(tt->getType() == T_pjlvalue); tt = builder.CreateLoad(emit_typeptr_addr(tt), false); tt = builder.CreateIntToPtr(builder.CreateAnd( builder.CreatePtrToInt(tt, T_size), ConstantInt::get(T_size,~(uptrint_t)15)), - jl_pvalue_llvmt); + T_pjlvalue); return tt; } static Value *emit_typeof(const jl_cgval_t &p) @@ -874,7 +879,7 @@ static Value *emit_datatype_types(const jl_cgval_t &dt) CreateBitCast(builder. CreateGEP(builder.CreateBitCast(dt.V, T_pint8), ConstantInt::get(T_size, offsetof(jl_datatype_t, types))), - jl_ppvalue_llvmt)); + T_ppjlvalue)); } static Value *emit_datatype_nfields(const jl_cgval_t &dt) @@ -1085,7 +1090,8 @@ static Value *emit_unbox(Type *to, const jl_cgval_t &x, jl_value_t *jt); static jl_cgval_t typed_load(Value *ptr, Value *idx_0based, jl_value_t *jltype, jl_codectx_t *ctx, MDNode* tbaa, size_t alignment = 0) { - Type *elty = julia_type_to_llvm(jltype); + bool isboxed; + Type *elty = julia_type_to_llvm(jltype, &isboxed); assert(elty != NULL); if (type_is_ghost(elty)) return ghostValue(jltype); @@ -1116,13 +1122,13 @@ static jl_cgval_t typed_load(Value *ptr, Value *idx_0based, jl_value_t *jltype, else { elt = load; } - if (elty == jl_pvalue_llvmt) { + if (isboxed) { null_pointer_check(elt, ctx); } //} if (isbool) - return mark_julia_type(builder.CreateTrunc(elt, T_int1), jltype); - return mark_julia_type(elt, jltype); + return mark_julia_type(builder.CreateTrunc(elt, T_int1), false, jltype); + return mark_julia_type(elt, isboxed, jltype); } static void typed_store(Value *ptr, Value *idx_0based, const jl_cgval_t &rhs, @@ -1260,7 +1266,7 @@ static void jl_add_linfo_root(jl_lambda_info_t *li, jl_value_t *val); static Value *data_pointer(Value *x) { - return builder.CreateBitCast(x, jl_ppvalue_llvmt); + return builder.CreateBitCast(x, T_ppjlvalue); } static bool emit_getfield_unknownidx(jl_cgval_t *ret, const jl_cgval_t &strct, Value *idx, jl_datatype_t *stt, jl_codectx_t *ctx) @@ -1271,11 +1277,11 @@ static bool emit_getfield_unknownidx(jl_cgval_t *ret, const jl_cgval_t &strct, V idx = emit_bounds_check(strct, (jl_value_t*)stt, idx, ConstantInt::get(T_size, nfields), ctx); Value *fld = tbaa_decorate(tbaa_user, builder.CreateLoad( builder.CreateGEP( - builder.CreateBitCast(strct.V, jl_ppvalue_llvmt), + builder.CreateBitCast(strct.V, T_ppjlvalue), idx))); if ((unsigned)stt->ninitialized != nfields) null_pointer_check(fld, ctx); - *ret = mark_julia_type(fld, jl_any_type); + *ret = mark_julia_type(fld, true, jl_any_type); return true; } else if (is_tupletype_homogeneous(stt->types)) { @@ -1293,7 +1299,7 @@ static bool emit_getfield_unknownidx(jl_cgval_t *ret, const jl_cgval_t &strct, V #else Value *fld = builder.CreateCall2(prepare_call(jlgetnthfieldchecked_func), strct.V, idx); #endif - *ret = mark_julia_type(fld, jl_any_type); + *ret = mark_julia_type(fld, true, jl_any_type); return true; } } @@ -1323,7 +1329,7 @@ static bool emit_getfield_unknownidx(jl_cgval_t *ret, const jl_cgval_t &strct, V if (jt == (jl_value_t*)jl_bool_type) { fld = builder.CreateTrunc(fld, T_int1); } - *ret = mark_julia_type(fld, jt); + *ret = mark_julia_type(fld, false, jt); return true; } return false; @@ -1347,10 +1353,10 @@ static jl_cgval_t emit_getfield_knownidx(const jl_cgval_t &strct, unsigned idx, ConstantInt::get(T_size, jl_field_offset(jt,idx))); MDNode *tbaa = strct.isimmutable ? tbaa_immut : tbaa_user; if (jl_field_isptr(jt,idx)) { - Value *fldv = tbaa_decorate(tbaa, builder.CreateLoad(builder.CreateBitCast(addr,jl_ppvalue_llvmt))); + Value *fldv = tbaa_decorate(tbaa, builder.CreateLoad(builder.CreateBitCast(addr, T_ppjlvalue))); if (idx >= (unsigned)jt->ninitialized) null_pointer_check(fldv, ctx); - return mark_julia_type(fldv, jfty); + return mark_julia_type(fldv, true, jfty); } else { int align = jl_field_offset(jt,idx); @@ -1376,7 +1382,7 @@ static jl_cgval_t emit_getfield_knownidx(const jl_cgval_t &strct, unsigned idx, fldv = builder.CreateTrunc(fldv, T_int1); } assert(!jl_field_isptr(jt, idx)); - return mark_julia_type(fldv, jfty); + return mark_julia_type(fldv, false, jfty); } } @@ -1447,7 +1453,7 @@ static Value *emit_arraylen_prim(Value *t, jl_value_t *ty) } else { std::vector fargt(0); - fargt.push_back(jl_pvalue_llvmt); + fargt.push_back(T_pjlvalue); FunctionType *ft = FunctionType::get(T_size, fargt, false); Value *alen = jl_Module->getOrInsertFunction("jl_array_len_", ft); return builder.CreateCall(prepare_call(alen), t); @@ -1703,7 +1709,7 @@ static Value *boxed(const jl_cgval_t &vinfo, jl_codectx_t *ctx, jl_value_t *jt) } if (jt == jl_bottom_type || jt == NULL) { // We have an undef value on a (hopefully) dead branch - return UndefValue::get(jl_pvalue_llvmt); + return UndefValue::get(T_pjlvalue); } if (vinfo.isghost) { jl_value_t *s = static_void_instance(jt); @@ -1770,7 +1776,7 @@ static void emit_cpointercheck(const jl_cgval_t &x, const std::string &msg, jl_codectx_t *ctx) { Value *t = emit_typeof(x); - emit_typecheck(mark_julia_type(t, jl_any_type), (jl_value_t*)jl_datatype_type, msg, ctx); + emit_typecheck(mark_julia_type(t, true, jl_any_type), (jl_value_t*)jl_datatype_type, msg, ctx); Value *istype = builder.CreateICmpEQ(emit_nthptr(t, (ssize_t)(offsetof(jl_datatype_t,name)/sizeof(char*)), tbaa_datatype), @@ -1835,7 +1841,7 @@ static void emit_write_barrier(jl_codectx_t* ctx, Value *parent, Value *ptr) Value* ptr_not_marked = builder.CreateICmpEQ(ptr_mark_bit, ConstantInt::get(T_size, 0)); builder.CreateCondBr(ptr_not_marked, barrier_trigger, cont); builder.SetInsertPoint(barrier_trigger); - builder.CreateCall(prepare_call(queuerootfun), builder.CreateBitCast(parent, jl_pvalue_llvmt)); + builder.CreateCall(prepare_call(queuerootfun), builder.CreateBitCast(parent, T_pjlvalue)); builder.CreateBr(cont); ctx->f->getBasicBlockList().push_back(cont); builder.SetInsertPoint(cont); @@ -1867,7 +1873,7 @@ static void emit_setfield(jl_datatype_t *sty, const jl_cgval_t &strct, size_t id if (jl_field_isptr(sty, idx0)) { Value *r = boxed(rhs, ctx); builder.CreateStore(r, - builder.CreateBitCast(addr, jl_ppvalue_llvmt)); + builder.CreateBitCast(addr, T_ppjlvalue)); if (wb) emit_checked_write_barrier(ctx, strct.V, r); } else { @@ -1916,7 +1922,7 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg } idx++; } - return mark_julia_type(strct, ty); + return mark_julia_type(strct, false, ty); } Value *f1 = NULL; int fieldStart = ctx->gc.argDepth; @@ -1940,11 +1946,11 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg make_gcroot(f1, ctx); } Value *strct = emit_allocobj(sty->size); - jl_cgval_t strctinfo = mark_julia_type(strct, ty); + jl_cgval_t strctinfo = mark_julia_type(strct, true, ty); builder.CreateStore(literal_pointer_val((jl_value_t*)ty), emit_typeptr_addr(strct)); if (f1) { - jl_cgval_t f1info = mark_julia_type(f1, jl_any_type); + jl_cgval_t f1info = mark_julia_type(f1, true, jl_any_type); if (!jl_subtype(expr_type(args[1],ctx), jl_field_type(sty,0), 0)) emit_typecheck(f1info, jl_field_type(sty,0), "new", ctx); emit_setfield(sty, strctinfo, 0, f1info, ctx, false, false); @@ -1962,7 +1968,7 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg builder.CreatePointerCast( builder.CreateGEP(builder.CreateBitCast(strct, T_pint8), ConstantInt::get(T_size, jl_field_offset(sty,i))), - jl_ppvalue_llvmt)); + T_ppjlvalue)); } } bool need_wb = false; @@ -1995,11 +2001,12 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg if (nargs >= 2) return emit_unboxed(args[1], ctx); // do side effects Type *lt = julia_type_to_llvm(ty); - return mark_julia_type(UndefValue::get(lt), ty); + assert(lt != T_pjlvalue); + return mark_julia_type(UndefValue::get(lt), false, ty); } else { // 0 fields, singleton assert(sty->instance != NULL); - return mark_julia_type(literal_pointer_val(sty->instance), sty); + return mark_julia_const(sty->instance); } } diff --git a/src/codegen.cpp b/src/codegen.cpp index 1d6e59b9140a6..12aa0358db5f5 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -115,6 +115,10 @@ using namespace llvm::legacy; #define LLVM37_param(x) #endif +#ifndef LLVM35 +#define AddrSpaceCastInst BitCastInst +#endif + extern "C" { #include "builtin_proto.h" @@ -193,9 +197,9 @@ static DataLayout *jl_data_layout; static bool imaging_mode = false; // types -static Type *jl_value_llvmt; -static Type *jl_pvalue_llvmt; -static Type *jl_ppvalue_llvmt; +static Type *T_jlvalue; +static Type *T_pjlvalue; +static Type *T_ppjlvalue; static Type* jl_parray_llvmt; static FunctionType *jl_func_sig; static Type *jl_pfptr_llvmt; @@ -225,6 +229,9 @@ static Type *T_psize; static Type *T_pfloat32; static Type *T_pfloat64; +static Type *T_ppint8; +static Type *T_pppint8; + static Type *T_void; // type-based alias analysis nodes. Indentation of comments indicates hierarchy. @@ -267,7 +274,7 @@ static Value *V_null; static Type *NoopType; static Value *literal_pointer_val(jl_value_t *p); extern "C" { -DLLEXPORT Type *julia_type_to_llvm(jl_value_t *jt); +DLLEXPORT Type *julia_type_to_llvm(jl_value_t *jt, bool *isboxed=NULL); } static bool type_is_ghost(Type *ty) { @@ -390,15 +397,14 @@ struct jl_cgval_t { bool isghost; // whether this value is "ghost" bool ispointer; // whether this value is actually pointer to the value bool isimmutable; // V points to something that is definitely immutable (e.g. not stack allocated) - //bool isbox; // points to memory inside a jl_box_t* //bool isstack; // points to stack-allocated memory //bool isarg; // derived from an argument bool needsgcroot; // this value needs a gcroot - jl_cgval_t(Value *V, jl_value_t *typ) : // general constructor (with pointer type auto-detect) + jl_cgval_t(Value *V, bool isboxed, jl_value_t *typ) : // general constructor (with pointer type auto-detect) V(V), // V is allowed to be NULL in a jl_varinfo_t context, but not during codegen contexts typ(typ), //T(julia_type_to_llvm(typ)), - isboxed(V && V->getType() == jl_pvalue_llvmt), + isboxed(isboxed), isghost(false), ispointer(this->isboxed), isimmutable(this->isboxed && jl_is_immutable_datatype(typ)), @@ -584,7 +590,7 @@ static AllocaInst *emit_static_alloca(Type *lty) static inline jl_cgval_t ghostValue(jl_value_t *typ) { - assert(typ == jl_bottom_type || jl_is_datatype(typ)); + assert(typ == jl_bottom_type || (jl_is_datatype(typ) && jl_datatype_size(typ) == 0)); return jl_cgval_t(typ); } static inline jl_cgval_t ghostValue(jl_datatype_t *typ) @@ -595,29 +601,31 @@ static inline jl_cgval_t ghostValue(jl_datatype_t *typ) static inline jl_cgval_t mark_julia_slot(Value *v, jl_value_t *typ) { // eagerly put this back onto the stack - jl_cgval_t tagval(v, typ); + assert(v->getType() != T_pjlvalue); + jl_cgval_t tagval(v, false, typ); tagval.ispointer = true; return tagval; } -static inline jl_cgval_t mark_julia_type(Value *v, jl_value_t *typ) +static inline jl_cgval_t mark_julia_type(Value *v, bool isboxed, jl_value_t *typ) { Type *T = julia_type_to_llvm(typ); if (type_is_ghost(T)) { return ghostValue(typ); } - if (v && T->isAggregateType() && v->getType() != jl_pvalue_llvmt) { + if (v && T->isAggregateType() && !isboxed) { + assert(v->getType() != T_pjlvalue); // eagerly put this back onto the stack // llvm mem2reg pass will remove this if unneeded Value *loc = emit_static_alloca(T); builder.CreateStore(v, loc); return mark_julia_slot(loc, typ); } - return jl_cgval_t(v, typ); + return jl_cgval_t(v, isboxed, typ); } -static inline jl_cgval_t mark_julia_type(Value *v, jl_datatype_t *typ) +static inline jl_cgval_t mark_julia_type(Value *v, bool isboxed, jl_datatype_t *typ) { - return mark_julia_type(v, (jl_value_t*)typ); + return mark_julia_type(v, isboxed, (jl_value_t*)typ); } static inline jl_cgval_t remark_julia_type(const jl_cgval_t &v, jl_value_t *typ) @@ -630,11 +638,15 @@ static inline jl_cgval_t remark_julia_type(const jl_cgval_t &v, jl_value_t *typ) } static inline jl_cgval_t mark_julia_const(jl_value_t *jv) { - jl_value_t *typ = jl_typeof(jv); + jl_value_t *typ; + if (jl_is_datatype(jv) || jl_is_uniontype(jv) || jl_is_typector(jv)) + typ = (jl_value_t*)jl_wrap_Type(jv); + else + typ = jl_typeof(jv); if (type_is_ghost(julia_type_to_llvm(typ))) { return ghostValue(typ); } - return jl_cgval_t(literal_pointer_val(jv), typ); + return jl_cgval_t(literal_pointer_val(jv), true, typ); } @@ -708,8 +720,8 @@ static void alloc_local(jl_sym_t *s, jl_codectx_t *ctx) jl_varinfo_t &vi = ctx->vars[s]; jl_value_t *jt = vi.value.typ; assert(store_unboxed_p(s,ctx)); - Type *vtype = julia_struct_to_llvm(jt); - assert(vtype != jl_pvalue_llvmt); + Type *vtype = julia_type_to_llvm(jt); + assert(vtype != T_pjlvalue); if (type_is_ghost(vtype)) { vi.value = ghostValue(jt); return; @@ -1834,7 +1846,7 @@ static jl_cgval_t emit_boxed_rooted(jl_value_t *e, jl_codectx_t *ctx) // TODO: m if (!v.isboxed) { Value *vbox = boxed(v, ctx); make_gcroot(vbox, ctx); - v = jl_cgval_t(vbox, v.typ); // bypass normal auto-unbox behavior for isghost + v = jl_cgval_t(vbox, true, v.typ); // XXX: bypasses the normal auto-unbox behavior for isghost! } else if (might_need_root(e)) { // TODO: v.needsgcroot make_gcroot(v.V, ctx); @@ -1959,7 +1971,7 @@ static jl_cgval_t emit_getfield(jl_value_t *expr, jl_sym_t *name, jl_codectx_t * if (bnd && bnd->value != NULL) { if (bnd->constp && jl_isbits(jl_typeof(bnd->value))) return emit_unboxed(bnd->value, ctx); - return mark_julia_type(builder.CreateLoad(bp), bnd->constp ? jl_typeof(bnd->value) : (jl_value_t*)jl_any_type); + return mark_julia_type(builder.CreateLoad(bp), true, bnd->constp ? jl_typeof(bnd->value) : (jl_value_t*)jl_any_type); } // todo: use type info to avoid undef check return emit_checked_var(bp, name, ctx); @@ -1998,7 +2010,7 @@ static jl_cgval_t emit_getfield(jl_value_t *expr, jl_sym_t *name, jl_codectx_t * ConstantInt::get(T_int32,2)); #endif ctx->gc.argDepth = argStart; - return mark_julia_type(result, jl_any_type); // (typ will be patched up by caller) + return mark_julia_type(result, true, jl_any_type); // (typ will be patched up by caller) } static Value *emit_bits_compare(const jl_cgval_t &arg1, const jl_cgval_t &arg2, jl_codectx_t *ctx) @@ -2023,7 +2035,7 @@ static Value *emit_bits_compare(const jl_cgval_t &arg1, const jl_cgval_t &arg2, Value *subAns, *fld1, *fld2; fld1 = builder.CreateExtractElement(varg1, ConstantInt::get(T_int32,i)), fld2 = builder.CreateExtractElement(varg2, ConstantInt::get(T_int32,i)), - subAns = emit_bits_compare(mark_julia_type(fld1, fldty), mark_julia_type(fld2, fldty), ctx); + subAns = emit_bits_compare(mark_julia_type(fld1, false, fldty), mark_julia_type(fld2, false, fldty), ctx); answer = builder.CreateAnd(answer, subAns); } return answer; @@ -2187,7 +2199,7 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, if (rt1) { rt2 = static_eval(args[2], ctx, true); if (rt2) { - *ret = mark_julia_type(ConstantInt::get(T_int1, jl_egal(rt1, rt2)), jl_bool_type); + *ret = mark_julia_type(ConstantInt::get(T_int1, jl_egal(rt1, rt2)), false, jl_bool_type); JL_GC_POP(); return true; } @@ -2206,7 +2218,7 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, // emit comparison test Value *ans = emit_f_is(v1, v2, ctx); ctx->gc.argDepth = last_depth; - *ret = mark_julia_type(ans, jl_bool_type); + *ret = mark_julia_type(ans, false, jl_bool_type); JL_GC_POP(); return true; } @@ -2214,7 +2226,7 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, else if (f->fptr == &jl_f_typeof && nargs==1) { jl_cgval_t arg1 = emit_expr(args[1], ctx); Value *lty = emit_typeof(arg1); - *ret = mark_julia_type(lty, jl_datatype_type); + *ret = mark_julia_type(lty, true, jl_datatype_type); JL_GC_POP(); return true; } @@ -2273,13 +2285,13 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, if (jl_is_type_type(ty) && !jl_has_typevars(jl_tparam0(ty))) { jl_value_t *tp0 = jl_tparam0(ty); if (jl_subtype(arg, tp0, 0)) { - *ret = mark_julia_type(ConstantInt::get(T_int1,1), jl_bool_type); + *ret = mark_julia_type(ConstantInt::get(T_int1, 1), false, jl_bool_type); JL_GC_POP(); return true; } if (!jl_subtype(tp0, (jl_value_t*)jl_type_type, 0)) { if (jl_is_leaf_type(arg)) { - *ret = mark_julia_type(ConstantInt::get(T_int1,0), jl_bool_type); + *ret = mark_julia_type(ConstantInt::get(T_int1, 0), false, jl_bool_type); JL_GC_POP(); return true; } @@ -2288,6 +2300,7 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, *ret = mark_julia_type( builder.CreateICmpEQ(emit_typeof(arg1), literal_pointer_val(tp0)), + false, jl_bool_type); JL_GC_POP(); return true; @@ -2303,7 +2316,7 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, jl_is_type_type(rt2) && !jl_is_typevar(jl_tparam0(rt2))) { int issub = jl_subtype(jl_tparam0(rt1), jl_tparam0(rt2), 0); // TODO: emit args[1] and args[2] in case of side effects? - *ret = mark_julia_type(ConstantInt::get(T_int1, issub), jl_bool_type); + *ret = mark_julia_type(ConstantInt::get(T_int1, issub), false, jl_bool_type); JL_GC_POP(); return true; } @@ -2334,7 +2347,7 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, ConstantInt::get(T_size, ctx->nReqArgs)), nva); #endif - *ret = mark_julia_type(r, expr_type(expr, ctx)); + *ret = mark_julia_type(r, true, expr_type(expr, ctx)); JL_GC_POP(); return true; } @@ -2369,7 +2382,7 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, if (jl_is_array_type(aty)) { // todo: also allow e.g. Union of several array types jl_cgval_t arg1 = emit_expr(args[1], ctx); - *ret = mark_julia_type(emit_arraylen(arg1, args[1], ctx), jl_long_type); + *ret = mark_julia_type(emit_arraylen(arg1, args[1], ctx), false, jl_long_type); JL_GC_POP(); return true; } @@ -2386,12 +2399,12 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, if (jl_is_long(args[2])) { uint32_t idx = (uint32_t)jl_unbox_long(args[2]); if (idx > 0 && idx <= ndims) { - *ret = mark_julia_type(emit_arraysize(ary, args[1], idx, ctx), jl_long_type); + *ret = mark_julia_type(emit_arraysize(ary, args[1], idx, ctx), false, jl_long_type); JL_GC_POP(); return true; } else if (idx > ndims) { - *ret = mark_julia_type(ConstantInt::get(T_size, 1), jl_long_type); + *ret = mark_julia_type(ConstantInt::get(T_size, 1), false, jl_long_type); JL_GC_POP(); return true; } @@ -2420,7 +2433,7 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, PHINode *result = builder.CreatePHI(T_size, 2); result->addIncoming(v_one, outBB); result->addIncoming(v_sz, inBB); - *ret = mark_julia_type(result, jl_long_type); + *ret = mark_julia_type(result, false, jl_long_type); JL_GC_POP(); return true; } @@ -2505,10 +2518,10 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, Value *own_ptr = builder.CreateLoad( builder.CreateBitCast(builder.CreateConstGEP1_32( builder.CreateBitCast(ary.V,T_pint8), jl_array_data_owner_offset(nd)), - jl_ppvalue_llvmt)); + T_ppjlvalue)); builder.CreateBr(mergeBB); builder.SetInsertPoint(mergeBB); - data_owner = builder.CreatePHI(jl_pvalue_llvmt, 2); + data_owner = builder.CreatePHI(T_pjlvalue, 2); data_owner->addIncoming(ary.V, curBB); data_owner->addIncoming(own_ptr, ownedBB); } @@ -2541,11 +2554,12 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, Value *idx = emit_unbox(T_size, emit_unboxed(args[2], ctx), fldt); idx = emit_bounds_check( - jl_cgval_t(builder.CreateGEP(ctx->argArray, ConstantInt::get(T_size, ctx->nReqArgs)), NULL), + jl_cgval_t(builder.CreateGEP(ctx->argArray, ConstantInt::get(T_size, ctx->nReqArgs)), false, NULL), NULL, idx, valen, ctx); idx = builder.CreateAdd(idx, ConstantInt::get(T_size, ctx->nReqArgs)); *ret = mark_julia_type( - tbaa_decorate(tbaa_user,builder.CreateLoad(builder.CreateGEP(ctx->argArray,idx))), + tbaa_decorate(tbaa_user, builder.CreateLoad(builder.CreateGEP(ctx->argArray, idx))), + true, expr_type(expr, ctx)); JL_GC_POP(); return true; @@ -2606,7 +2620,7 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, else if (f->fptr == &jl_f_nfields && nargs==1) { if (ctx->vaStack && symbol_eq(args[1], ctx->vaName) && !ctx->vars[ctx->vaName].isAssigned) { - *ret = mark_julia_type(emit_n_varargs(ctx), jl_long_type); + *ret = mark_julia_type(emit_n_varargs(ctx), false, jl_long_type); JL_GC_POP(); return true; } @@ -2616,7 +2630,7 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, if (jl_is_leaf_type(tp0)) { emit_expr(args[1], ctx); assert(jl_is_datatype(tp0)); - *ret = mark_julia_type(ConstantInt::get(T_size, jl_datatype_nfields(tp0)), jl_long_type); + *ret = mark_julia_type(ConstantInt::get(T_size, jl_datatype_nfields(tp0)), false, jl_long_type); JL_GC_POP(); return true; } @@ -2628,7 +2642,7 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, sz = emit_datatype_nfields(arg1); else sz = ConstantInt::get(T_size, jl_datatype_nfields(aty)); - *ret = mark_julia_type(sz, jl_long_type); + *ret = mark_julia_type(sz, false, jl_long_type); JL_GC_POP(); return true; } @@ -2645,8 +2659,8 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, Value *types_len = emit_datatype_nfields(ty); Value *idx = emit_unbox(T_size, emit_unboxed(args[2], ctx), (jl_value_t*)jl_long_type); emit_bounds_check(ty, (jl_value_t*)jl_datatype_type, idx, types_len, ctx); - Value *fieldtyp = builder.CreateLoad(builder.CreateGEP(builder.CreateBitCast(types_svec, jl_ppvalue_llvmt), idx)); - *ret = mark_julia_type(fieldtyp, expr_type(expr, ctx)); + Value *fieldtyp = builder.CreateLoad(builder.CreateGEP(builder.CreateBitCast(types_svec, T_ppjlvalue), idx)); + *ret = mark_julia_type(fieldtyp, true, expr_type(expr, ctx)); JL_GC_POP(); return true; } @@ -2666,7 +2680,7 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, sty != jl_datatype_type) { if (jl_is_leaf_type((jl_value_t*)sty) || (sty->name->names == jl_emptysvec && sty->size > 0)) { - *ret = mark_julia_type(ConstantInt::get(T_size, sty->size), jl_long_type); + *ret = mark_julia_type(ConstantInt::get(T_size, sty->size), false, jl_long_type); JL_GC_POP(); return true; } @@ -2705,7 +2719,7 @@ static Value *emit_jlcall(Value *theFptr, Value *theF, int argStart, if (nargs > 0) myargs = emit_temp_slot(argStart, ctx); else - myargs = Constant::getNullValue(jl_ppvalue_llvmt); + myargs = Constant::getNullValue(T_ppjlvalue); #ifdef LLVM37 Value *result = builder.CreateCall(prepare_call(theFptr), {theF, myargs, ConstantInt::get(T_int32,nargs)}); @@ -2735,9 +2749,11 @@ static jl_cgval_t emit_call_function_object(jl_function_t *f, Value *theF, Value jl_value_t **args, size_t nargs, jl_codectx_t *ctx) { - if (f!=NULL && specialized && f->linfo!=NULL && f->linfo->specFunctionObject!=NULL) { + if (f!=NULL && specialized && f->linfo!=NULL && f->linfo->specFunctionObject != NULL) { // emit specialized call site jl_value_t *jlretty = jl_ast_rettype(f->linfo, f->linfo->ast); + bool retboxed; + (void)julia_type_to_llvm(jlretty, &retboxed); Function *cf = (Function*)f->linfo->specFunctionObject; FunctionType *cft = cf->getFunctionType(); size_t nfargs = cft->getNumParams(); @@ -2753,13 +2769,15 @@ static jl_cgval_t emit_call_function_object(jl_function_t *f, Value *theF, Value for(size_t i=0; i < nargs; i++) { Type *at = cft->getParamType(idx); jl_value_t *jt = jl_nth_slot_type(f->linfo->specTypes,i); - Type *et = julia_type_to_llvm(jt); + bool isboxed; + Type *et = julia_type_to_llvm(jt, &isboxed); if (type_is_ghost(et)) { // Still emit the expression in case it has side effects emit_expr(args[i+1], ctx); continue; } - if (at == jl_pvalue_llvmt) { + if (isboxed) { + assert(at == T_pjlvalue && et == T_pjlvalue); jl_cgval_t origval = emit_expr(args[i+1], ctx); argvals[idx] = boxed(origval, ctx,expr_type(args[i+1],ctx)); assert(dyn_cast(argvals[idx]) == 0); @@ -2796,9 +2814,9 @@ static jl_cgval_t emit_call_function_object(jl_function_t *f, Value *theF, Value assert(idx == nfargs); CallInst *call = builder.CreateCall(prepare_call(cf), ArrayRef(&argvals[0], nfargs)); call->setAttributes(cf->getAttributes()); - return sret ? mark_julia_slot(result, jlretty) : mark_julia_type(call, jlretty); + return sret ? mark_julia_slot(result, jlretty) : mark_julia_type(call, retboxed, jlretty); } - return mark_julia_type(emit_jlcall(theFptr, theF, &args[1], nargs, ctx), jl_any_type); // (typ will be patched up by caller) + return mark_julia_type(emit_jlcall(theFptr, theF, &args[1], nargs, ctx), true, jl_any_type); // (typ will be patched up by caller) } static Value *emit_is_function(Value *x, jl_codectx_t *ctx) @@ -2915,7 +2933,7 @@ static jl_cgval_t emit_call(jl_value_t **args, size_t arglen, jl_codectx_t *ctx, if (nargs > 0) myargs = emit_temp_slot(argStart + 1, ctx); // argStart holds theFunc, argStart + 1 holds the start of the argument list else - myargs = Constant::getNullValue(jl_ppvalue_llvmt); // no arguments + myargs = Constant::getNullValue(T_ppjlvalue); // no arguments theFptr = emit_nthptr_recast(theFunc, (ssize_t)(offsetof(jl_function_t,fptr)/sizeof(void*)), tbaa_func, jl_pfptr_llvmt); #ifdef LLVM37 Value *r1 = builder.CreateCall(prepare_call(theFptr), {theFunc, myargs, @@ -2933,7 +2951,7 @@ static jl_cgval_t emit_call(jl_value_t **args, size_t arglen, jl_codectx_t *ctx, Value *r2; if (!jl_is_gf(call_func)) { just_emit_error("\"call\" is not a generic function", ctx); - r2 = UndefValue::get(jl_pvalue_llvmt); + r2 = UndefValue::get(T_pjlvalue); } else { #ifdef LLVM37 @@ -2951,10 +2969,10 @@ static jl_cgval_t emit_call(jl_value_t **args, size_t arglen, jl_codectx_t *ctx, builder.CreateBr(mergeBB1); ctx->f->getBasicBlockList().push_back(mergeBB1); builder.SetInsertPoint(mergeBB1); - PHINode *ph = builder.CreatePHI(jl_pvalue_llvmt, 2); + PHINode *ph = builder.CreatePHI(T_pjlvalue, 2); ph->addIncoming(r1, funcBB1); ph->addIncoming(r2, elseBB1); - result = mark_julia_type(ph, jl_any_type); + result = mark_julia_type(ph, true, jl_any_type); } if (result.typ == (jl_value_t*)jl_any_type) result = remark_julia_type(result, expr_type(expr, ctx)); // patch up typ if necessary @@ -3000,9 +3018,9 @@ static Value *global_binding_pointer(jl_module_t *m, jl_sym_t *s, b = jl_get_binding(m, s); if (b == NULL) { // var not found. switch to delayed lookup. - Constant *initnul = ConstantPointerNull::get((PointerType*)jl_pvalue_llvmt); + Constant *initnul = ConstantPointerNull::get((PointerType*)T_pjlvalue); GlobalVariable *bindinggv = - new GlobalVariable(*jl_Module, jl_pvalue_llvmt, + new GlobalVariable(*jl_Module, T_pjlvalue, false, GlobalVariable::PrivateLinkage, initnul, "delayedvar"); Value *cachedval = builder.CreateLoad(bindinggv); @@ -3025,10 +3043,10 @@ static Value *global_binding_pointer(jl_module_t *m, jl_sym_t *s, builder.CreateBr(have_val); ctx->f->getBasicBlockList().push_back(have_val); builder.SetInsertPoint(have_val); - PHINode *p = builder.CreatePHI(jl_pvalue_llvmt, 2); + PHINode *p = builder.CreatePHI(T_pjlvalue, 2); p->addIncoming(cachedval, currentbb); p->addIncoming(bval, not_found); - return julia_binding_gv(builder.CreateBitCast(p, jl_ppvalue_llvmt)); + return julia_binding_gv(builder.CreateBitCast(p, T_ppjlvalue)); } if (b->deprecated) cg_bdw(b, ctx); } @@ -3038,14 +3056,10 @@ static Value *global_binding_pointer(jl_module_t *m, jl_sym_t *s, static jl_cgval_t emit_checked_var(Value *bp, jl_sym_t *name, jl_codectx_t *ctx, bool isvol) { - assert(bp->getType() == jl_ppvalue_llvmt); + assert(bp->getType() == T_ppjlvalue); Value *v = builder.CreateLoad(bp, isvol); - // in unreachable code, there might be a poorly-typed instance of a variable - // that has a concrete type everywhere it's actually used. tolerate this - // situation by just skipping the NULL check if it wouldn't be valid. (issue #7836) - if (v->getType() == jl_pvalue_llvmt) - undef_var_error_if_null(v, name, ctx); - return mark_julia_type(v, jl_any_type); + undef_var_error_if_null(v, name, ctx); + return mark_julia_type(v, true, jl_any_type); } static jl_cgval_t emit_var(jl_sym_t *sym, jl_codectx_t *ctx, bool isboxed) @@ -3067,11 +3081,13 @@ static jl_cgval_t emit_var(jl_sym_t *sym, jl_codectx_t *ctx, bool isboxed) if (jbp->constp) { if (!isboxed && jl_isbits(jl_typeof(jbp->value))) return emit_unboxed(jbp->value, ctx); + else + return mark_julia_const(jbp->value); } // double-check that a global variable is actually defined. this // can be a problem in parallel when a definition is missing on // one machine. - return mark_julia_type(builder.CreateLoad(bp), jbp->constp ? jl_typeof(jbp->value) : (jl_value_t*)jl_any_type); + return mark_julia_type(builder.CreateLoad(bp), true, jl_any_type); } return emit_checked_var(bp, sym, ctx); } @@ -3085,7 +3101,7 @@ static jl_cgval_t emit_var(jl_sym_t *sym, jl_codectx_t *ctx, bool isboxed) // if the jl_box_t in the closure env, it will be const in the function load = tbaa_decorate(tbaa_const, load); } - bp = builder.CreatePointerCast(load, jl_ppvalue_llvmt); + bp = builder.CreatePointerCast(load, T_ppjlvalue); } if (vi.isArgument || // arguments are always defined ((vi.closureidx == -1 || !vi.isAssigned) && !vi.usedUndef)) { @@ -3095,7 +3111,7 @@ static jl_cgval_t emit_var(jl_sym_t *sym, jl_codectx_t *ctx, bool isboxed) // if it's in the closure env, but only assigned by the parent function, it will be const while in the child function v = tbaa_decorate(tbaa_const, v); } - return mark_julia_type(v, vi.value.typ); + return mark_julia_type(v, true, vi.value.typ); } else { jl_cgval_t v = emit_checked_var(bp, sym, ctx, vi.isVolatile); @@ -3110,7 +3126,7 @@ static jl_cgval_t emit_var(jl_sym_t *sym, jl_codectx_t *ctx, bool isboxed) Value *v = vi.value.V; if (v->getType() != T) v = builder.CreatePointerCast(v, T); - return mark_julia_type(builder.CreateLoad(v, true), vi.value.typ); + return mark_julia_type(builder.CreateLoad(v, true), false, vi.value.typ); } else { return vi.value; @@ -3127,8 +3143,8 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx) jl_value_t *declType = (jl_is_array(gensym_types) ? jl_cellref(gensym_types, idx) : (jl_value_t*)jl_any_type); jl_cgval_t slot; // slot is a jl_value_t or jl_value_t* if (store_unboxed_p(declType)) { - Type *vtype = julia_struct_to_llvm(declType); - assert(vtype != jl_pvalue_llvmt); + Type *vtype = julia_type_to_llvm(declType); + assert(vtype != T_pjlvalue); if (type_is_ghost(vtype)) { slot = emit_expr(r, ctx); } @@ -3137,6 +3153,7 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx) if (slot.ispointer) { // emit a copy of boxed isbits values. TODO: elid this copy if unnecessary slot = mark_julia_type( emit_unbox(julia_type_to_llvm(declType), slot, declType), + false, declType); } } @@ -3148,7 +3165,7 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx) Value *bp = emit_local_slot(ctx->gc.argSpaceSize++, ctx); builder.CreateStore(rval, bp); } - slot = mark_julia_type(rval, declType); + slot = mark_julia_type(rval, true, declType); } ctx->gensym_SAvalues.at(idx) = slot; // now gensym_SAvalues[idx] contains the SAvalue assert(ctx->gensym_assigned.at(idx) = true); // (assignment, not comparison test) @@ -3219,7 +3236,7 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx) return; } if (vi.isBox) { - bp = builder.CreatePointerCast(builder.CreateLoad(bp), jl_ppvalue_llvmt); + bp = builder.CreatePointerCast(builder.CreateLoad(bp), T_ppjlvalue); } builder.CreateStore(rval, bp, vi.isVolatile); if (vi.isBox) { @@ -3233,6 +3250,7 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx) if (store_unboxed_p(vi.value.typ) && rval_info.ispointer) { // emit a copy of boxed isbits values. TODO: elid this copy if unnecessary rval_info = mark_julia_type( emit_unbox(julia_type_to_llvm(vi.value.typ), rval_info, vi.value.typ), + false, vi.value.typ); } vi.value = rval_info; @@ -3343,11 +3361,10 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx, bool isboxed, b jl_binding_t *b = jl_get_binding(mod, var); if (b == NULL) b = jl_get_binding_wr(mod, var); - Value *bp = julia_binding_gv(b); - if (b->constp && b->value!=NULL) { - return mark_julia_type(builder.CreateLoad(bp), jl_typeof(b->value)); + if (b->constp && b->value != NULL) { + return mark_julia_const(b->value); } - return emit_checked_var(bp, var, ctx); + return emit_checked_var(julia_binding_gv(b), var, ctx); } if (jl_is_newvarnode(expr)) { assert(!valuepos); @@ -3361,14 +3378,14 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx, bool isboxed, b if (vi.isBox) { builder.CreateStore(builder.CreateCall(prepare_call(jlbox_func), V_null), lv); } - else if (lv->getType() == jl_ppvalue_llvmt && vi.usedUndef) { + else if (vi.usedUndef) { builder.CreateStore(V_null, lv); } } return jl_cgval_t(); } if (jl_is_lambda_info(expr)) { - return mark_julia_type(emit_lambda_closure(expr, ctx), jl_function_type); + return mark_julia_type(emit_lambda_closure(expr, ctx), true, jl_function_type); } if (!jl_is_expr(expr)) { int needroot = true; @@ -3476,8 +3493,8 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx, bool isboxed, b bp = vi.memloc; if (vi.isBox) { // bp is a jl_box_t* - bp = builder.CreatePointerCast(builder.CreateLoad(bp), jl_ppvalue_llvmt); - bp_owner = builder.CreateBitCast(bp, jl_pvalue_llvmt); + bp = builder.CreatePointerCast(builder.CreateLoad(bp), T_ppjlvalue); + bp_owner = builder.CreateBitCast(bp, T_pjlvalue); } } } @@ -3485,6 +3502,7 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx, bool isboxed, b Value *mdargs[4] = { name, bp, bp_owner, literal_pointer_val(bnd) }; return mark_julia_type( builder.CreateCall(prepare_call(jlgenericfunction_func), ArrayRef(&mdargs[0], 4)), + true, jl_function_type); } else { @@ -3499,6 +3517,7 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx, bool isboxed, b ctx->gc.argDepth = last_depth; return mark_julia_type( builder.CreateCall(prepare_call(jlmethod_func), ArrayRef(&mdargs[0], 9)), + true, jl_function_type); } } @@ -3539,10 +3558,10 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx, bool isboxed, b } Value *typ = boxed(emit_expr(args[0], ctx), ctx); Value *val = emit_jlcall(jlnew_func, typ, &args[1], nargs-1, ctx); - return mark_julia_type(val, ty); + return mark_julia_type(val, true, ty); } else if (head == exc_sym) { // *jl_exception_in_transit - return mark_julia_type(builder.CreateLoad(prepare_global(jlexc_var), /*isvolatile*/true), jl_any_type); + return mark_julia_type(builder.CreateLoad(prepare_global(jlexc_var), /*isvolatile*/true), true, jl_any_type); } else if (head == leave_sym) { assert(jl_is_long(args[0])); @@ -3620,7 +3639,7 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx, bool isboxed, b } } jl_cgval_t ast = emit_expr(arg, ctx); - return mark_julia_type(builder.CreateCall(prepare_call(jlcopyast_func), boxed(ast, ctx)), ast.typ); + return mark_julia_type(builder.CreateCall(prepare_call(jlcopyast_func), boxed(ast, ctx)), true, ast.typ); } else if (head == simdloop_sym) { if (!llvm::annotateSimdLoop(builder.GetInsertBlock())) @@ -3673,7 +3692,7 @@ static void allocate_gc_frame(size_t n_roots, BasicBlock *b0, jl_codectx_t *ctx) gc->argDepth = 0; gc->maxDepth = 0; - gc->gcframe = builder.CreateAlloca(jl_pvalue_llvmt, ConstantInt::get(T_int32, 0)); + gc->gcframe = builder.CreateAlloca(T_pjlvalue, ConstantInt::get(T_int32, 0)); #ifdef JL_DEBUG_BUILD gc->gcframe->setName("gcrootframe"); #endif @@ -3719,7 +3738,7 @@ emit_gcpops(jl_codectx_t *ctx) Instruction *gcpop = (Instruction*)builder.CreateConstGEP1_32(ctx->gc.gcframe, 1); builder.CreateStore(builder.CreatePointerCast(builder.CreateLoad(gcpop), - jl_ppvalue_llvmt), + T_ppjlvalue), prepare_global(jlpgcstack_var)); } } @@ -3743,7 +3762,7 @@ static void finalize_gc_frame(jl_codectx_t *ctx) builder.CreateStore(ConstantInt::get(T_size, (gc->argSpaceSize + gc->maxDepth) << 1), builder.CreateBitCast(builder.CreateConstGEP1_32(newgcframe, 0), T_psize)); builder.CreateStore(builder.CreateLoad(prepare_global(jlpgcstack_var)), - builder.CreatePointerCast(builder.CreateConstGEP1_32(newgcframe, 1), PointerType::get(jl_ppvalue_llvmt,0))); + builder.CreatePointerCast(builder.CreateConstGEP1_32(newgcframe, 1), PointerType::get(T_ppjlvalue,0))); builder.CreateStore(newgcframe, prepare_global(jlpgcstack_var)); // Initialize the slots for temporary variables to NULL for (int i = 0; i < gc->argSpaceSize; i++) { @@ -3770,7 +3789,8 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t } } // Generate a c-callable wrapper - Type *crt = ((isref&1) ? jl_pvalue_llvmt : julia_struct_to_llvm(jlrettype)); + bool toboxed; + Type *crt = ((isref & 1) ? T_pjlvalue : julia_struct_to_llvm(jlrettype, &toboxed)); if (crt == NULL) jl_error("cfunction: return type doesn't correspond to a C type"); size_t i; @@ -3783,6 +3803,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t } std::vector fargt(0); + std::vector fargt_isboxed(0); std::vector fargt_sig(0); Type* fargt_vasig; std::vector inRegList(0); @@ -3790,7 +3811,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t attr_type attrs; Type *prt = NULL; int sret = 0; - std::string err_msg = generate_func_sig(&crt, &prt, sret, fargt, fargt_sig, fargt_vasig, inRegList, byRefList, attrs, + std::string err_msg = generate_func_sig(&crt, &prt, sret, fargt, fargt_isboxed, fargt_sig, fargt_vasig, inRegList, byRefList, attrs, ((isref&1) ? (jl_value_t*)jl_any_type : jlrettype), argt->parameters, nargs); if (!err_msg.empty()) jl_error(err_msg.c_str()); @@ -3889,96 +3910,102 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t for (size_t i = 0; i < nargs; i++) { Value *val = AI++; jl_value_t *jargty = jl_nth_slot_type(lam->specTypes, i); + bool isboxed, argboxed; + Type *t = julia_type_to_llvm(jargty, &isboxed); + (void)julia_struct_to_llvm(jargty, &argboxed); + jl_cgval_t inputarg; // figure out how to unpack this type if (isref & (2<getPointerTo()); val = builder.CreateAlignedLoad(val, 1); // make no alignment assumption about pointer from C + inputarg = mark_julia_type(val, false, jargty); } } } + else if (argboxed) { + inputarg = mark_julia_type(val, true, jargty); + } else { // undo whatever we might have done to this poor argument bool issigned = jl_signed_type && jl_subtype(jargty, (jl_value_t*)jl_signed_type, 0); val = llvm_type_rewrite(val, val->getType(), fargt[i], true, byRefList[i], issigned, &ctx); + if (isboxed) { + Value *mem = emit_allocobj(jl_datatype_size(jargty)); + builder.CreateStore(literal_pointer_val((jl_value_t*)jargty), + emit_typeptr_addr(mem)); + builder.CreateAlignedStore(val, + builder.CreateBitCast(mem, val->getType()->getPointerTo()), + 16); // julia's gc gives 16-byte aligned addresses + inputarg = mark_julia_type(mem, true, jargty); + } + else { + inputarg = mark_julia_type(val, false, jargty); + } } // figure out how to repack this type - Type *at = specsig ? theFptr->getFunctionType()->getParamType(FParamIndex++) : jl_pvalue_llvmt; - if (val->getType() != at) { - if (at == jl_pvalue_llvmt) { - assert(jl_is_leaf_type(jargty)); - if (jl_datatype_nfields(jargty) == 0) { - val = literal_pointer_val(jl_new_struct_uninit((jl_datatype_t*)jargty)); - } - else if (jl_isbits(jargty)) { - val = boxed(mark_julia_type(val, jargty), &ctx, jargty); - } - else { - Value *mem = emit_allocobj(jl_datatype_size(jargty)); - builder.CreateStore(literal_pointer_val((jl_value_t*)jargty), - emit_typeptr_addr(mem)); - builder.CreateAlignedStore(val, builder.CreateBitCast(mem, val->getType()->getPointerTo()), 16); // julia's gc gives 16-byte aligned addresses - val = mem; - } - if (specsig) - make_gcroot(val, &ctx); - } - else if (val->getType()->isAggregateType()) { - assert(at->isPointerTy() && at->getContainedType(0) == val->getType()); - // aggregate types are passed by pointer - Value *loc = emit_static_alloca(val->getType(), &ctx); - builder.CreateStore(val, loc); - val = loc; + if (!specsig) { + Value *arg = boxed(inputarg, &ctx); + make_gcroot(arg, &ctx); + } + else { + Value *arg; + Type *at = theFptr->getFunctionType()->getParamType(FParamIndex++); + if (isboxed) { + arg = boxed(inputarg, &ctx); + make_gcroot(arg, &ctx); } else { - val = emit_unbox(at, mark_julia_type(val, jargty), jargty); - assert(dyn_cast(val) == 0); + arg = emit_unbox(t, inputarg, jargty); + assert(!isa(arg)); + if (t->isAggregateType()) { + assert(at->isPointerTy() && at->getContainedType(0) == t); + // aggregate types are passed by pointer + Value *loc = emit_static_alloca(t, &ctx); + builder.CreateStore(arg, loc); + arg = loc; + } } - } - // add to argument list - if (specsig) - args.push_back(val); - else - make_gcroot(val, &ctx); + // add to argument list + args.push_back(arg); + } } // Create the call - Value *r; + jl_cgval_t retval; if (specsig) { + bool retboxed; CallInst *call = builder.CreateCall(prepare_call(theFptr), ArrayRef(args)); call->setAttributes(theFptr->getAttributes()); - if (jlfunc_sret) - r = builder.CreateLoad(result); - else - r = call; + (void)julia_type_to_llvm(jlrettype, &retboxed); + retval = mark_julia_type(jlfunc_sret ? (Value*)builder.CreateLoad(result) : (Value*)call, retboxed, jlrettype); } else { - r = emit_jlcall(theFptr, literal_pointer_val((jl_value_t*)ff), 0, nargs, &ctx); + Value *ret = emit_jlcall(theFptr, literal_pointer_val((jl_value_t*)ff), 0, nargs, &ctx); + retval = mark_julia_type(ret, true, jlrettype); } // Prepare the return value - if (isref&1) { + Value *r; + if (isref & 1) { assert(!sret); // return a jl_value_t* - if (r->getType() != jl_pvalue_llvmt) { - r = boxed(mark_julia_type(r, jlrettype), &ctx, jlrettype); - } + r = boxed(retval, &ctx); } else if (sret && jlfunc_sret) { // nothing to do @@ -3987,7 +4014,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t if (sret) prt = fargt_sig[0]->getContainedType(0); // sret is a PointerType bool issigned = jl_signed_type && jl_subtype(jlrettype, (jl_value_t*)jl_signed_type, 0); - Value *v = julia_to_native(crt, jlrettype, mark_julia_type(r, jlrettype), + Value *v = julia_to_native(crt, toboxed, jlrettype, retval, false, false, false, false, false, 0, &ctx, NULL); r = llvm_type_rewrite(v, crt, prt, false, false, issigned, &ctx); if (sret) @@ -4088,13 +4115,14 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Funct } for(size_t i=0; i < nargs; i++) { jl_value_t *ty = jl_nth_slot_type(lam->specTypes, i); - Type *lty = julia_type_to_llvm(ty); + bool isboxed; + Type *lty = julia_type_to_llvm(ty, &isboxed); if (lty != NULL && type_is_ghost(lty)) continue; Value *argPtr = builder.CreateGEP(argArray, ConstantInt::get(T_size, i)); Value *theArg = builder.CreateLoad(argPtr); - if (lty != NULL && lty != jl_pvalue_llvmt) { + if (lty != NULL && !isboxed) { theArg = builder.CreatePointerCast(theArg, PointerType::get(lty,0)); if (!lty->isAggregateType()) // keep "aggregate" type values in place as pointers theArg = builder.CreateLoad(theArg); @@ -4107,17 +4135,13 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Funct // wrappers can be reused for different functions of the same type. CallInst *call = builder.CreateCall(prepare_call(f), ArrayRef(&args[0], nfargs)); call->setAttributes(f->getAttributes()); - Value *r; - if (sret || call->getType() != jl_pvalue_llvmt) { - jl_value_t *ty = jl_ast_rettype(lam, (jl_value_t*)ast); - jl_cgval_t r_typed = sret ? mark_julia_slot(result, ty) : mark_julia_type(call, ty); - r = boxed(r_typed, &ctx); - } - else { - r = call; - } - builder.CreateRet(r); + jl_value_t *jlretty = jl_ast_rettype(lam, (jl_value_t*)ast); + bool retboxed; + (void)julia_type_to_llvm(jlretty, &retboxed); + if (sret) { assert(!retboxed); } + jl_cgval_t retval = sret ? mark_julia_slot(result, jlretty) : mark_julia_type(call, retboxed, jlretty); + builder.CreateRet(boxed(retval, &ctx)); finalize_gc_frame(&ctx); FPM->run(*w); @@ -4181,13 +4205,13 @@ static Function *emit_function(jl_lambda_info_t *lam) jl_sym_t *argname = jl_decl_var(arg); jl_varinfo_t &varinfo = ctx.vars[argname]; varinfo.isArgument = true; - jl_value_t *ty = lam->specTypes ? jl_nth_slot_type(lam->specTypes, i) : (jl_value_t*)jl_any_type; - varinfo.value = mark_julia_type((Value*)NULL, ty); + jl_value_t *ty = jl_nth_slot_type(lam->specTypes, i); + varinfo.value = mark_julia_type((Value*)NULL, false, ty); } if (va) { jl_varinfo_t &varinfo = ctx.vars[ctx.vaName]; varinfo.isArgument = true; - varinfo.value = mark_julia_type((Value*)NULL, jl_tuple_type); + varinfo.value = mark_julia_type((Value*)NULL, false, jl_tuple_type); } for(i=0; i < vinfoslen; i++) { @@ -4208,7 +4232,7 @@ static Function *emit_function(jl_lambda_info_t *lam) jl_value_t *typ = jl_cellref(vi,1); if (!jl_is_type(typ)) typ = (jl_value_t*)jl_any_type; - varinfo.value = mark_julia_type((Value*)NULL, typ); + varinfo.value = mark_julia_type((Value*)NULL, false, typ); } bool hasCapt = (captvinfoslen > 0); for(i=0; i < captvinfoslen; i++) { @@ -4228,7 +4252,7 @@ static Function *emit_function(jl_lambda_info_t *lam) jl_value_t *typ = jl_cellref(vi,1); if (!jl_is_type(typ)) typ = (jl_value_t*)jl_any_type; - varinfo.value = mark_julia_type((Value*)NULL, typ); + varinfo.value = mark_julia_type((Value*)NULL, false, typ); varinfo.hasGCRoot = true; } @@ -4282,8 +4306,16 @@ static Function *emit_function(jl_lambda_info_t *lam) ctx.sret = false; if (specsig) { // assumes !va std::vector fsig(0); - Type *rt = (jlrettype == (jl_value_t*)jl_void_type ? T_void : julia_type_to_llvm(jlrettype)); - if (rt != jl_pvalue_llvmt && rt != T_void && deserves_sret(jlrettype, rt)) { + Type *rt; + bool retboxed; + if (jlrettype == (jl_value_t*)jl_void_type) { + rt = T_void; + retboxed = false; + } + else { + rt = julia_type_to_llvm(jlrettype, &retboxed); + } + if (!retboxed && rt != T_void && deserves_sret(jlrettype, rt)) { ctx.sret = true; fsig.push_back(rt->getPointerTo()); rt = T_void; @@ -4677,7 +4709,7 @@ static Function *emit_function(jl_lambda_info_t *lam) if (!varinfo.value.isghost) { varinfo.memloc = builder.CreatePointerCast( emit_nthptr_addr(envArg, i + offsetof(jl_svec_t,data) / sizeof(void*)), - jl_ppvalue_llvmt); + T_ppjlvalue); } } } @@ -4778,17 +4810,18 @@ static Function *emit_function(jl_lambda_info_t *lam) if (!vi.value.isghost) { if (specsig) { jl_value_t *argType = jl_nth_slot_type(lam->specTypes, i); - Type *llvmArgType = julia_type_to_llvm(argType); + bool isboxed; + Type *llvmArgType = julia_type_to_llvm(argType, &isboxed); if (type_is_ghost(llvmArgType)) // this argument is not actually passed theArg = ghostValue(argType); else if (llvmArgType->isAggregateType()) theArg = mark_julia_slot(AI++, argType); // this argument is by-pointer else - theArg = mark_julia_type(AI++, argType); + theArg = mark_julia_type(AI++, isboxed, argType); } else { Value *argPtr = builder.CreateGEP(argArray, ConstantInt::get(T_size, i)); - theArg = mark_julia_type(builder.CreateLoad(argPtr), vi.value.typ); + theArg = mark_julia_type(builder.CreateLoad(argPtr), true, vi.value.typ); } Value *lv = vi.memloc; @@ -4998,9 +5031,17 @@ static Function *emit_function(jl_lambda_info_t *lam) if (jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == return_sym) { jl_expr_t *ex = (jl_expr_t*)stmt; Value *retval; - Type *retty = ctx.sret ? f->getFunctionType()->getParamType(0)->getContainedType(0) : f->getReturnType(); - if (retty == jl_pvalue_llvmt) { - retval = boxed(emit_expr(jl_exprarg(ex,0), &ctx, true),&ctx,expr_type(stmt,&ctx)); + bool retboxed; + Type *retty; + if (specsig) { + retty = julia_type_to_llvm(jlrettype, &retboxed); + } + else { + retty = T_pjlvalue; + retboxed = true; + } + if (retboxed) { + retval = boxed(emit_expr(jl_exprarg(ex,0), &ctx, true), &ctx, expr_type(stmt, &ctx)); } else if (!type_is_ghost(retty)) { retval = emit_unbox(retty, @@ -5075,7 +5116,7 @@ static MDNode *tbaa_make_child( const char *name, MDNode *parent, bool isConstan static GlobalVariable *global_to_llvm(const std::string &cname, void *addr, Module *m) { GlobalVariable *gv = - new GlobalVariable(*m, jl_pvalue_llvmt, true, + new GlobalVariable(*m, T_pjlvalue, true, GlobalVariable::ExternalLinkage, NULL, cname); add_named_global(gv, addr); return gv; @@ -5102,9 +5143,17 @@ extern "C" void jl_fptr_to_llvm(void *fptr, jl_lambda_info_t *lam, int specsig) if (specsig) { // assumes !va std::vector fsig(0); jl_value_t *jlrettype = jl_ast_rettype(lam, (jl_value_t*)lam->ast); - Type *rt = (jlrettype == (jl_value_t*)jl_void_type ? T_void : julia_type_to_llvm(jlrettype)); + bool retboxed; + Type *rt; + if (jlrettype == (jl_value_t*)jl_void_type) { + rt = T_void; + retboxed = false; + } + else { + rt = julia_type_to_llvm(jlrettype, &retboxed); + } bool sret = false; - if (rt != jl_pvalue_llvmt && rt != T_void && deserves_sret(jlrettype, rt)) { + if (!retboxed && rt != T_void && deserves_sret(jlrettype, rt)) { sret = true; fsig.push_back(rt->getPointerTo()); rt = T_void; @@ -5174,6 +5223,8 @@ static void init_julia_llvm_env(Module *m) T_int1 = Type::getInt1Ty(getGlobalContext()); T_int8 = Type::getInt8Ty(getGlobalContext()); T_pint8 = PointerType::get(T_int8, 0); + T_ppint8 = PointerType::get(T_pint8, 0); + T_pppint8 = PointerType::get(T_ppint8, 0); T_int16 = Type::getInt16Ty(getGlobalContext()); T_pint16 = PointerType::get(T_int16, 0); T_int32 = Type::getInt32Ty(getGlobalContext()); @@ -5202,7 +5253,7 @@ static void init_julia_llvm_env(Module *m) Type *valueStructElts[1] = { PointerType::getUnqual(valueSt) }; ArrayRef vselts(valueStructElts); valueSt->setBody(vselts); - jl_value_llvmt = valueSt; + T_jlvalue = valueSt; DIBuilder dbuilder(*m); #ifdef LLVM37 @@ -5259,19 +5310,19 @@ static void init_julia_llvm_env(Module *m) dbuilder.getOrCreateArray(diargs)); #endif - jl_pvalue_llvmt = PointerType::get(jl_value_llvmt, 0); - jl_ppvalue_llvmt = PointerType::get(jl_pvalue_llvmt, 0); - two_pvalue_llvmt.push_back(jl_pvalue_llvmt); - two_pvalue_llvmt.push_back(jl_pvalue_llvmt); - three_pvalue_llvmt.push_back(jl_pvalue_llvmt); - three_pvalue_llvmt.push_back(jl_pvalue_llvmt); - three_pvalue_llvmt.push_back(jl_pvalue_llvmt); - V_null = Constant::getNullValue(jl_pvalue_llvmt); + T_pjlvalue = PointerType::get(T_jlvalue, 0); + T_ppjlvalue = PointerType::get(T_pjlvalue, 0); + two_pvalue_llvmt.push_back(T_pjlvalue); + two_pvalue_llvmt.push_back(T_pjlvalue); + three_pvalue_llvmt.push_back(T_pjlvalue); + three_pvalue_llvmt.push_back(T_pjlvalue); + three_pvalue_llvmt.push_back(T_pjlvalue); + V_null = Constant::getNullValue(T_pjlvalue); std::vector ftargs(0); - ftargs.push_back(jl_pvalue_llvmt); - ftargs.push_back(jl_ppvalue_llvmt); + ftargs.push_back(T_pjlvalue); + ftargs.push_back(T_ppjlvalue); ftargs.push_back(T_int32); - jl_func_sig = FunctionType::get(jl_pvalue_llvmt, ftargs, false); + jl_func_sig = FunctionType::get(T_pjlvalue, ftargs, false); assert(jl_func_sig != NULL); jl_pfptr_llvmt = PointerType::get(PointerType::get(jl_func_sig, 0), 0); @@ -5288,7 +5339,7 @@ static void init_julia_llvm_env(Module *m) jl_parray_llvmt = PointerType::get(jl_array_llvmt,0); jlpgcstack_var = - new GlobalVariable(*m, jl_ppvalue_llvmt, + new GlobalVariable(*m, T_ppjlvalue, false, GlobalVariable::ExternalLinkage, NULL, "jl_pgcstack"); add_named_global(jlpgcstack_var, (void*)&jl_pgcstack); @@ -5354,7 +5405,7 @@ static void init_julia_llvm_env(Module *m) add_named_global(jlerror_func, (void*)&jl_error); std::vector args1_(0); - args1_.push_back(jl_pvalue_llvmt); + args1_.push_back(T_pjlvalue); jlthrow_func = Function::Create(FunctionType::get(T_void, args1_, false), Function::ExternalLinkage, @@ -5370,7 +5421,7 @@ static void init_julia_llvm_env(Module *m) add_named_global(jlundefvarerror_func, (void*)&jl_undefined_var_error); std::vector args2_boundserrorv(0); - args2_boundserrorv.push_back(jl_pvalue_llvmt); + args2_boundserrorv.push_back(T_pjlvalue); args2_boundserrorv.push_back(T_psize); args2_boundserrorv.push_back(T_size); jlboundserrorv_func = @@ -5381,7 +5432,7 @@ static void init_julia_llvm_env(Module *m) add_named_global(jlboundserrorv_func, (void*)&jl_bounds_error_ints); std::vector args2_boundserror(0); - args2_boundserror.push_back(jl_pvalue_llvmt); + args2_boundserror.push_back(T_pjlvalue); args2_boundserror.push_back(T_size); jlboundserror_func = Function::Create(FunctionType::get(T_void, args2_boundserror, false), @@ -5391,7 +5442,7 @@ static void init_julia_llvm_env(Module *m) add_named_global(jlboundserror_func, (void*)&jl_bounds_error_int); std::vector args3_vboundserror(0); - args3_vboundserror.push_back(jl_ppvalue_llvmt); + args3_vboundserror.push_back(T_ppjlvalue); args3_vboundserror.push_back(T_size); args3_vboundserror.push_back(T_size); jlvboundserror_func = @@ -5403,7 +5454,7 @@ static void init_julia_llvm_env(Module *m) std::vector args3_uboundserror(0); args3_uboundserror.push_back(T_pint8); - args3_uboundserror.push_back(jl_pvalue_llvmt); + args3_uboundserror.push_back(T_pjlvalue); args3_uboundserror.push_back(T_size); jluboundserror_func = Function::Create(FunctionType::get(T_void, args3_uboundserror, false), @@ -5413,7 +5464,7 @@ static void init_julia_llvm_env(Module *m) add_named_global(jluboundserror_func, (void*)&jl_bounds_error_unboxed_int); std::vector args2_throw(0); - args2_throw.push_back(jl_pvalue_llvmt); + args2_throw.push_back(T_pjlvalue); args2_throw.push_back(T_int32); jlthrow_line_func = (Function*)m->getOrInsertFunction("jl_throw_with_superfluous_argument", @@ -5449,8 +5500,8 @@ static void init_julia_llvm_env(Module *m) std::vector te_args(0); te_args.push_back(T_pint8); te_args.push_back(T_pint8); - te_args.push_back(jl_pvalue_llvmt); - te_args.push_back(jl_pvalue_llvmt); + te_args.push_back(T_pjlvalue); + te_args.push_back(T_pjlvalue); te_args.push_back(T_int32); jltypeerror_func = Function::Create(FunctionType::get(T_void, te_args, false), @@ -5460,8 +5511,8 @@ static void init_julia_llvm_env(Module *m) add_named_global(jltypeerror_func, (void*)&jl_type_error_rt_line); std::vector args_2ptrs(0); - args_2ptrs.push_back(jl_pvalue_llvmt); - args_2ptrs.push_back(jl_pvalue_llvmt); + args_2ptrs.push_back(T_pjlvalue); + args_2ptrs.push_back(T_pjlvalue); jlcheckassign_func = Function::Create(FunctionType::get(T_void, args_2ptrs, false), Function::ExternalLinkage, @@ -5469,7 +5520,7 @@ static void init_julia_llvm_env(Module *m) add_named_global(jlcheckassign_func, (void*)&jl_checked_assignment); std::vector args_1ptr(0); - args_1ptr.push_back(jl_pvalue_llvmt); + args_1ptr.push_back(T_pjlvalue); jldeclareconst_func = Function::Create(FunctionType::get(T_void, args_1ptr, false), Function::ExternalLinkage, @@ -5477,7 +5528,7 @@ static void init_julia_llvm_env(Module *m) add_named_global(jldeclareconst_func, (void*)&jl_declare_constant); jlgetbindingorerror_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, args_2ptrs, false), + Function::Create(FunctionType::get(T_pjlvalue, args_2ptrs, false), Function::ExternalLinkage, "jl_get_binding_or_error", m); add_named_global(jlgetbindingorerror_func, (void*)&jl_get_binding_or_error); @@ -5518,8 +5569,8 @@ static void init_julia_llvm_env(Module *m) add_named_global(queuerootfun, (void*)&jl_gc_queue_root); std::vector wbargs(0); - wbargs.push_back(jl_pvalue_llvmt); - wbargs.push_back(jl_pvalue_llvmt); + wbargs.push_back(T_pjlvalue); + wbargs.push_back(T_pjlvalue); wbfunc = Function::Create(FunctionType::get(T_void, wbargs, false), Function::ExternalLinkage, "jl_gc_wb_slow", m); @@ -5530,31 +5581,31 @@ static void init_julia_llvm_env(Module *m) expect_func = Intrinsic::getDeclaration(m, Intrinsic::expect, exp_args); std::vector args3(0); - args3.push_back(jl_pvalue_llvmt); + args3.push_back(T_pjlvalue); jlbox_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, args3, false), + Function::Create(FunctionType::get(T_pjlvalue, args3, false), Function::ExternalLinkage, "jl_new_box", m); add_named_global(jlbox_func, (void*)&jl_new_box); jltopeval_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, args3, false), + Function::Create(FunctionType::get(T_pjlvalue, args3, false), Function::ExternalLinkage, "jl_toplevel_eval", m); add_named_global(jltopeval_func, (void*)&jl_toplevel_eval); jlcopyast_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, args3, false), + Function::Create(FunctionType::get(T_pjlvalue, args3, false), Function::ExternalLinkage, "jl_copy_ast", m); add_named_global(jlcopyast_func, (void*)&jl_copy_ast); std::vector args4(0); args4.push_back(T_pint8); - args4.push_back(jl_pvalue_llvmt); - args4.push_back(jl_pvalue_llvmt); + args4.push_back(T_pjlvalue); + args4.push_back(T_pjlvalue); jlclosure_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, args4, false), + Function::Create(FunctionType::get(T_pjlvalue, args4, false), Function::ExternalLinkage, "jl_new_closure", m); add_named_global(jlclosure_func, (void*)&jl_new_closure); @@ -5562,34 +5613,34 @@ static void init_julia_llvm_env(Module *m) std::vector args5(0); args5.push_back(T_size); jlnsvec_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, args5, true), + Function::Create(FunctionType::get(T_pjlvalue, args5, true), Function::ExternalLinkage, "jl_svec", m); add_named_global(jlnsvec_func, (void*)&jl_svec); std::vector mdargs(0); - mdargs.push_back(jl_pvalue_llvmt); - mdargs.push_back(jl_ppvalue_llvmt); - mdargs.push_back(jl_pvalue_llvmt); - mdargs.push_back(jl_pvalue_llvmt); - mdargs.push_back(jl_pvalue_llvmt); - mdargs.push_back(jl_pvalue_llvmt); - mdargs.push_back(jl_pvalue_llvmt); - mdargs.push_back(jl_pvalue_llvmt); + mdargs.push_back(T_pjlvalue); + mdargs.push_back(T_ppjlvalue); + mdargs.push_back(T_pjlvalue); + mdargs.push_back(T_pjlvalue); + mdargs.push_back(T_pjlvalue); + mdargs.push_back(T_pjlvalue); + mdargs.push_back(T_pjlvalue); + mdargs.push_back(T_pjlvalue); mdargs.push_back(T_int32); jlmethod_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, mdargs, false), + Function::Create(FunctionType::get(T_pjlvalue, mdargs, false), Function::ExternalLinkage, "jl_method_def", m); add_named_global(jlmethod_func, (void*)&jl_method_def); std::vector funcdefargs(0); - funcdefargs.push_back(jl_pvalue_llvmt); - funcdefargs.push_back(jl_ppvalue_llvmt); - funcdefargs.push_back(jl_pvalue_llvmt); - funcdefargs.push_back(jl_pvalue_llvmt); + funcdefargs.push_back(T_pjlvalue); + funcdefargs.push_back(T_ppjlvalue); + funcdefargs.push_back(T_pjlvalue); + funcdefargs.push_back(T_pjlvalue); jlgenericfunction_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, funcdefargs, false), + Function::Create(FunctionType::get(T_pjlvalue, funcdefargs, false), Function::ExternalLinkage, "jl_generic_function_def", m); add_named_global(jlgenericfunction_func, (void*)&jl_generic_function_def); @@ -5638,8 +5689,8 @@ static void init_julia_llvm_env(Module *m) add_named_global(jlleave_func, (void*)&jl_pop_handler); std::vector args_2vals(0); - args_2vals.push_back(jl_pvalue_llvmt); - args_2vals.push_back(jl_pvalue_llvmt); + args_2vals.push_back(T_pjlvalue); + args_2vals.push_back(T_pjlvalue); jlegal_func = Function::Create(FunctionType::get(T_int32, args_2vals, false), Function::ExternalLinkage, @@ -5647,8 +5698,8 @@ static void init_julia_llvm_env(Module *m) add_named_global(jlegal_func, (void*)&jl_egal); std::vector subt_args(0); - subt_args.push_back(jl_pvalue_llvmt); - subt_args.push_back(jl_pvalue_llvmt); + subt_args.push_back(T_pjlvalue); + subt_args.push_back(T_pjlvalue); subt_args.push_back(T_int32); jlsubtype_func = Function::Create(FunctionType::get(T_int32, subt_args, false), @@ -5659,26 +5710,26 @@ static void init_julia_llvm_env(Module *m) std::vector aoargs(0); aoargs.push_back(T_size); jlallocobj_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, aoargs, false), + Function::Create(FunctionType::get(T_pjlvalue, aoargs, false), Function::ExternalLinkage, "jl_gc_allocobj", m); add_named_global(jlallocobj_func, (void*)&jl_gc_allocobj); std::vector empty_args(0); jlalloc1w_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, empty_args, false), + Function::Create(FunctionType::get(T_pjlvalue, empty_args, false), Function::ExternalLinkage, "jl_gc_alloc_1w", m); add_named_global(jlalloc1w_func, (void*)&jl_gc_alloc_1w); jlalloc2w_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, empty_args, false), + Function::Create(FunctionType::get(T_pjlvalue, empty_args, false), Function::ExternalLinkage, "jl_gc_alloc_2w", m); add_named_global(jlalloc2w_func, (void*)&jl_gc_alloc_2w); jlalloc3w_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, empty_args, false), + Function::Create(FunctionType::get(T_pjlvalue, empty_args, false), Function::ExternalLinkage, "jl_gc_alloc_3w", m); add_named_global(jlalloc3w_func, (void*)&jl_gc_alloc_3w); @@ -5686,7 +5737,7 @@ static void init_julia_llvm_env(Module *m) std::vector atargs(0); atargs.push_back(T_size); jl_alloc_svec_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, atargs, false), + Function::Create(FunctionType::get(T_pjlvalue, atargs, false), Function::ExternalLinkage, "jl_alloc_svec", m); add_named_global(jl_alloc_svec_func, (void*)&jl_alloc_svec); @@ -5702,19 +5753,19 @@ static void init_julia_llvm_env(Module *m) add_named_global(jldlsym_func, (void*)&jl_load_and_lookup); std::vector newbits_args(0); - newbits_args.push_back(jl_pvalue_llvmt); + newbits_args.push_back(T_pjlvalue); newbits_args.push_back(T_pint8); jlnewbits_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, newbits_args, false), + Function::Create(FunctionType::get(T_pjlvalue, newbits_args, false), Function::ExternalLinkage, "jl_new_bits", m); add_named_global(jlnewbits_func, (void*)&jl_new_bits); std::vector getnthfld_args(0); - getnthfld_args.push_back(jl_pvalue_llvmt); + getnthfld_args.push_back(T_pjlvalue); getnthfld_args.push_back(T_size); jlgetnthfieldchecked_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, getnthfld_args, false), + Function::Create(FunctionType::get(T_pjlvalue, getnthfld_args, false), Function::ExternalLinkage, "jl_get_nth_field_checked", m); add_named_global(jlgetnthfieldchecked_func, (void*)*jl_get_nth_field_checked); @@ -6062,13 +6113,13 @@ extern "C" void jl_init_codegen(void) BOX_F(char,char); UBOX_F(gensym,size); - box8_func = boxfunc_llvm(ft2arg(jl_pvalue_llvmt, jl_pvalue_llvmt, T_int8), + box8_func = boxfunc_llvm(ft2arg(T_pjlvalue, T_pjlvalue, T_int8), "jl_box8", (void*)&jl_box8, m); - box16_func = boxfunc_llvm(ft2arg(jl_pvalue_llvmt, jl_pvalue_llvmt, T_int16), + box16_func = boxfunc_llvm(ft2arg(T_pjlvalue, T_pjlvalue, T_int16), "jl_box16", (void*)&jl_box16, m); - box32_func = boxfunc_llvm(ft2arg(jl_pvalue_llvmt, jl_pvalue_llvmt, T_int32), + box32_func = boxfunc_llvm(ft2arg(T_pjlvalue, T_pjlvalue, T_int32), "jl_box32", (void*)&jl_box32, m); - box64_func = boxfunc_llvm(ft2arg(jl_pvalue_llvmt, jl_pvalue_llvmt, T_int64), + box64_func = boxfunc_llvm(ft2arg(T_pjlvalue, T_pjlvalue, T_int64), "jl_box64", (void*)&jl_box64, m); } diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index d6231b77e5a76..552a1dd0cfdd2 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -231,7 +231,7 @@ static Constant *julia_const_to_llvm(jl_value_t *e, bool nested=false) } JL_GC_POP(); - Type *t = julia_struct_to_llvm(jt); + Type *t = julia_struct_to_llvm(jt, NULL); if (type_is_ghost(t)) return UndefValue::get(NoopType); if (t->isVectorTy()) @@ -254,7 +254,7 @@ static Constant *julia_const_to_llvm(jl_value_t *e, bool nested=false) static jl_cgval_t emit_unboxed(jl_value_t *e, jl_codectx_t *ctx) { Constant *c = julia_const_to_llvm(e); - if (c) return mark_julia_type(c, expr_type(e, ctx)); + if (c) return mark_julia_type(c, false, expr_type(e, ctx)); return emit_expr(e, ctx, false); } @@ -263,7 +263,7 @@ static jl_cgval_t ghostValue(jl_value_t *ty); // emit code to unpack a raw value from a box into registers static Value *emit_unbox(Type *to, const jl_cgval_t &x, jl_value_t *jt) { - assert(to != jl_pvalue_llvmt); + assert(to != T_pjlvalue); // TODO: fully validate that x.typ == jt? if (x.isghost) { if (type_is_ghost(to)) { @@ -325,8 +325,9 @@ static Value *auto_unbox(const jl_cgval_t &v, jl_codectx_t *ctx) emit_error("auto_unbox: unable to determine argument type", ctx); return UndefValue::get(T_void); } - Type *to = julia_type_to_llvm(v.typ); - if (to == NULL || to == jl_pvalue_llvmt) { + bool isboxed; + Type *to = julia_type_to_llvm(v.typ, &isboxed); + if (to == NULL || isboxed) { // might be some sort of incomplete (but valid) Ptr{T} type, for example unsigned int nb = jl_datatype_size(bt)*8; to = IntegerType::get(jl_LLVMContext, nb); @@ -372,8 +373,9 @@ static jl_value_t *staticeval_bitstype(jl_value_t *targ, const char *fname, jl_c static Type *staticeval_bitstype(jl_value_t *bt) { assert(jl_is_bitstype(bt)); - Type *to = julia_type_to_llvm(bt); - if (to == NULL || to == jl_pvalue_llvmt) { + bool isboxed; + Type *to = julia_type_to_llvm(bt, &isboxed); + if (to == NULL || isboxed) { unsigned int nb = jl_datatype_size(bt)*8; to = IntegerType::get(jl_LLVMContext, nb); } @@ -429,7 +431,7 @@ static jl_cgval_t generic_box(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ctx builder.CreateAlignedStore(emit_unbox(llvmt, v, v.typ), builder.CreatePointerCast(newobj, llvmt->getPointerTo()), alignment); else builder.CreateMemCpy(newobj, builder.CreateBitCast(v.V, T_pint8), nb, alignment); - return mark_julia_type(newobj, bt ? bt : (jl_value_t*)jl_any_type); + return mark_julia_type(newobj, true, bt ? bt : (jl_value_t*)jl_any_type); } if (!jl_is_bitstype(bt)) { @@ -484,7 +486,7 @@ static jl_cgval_t generic_box(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ctx vx = builder.CreateBitCast(vx, llvmt); } - return mark_julia_type(vx, bt); + return mark_julia_type(vx, false, bt); } // NOTE: signd (signed) only relevant if check == true @@ -502,7 +504,7 @@ static jl_cgval_t generic_trunc(jl_value_t *targ, jl_value_t *x, jl_codectx_t *c raise_exception_unless(builder.CreateICmpEQ(back, ix), prepare_global(jlinexacterr_var), ctx); } - return mark_julia_type(ans, jlto); + return mark_julia_type(ans, false, jlto); } static jl_cgval_t generic_sext(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ctx) @@ -513,7 +515,7 @@ static jl_cgval_t generic_sext(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ct Value *ix = JL_INT(auto_unbox(x, ctx)); if (ix->getType() == T_void) return jl_cgval_t(); // auto_unbox threw an error Value *ans = builder.CreateSExt(ix, to); - return mark_julia_type(ans, jlto); + return mark_julia_type(ans, false, jlto); } static jl_cgval_t generic_zext(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ctx) @@ -524,7 +526,7 @@ static jl_cgval_t generic_zext(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ct Value *ix = JL_INT(auto_unbox(x, ctx)); if (ix->getType() == T_void) return jl_cgval_t(); // auto_unbox threw an error Value *ans = builder.CreateZExt(ix, to); - return mark_julia_type(ans, jlto); + return mark_julia_type(ans, false, jlto); } static Value *emit_eqfsi(Value *x, Value *y) @@ -578,7 +580,7 @@ static jl_cgval_t emit_checked_fptosi(jl_value_t *targ, jl_value_t *x, jl_codect else { raise_exception_unless(emit_eqfsi(fx, ans), prepare_global(jlinexacterr_var), ctx); } - return mark_julia_type(ans, jlto); + return mark_julia_type(ans, false, jlto); } static jl_cgval_t emit_checked_fptoui(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ctx) @@ -598,14 +600,14 @@ static jl_cgval_t emit_checked_fptoui(jl_value_t *targ, jl_value_t *x, jl_codect else { raise_exception_unless(emit_eqfui(fx, ans), prepare_global(jlinexacterr_var), ctx); } - return mark_julia_type(ans, jlto); + return mark_julia_type(ans, false, jlto); } static jl_cgval_t emit_runtime_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ctx) { Value *preffunc = // TODO: move this to the codegen initialization code section jl_Module->getOrInsertFunction("jl_pointerref", - FunctionType::get(jl_pvalue_llvmt, two_pvalue_llvmt, false)); + FunctionType::get(T_pjlvalue, two_pvalue_llvmt, false)); int ldepth = ctx->gc.argDepth; jl_cgval_t parg = emit_boxed_rooted(e, ctx); Value *iarg = boxed(emit_expr(i, ctx), ctx); @@ -622,7 +624,7 @@ static jl_cgval_t emit_runtime_pointerref(jl_value_t *e, jl_value_t *i, jl_codec else { ety = (jl_value_t*)jl_any_type; } - return jl_cgval_t(ret, ety); + return mark_julia_type(ret, true, ety); } static jl_cgval_t emit_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ctx) @@ -645,8 +647,9 @@ static jl_cgval_t emit_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ct if (ety == (jl_value_t*)jl_any_type) return mark_julia_type( builder.CreateLoad(builder.CreateGEP( - builder.CreateBitCast(thePtr, jl_ppvalue_llvmt), + builder.CreateBitCast(thePtr, T_ppjlvalue), im1)), + true, ety); if (!jl_is_structtype(ety) || jl_is_array_type(ety) || !jl_is_leaf_type(ety)) { emit_error("pointerref: invalid pointer type", ctx); @@ -662,7 +665,7 @@ static jl_cgval_t emit_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ct thePtr = builder.CreateGEP(builder.CreateBitCast(thePtr, T_pint8), im1); builder.CreateMemCpy(builder.CreateBitCast(strct, T_pint8), thePtr, size, 1); - return mark_julia_type(strct, ety); + return mark_julia_type(strct, true, ety); } // TODO: alignment? return typed_load(thePtr, im1, ety, ctx, tbaa_user, 1); @@ -736,7 +739,7 @@ static jl_cgval_t emit_pointerset(jl_value_t *e, jl_value_t *x, jl_value_t *i, j // TODO: alignment? typed_store(thePtr, im1, val, ety, ctx, tbaa_user, NULL, 1); } - return mark_julia_type(thePtr, aty); + return mark_julia_type(thePtr, false, aty); } static Value *emit_srem(Value *x, Value *den, jl_codectx_t *ctx) @@ -858,7 +861,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, int nb = get_bitstype_nbits(bt); Value *xi = JL_INT(auto_unbox(args[2],ctx)); if (xi->getType() == T_void) return jl_cgval_t(); // auto_unbox threw an error - return mark_julia_type(builder.CreateUIToFP(xi, FTnbits(nb)), bt); + return mark_julia_type(builder.CreateUIToFP(xi, FTnbits(nb)), false, bt); } HANDLE(sitofp,2) { @@ -867,7 +870,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, int nb = get_bitstype_nbits(bt); Value *xi = JL_INT(auto_unbox(args[2],ctx)); if (xi->getType() == T_void) return jl_cgval_t(); // auto_unbox threw an error - return mark_julia_type(builder.CreateSIToFP(xi, FTnbits(nb)), bt); + return mark_julia_type(builder.CreateSIToFP(xi, FTnbits(nb)), false, bt); } case fptoui: @@ -876,6 +879,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, if (x->getType() == T_void) return jl_cgval_t(); // auto_unbox threw an error return mark_julia_type( builder.CreateFPToUI(FP(x), JL_INTT(x->getType())), + false, JL_JLINTT(x->getType())); } else if (nargs == 2) { @@ -884,7 +888,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, int nb = get_bitstype_nbits(bt); Value *xf = FP(auto_unbox(args[2],ctx)); if (xf->getType() == T_void) return jl_cgval_t(); // auto_unbox threw an error - return mark_julia_type(builder.CreateFPToUI(xf, Type::getIntNTy(jl_LLVMContext, nb)), bt); + return mark_julia_type(builder.CreateFPToUI(xf, Type::getIntNTy(jl_LLVMContext, nb)), false, bt); } else { jl_error("fptoui: wrong number of arguments"); @@ -895,6 +899,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, Value *x = FP(auto_unbox(args[1], ctx)); return mark_julia_type( builder.CreateFPToSI(FP(x), JL_INTT(x->getType())), + false, JL_JLINTT(x->getType())); } else if (nargs == 2) { @@ -903,7 +908,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, int nb = get_bitstype_nbits(bt); Value *xf = FP(auto_unbox(args[2],ctx)); if (xf->getType() == T_void) return jl_cgval_t(); // auto_unbox threw an error - return mark_julia_type(builder.CreateFPToSI(xf, Type::getIntNTy(jl_LLVMContext, nb)), bt); + return mark_julia_type(builder.CreateFPToSI(xf, Type::getIntNTy(jl_LLVMContext, nb)), false, bt); } else { jl_error("fptosi: wrong number of arguments"); @@ -915,7 +920,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, int nb = get_bitstype_nbits(bt); Value *xf = FP(auto_unbox(args[2],ctx)); if (xf->getType() == T_void) return jl_cgval_t(); // auto_unbox threw an error - return mark_julia_type(builder.CreateFPTrunc(xf, FTnbits(nb)), bt); + return mark_julia_type(builder.CreateFPTrunc(xf, FTnbits(nb)), false, bt); } HANDLE(fpext,2) { @@ -934,18 +939,17 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, builder.CreateStore(FP(x), builder.CreateBitCast(prepare_global(jlfloattemp_var),FT(x->getType())->getPointerTo()), true); x = builder.CreateLoad(builder.CreateBitCast(prepare_global(jlfloattemp_var),FT(x->getType())->getPointerTo()), true); #endif - return mark_julia_type( - builder.CreateFPExt(x, FTnbits(nb)), - bt); + return mark_julia_type(builder.CreateFPExt(x, FTnbits(nb)), false, bt); } HANDLE(select_value,3) { Value *isfalse = emit_condition(args[1], "select_value", ctx); // emit the first argument jl_value_t *t1 = expr_type(args[2], ctx); jl_value_t *t2 = expr_type(args[3], ctx); - Type *llt1 = julia_type_to_llvm(t1); + bool isboxed; + Type *llt1 = julia_type_to_llvm(t1, &isboxed); Value *ifelse_result; - if (llt1 != jl_pvalue_llvmt && t1 == t2) { + if (!isboxed && t1 == t2) { // emit X and Y arguments jl_cgval_t x = emit_unboxed(args[2], ctx); jl_cgval_t y = emit_unboxed(args[3], ctx); @@ -965,18 +969,18 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, if (y.ispointer) // TODO: elid this load if unnecessary vy = builder.CreateLoad(builder.CreatePointerCast(vy, llt1->getPointerTo(0))); ifelse_result = builder.CreateSelect(isfalse, vy, vx); - return mark_julia_type(ifelse_result, t2); + return mark_julia_type(ifelse_result, false, t2); } else { int argStart = ctx->gc.argDepth; Value *arg1 = boxed(emit_expr(args[2],ctx,false), ctx, expr_type(args[2],ctx)); - // TODO: if (arg1->getType() != jl_pvalue_llvmt) + // TODO: if (!arg1.isboxed || arg1.needsgcroot) make_gcroot(arg1, ctx); Value *arg2 = boxed(emit_expr(args[3],ctx,false), ctx, expr_type(args[3],ctx)); ifelse_result = builder.CreateSelect(isfalse, arg2, arg1); ctx->gc.argDepth = argStart; jl_value_t *jt = (t1 == t2 ? t1 : (jl_value_t*)jl_any_type); - return mark_julia_type(ifelse_result, jt); + return mark_julia_type(ifelse_result, true, jt); } } @@ -1010,8 +1014,9 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, if (!newtyp && r->getType() != x->getType()) // cast back to the exact original type (e.g. float vs. int) before remarking as a julia type r = builder.CreateBitCast(r, x->getType()); - return mark_julia_type(r, newtyp ? newtyp : xinfo.typ); + return mark_julia_type(r, false, newtyp ? newtyp : xinfo.typ); } + } assert(0); } @@ -1438,7 +1443,7 @@ static FunctionType *ft2arg(Type *ret, Type *arg1, Type *arg2) } #define BOX_F(ct,jl_ct) \ - box_##ct##_func = boxfunc_llvm(ft1arg(jl_pvalue_llvmt, T_##jl_ct), \ + box_##ct##_func = boxfunc_llvm(ft1arg(T_pjlvalue, T_##jl_ct), \ "jl_box_"#ct, (void*)&jl_box_##ct, m); #define SBOX_F(ct,jl_ct) BOX_F(ct,jl_ct); box_##ct##_func->addAttribute(1, Attribute::SExt); From 8ad67575a913750aa2959e1fc8eed31523584cc2 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 25 Sep 2015 17:36:02 -0400 Subject: [PATCH 0408/1938] fix missing assignment when passing a rooted, boxed, immutable value to another function (by-ref) and set !needsgcroot on constants and arguments --- src/codegen.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 12aa0358db5f5..7aacbb7803c73 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -399,7 +399,7 @@ struct jl_cgval_t { bool isimmutable; // V points to something that is definitely immutable (e.g. not stack allocated) //bool isstack; // points to stack-allocated memory //bool isarg; // derived from an argument - bool needsgcroot; // this value needs a gcroot + mutable bool needsgcroot; // this value needs a gcroot jl_cgval_t(Value *V, bool isboxed, jl_value_t *typ) : // general constructor (with pointer type auto-detect) V(V), // V is allowed to be NULL in a jl_varinfo_t context, but not during codegen contexts typ(typ), @@ -646,7 +646,9 @@ static inline jl_cgval_t mark_julia_const(jl_value_t *jv) if (type_is_ghost(julia_type_to_llvm(typ))) { return ghostValue(typ); } - return jl_cgval_t(literal_pointer_val(jv), true, typ); + jl_cgval_t constant(literal_pointer_val(jv), true, typ); + constant.needsgcroot = false; + return constant; } @@ -2795,7 +2797,7 @@ static jl_cgval_t emit_call_function_object(jl_function_t *f, Value *theF, Value // can lazy load on demand, no copy needed Value *argv = arg.V; if (argv->getType() != at) - builder.CreatePointerCast(argv, at); + argv = builder.CreatePointerCast(argv, at); argvals[idx] = argv; } else { @@ -4823,6 +4825,7 @@ static Function *emit_function(jl_lambda_info_t *lam) Value *argPtr = builder.CreateGEP(argArray, ConstantInt::get(T_size, i)); theArg = mark_julia_type(builder.CreateLoad(argPtr), true, vi.value.typ); } + theArg.needsgcroot = false; Value *lv = vi.memloc; if (lv == NULL) { From 60741878c1a59a24670c864cc6e152291f521812 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 28 Sep 2015 19:24:09 -0400 Subject: [PATCH 0409/1938] correct isimmutable info and add needsgcroot annotations so that the runtime computation matches the result of is_stable_expr --- src/cgutils.cpp | 7 +++++-- src/codegen.cpp | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 8deefc2cb9fef..7f0d2c5046bb3 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1282,6 +1282,7 @@ static bool emit_getfield_unknownidx(jl_cgval_t *ret, const jl_cgval_t &strct, V if ((unsigned)stt->ninitialized != nfields) null_pointer_check(fld, ctx); *ret = mark_julia_type(fld, true, jl_any_type); + ret->needsgcroot = strct.needsgcroot || !strct.isimmutable; return true; } else if (is_tupletype_homogeneous(stt->types)) { @@ -1352,11 +1353,13 @@ static jl_cgval_t emit_getfield_knownidx(const jl_cgval_t &strct, unsigned idx, builder.CreateGEP(builder.CreateBitCast(strct.V, T_pint8), ConstantInt::get(T_size, jl_field_offset(jt,idx))); MDNode *tbaa = strct.isimmutable ? tbaa_immut : tbaa_user; - if (jl_field_isptr(jt,idx)) { + if (jl_field_isptr(jt, idx)) { Value *fldv = tbaa_decorate(tbaa, builder.CreateLoad(builder.CreateBitCast(addr, T_ppjlvalue))); if (idx >= (unsigned)jt->ninitialized) null_pointer_check(fldv, ctx); - return mark_julia_type(fldv, true, jfty); + jl_cgval_t ret = mark_julia_type(fldv, true, jfty); + ret.needsgcroot = strct.needsgcroot || !strct.isimmutable; + return ret; } else { int align = jl_field_offset(jt,idx); diff --git a/src/codegen.cpp b/src/codegen.cpp index 7aacbb7803c73..39561617a4263 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -731,6 +731,7 @@ static void alloc_local(jl_sym_t *s, jl_codectx_t *ctx) // CreateAlloca is OK here because alloc_local is only called during prologue setup Value *lv = builder.CreateAlloca(vtype, 0, s->name); vi.value = mark_julia_slot(lv, jt); + vi.value.isimmutable &= vi.isSA; // slot is not immutable if there are multiple assignments assert(vi.value.isboxed == false); #ifdef LLVM36 if (ctx->debug_enabled) { @@ -2012,7 +2013,9 @@ static jl_cgval_t emit_getfield(jl_value_t *expr, jl_sym_t *name, jl_codectx_t * ConstantInt::get(T_int32,2)); #endif ctx->gc.argDepth = argStart; - return mark_julia_type(result, true, jl_any_type); // (typ will be patched up by caller) + jl_cgval_t ret = mark_julia_type(result, true, jl_any_type); // (typ will be patched up by caller) + //ret.needsgcroot = arg1.needsgcroot || !arg1.isimmutable || !jl_is_leaf_type(arg1.typ) || !is_datatype_all_pointers((jl_datatype_t*)arg1.typ); + return ret; } static Value *emit_bits_compare(const jl_cgval_t &arg1, const jl_cgval_t &arg2, jl_codectx_t *ctx) @@ -2563,6 +2566,7 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, tbaa_decorate(tbaa_user, builder.CreateLoad(builder.CreateGEP(ctx->argArray, idx))), true, expr_type(expr, ctx)); + ret->needsgcroot = false; JL_GC_POP(); return true; } From 8b2ac7225792936f31012160d262a248005979e6 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 2 Oct 2015 14:52:47 -0400 Subject: [PATCH 0410/1938] remove a usage of emit_boxed_rooted that could be incorrectly propagating a boxed ghost --- src/codegen.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 39561617a4263..365c6163f250f 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2266,11 +2266,13 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, FunctionType *ft = FunctionType::get(T_void, two_pvalue_llvmt, false); // TODO: move this to the codegen init section Value *typeassert = jl_Module->getOrInsertFunction("jl_typeassert", ft); int ldepth = ctx->gc.argDepth; - *ret = emit_boxed_rooted(args[1], ctx); + *ret = emit_expr(args[1], ctx); + Value *V = boxed(*ret, ctx); + make_gcroot(V, ctx); #ifdef LLVM37 - builder.CreateCall(prepare_call(typeassert), {ret->V, boxed(emit_expr(args[2], ctx),ctx)}); + builder.CreateCall(prepare_call(typeassert), {V, boxed(emit_expr(args[2], ctx),ctx)}); #else - builder.CreateCall2(prepare_call(typeassert), ret->V, boxed(emit_expr(args[2], ctx),ctx)); + builder.CreateCall2(prepare_call(typeassert), V, boxed(emit_expr(args[2], ctx),ctx)); #endif ctx->gc.argDepth = ldepth; JL_GC_POP(); From 39f818e29d9b3efab36e7ca5f5f46979e9e30571 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 17 Aug 2015 15:40:04 -0400 Subject: [PATCH 0411/1938] more --compile=all related improvements --- base/essentials.jl | 2 +- src/gf.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/base/essentials.jl b/base/essentials.jl index 036a2b2bbdd0e..a02585ad1d6b8 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -111,7 +111,7 @@ cconvert{P<:Ptr}(::Type{P}, x) = x # but defer the conversion to Ptr to unsafe_c unsafe_convert{T}(::Type{T}, x::T) = x # unsafe_convert (like convert) defaults to assuming the convert occurred unsafe_convert{P<:Ptr}(::Type{P}, x::Ptr) = convert(P, x) -reinterpret{T,S}(::Type{T}, x::S) = box(T,unbox(S,x)) +reinterpret{T}(::Type{T}, x) = box(T, x) sizeof(x) = Core.sizeof(x) diff --git a/src/gf.c b/src/gf.c index 5b88689620f04..6b3421ed4a9ac 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1617,6 +1617,7 @@ void jl_compile_all(void) htable_t h; htable_new(&h, 0); _compile_all(jl_main_module, &h); + htable_free(&h); } DLLEXPORT void jl_compile_hint(jl_function_t *f, jl_tupletype_t *types) From b498eab909597edf199b995e635a814bcf618f2d Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Fri, 2 Oct 2015 15:50:22 -0400 Subject: [PATCH 0412/1938] Revert "Run doc/genstdlib.jl after bootstrap, take 2" This reverts commit f7786157f06901b57aa3f860daa63607d08d3608. --- Makefile | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Makefile b/Makefile index aeb7b237632aa..be2d7157dac65 100644 --- a/Makefile +++ b/Makefile @@ -98,13 +98,7 @@ julia-sysimg-release : julia-inference julia-ui-release julia-sysimg-debug : julia-inference julia-ui-debug @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) JULIA_BUILD_MODE=debug -JULIA_ENABLE_DOCBUILD ?= 1 -julia-genstdlib : julia-sysimg-$(JULIA_BUILD_MODE) -ifeq ($(JULIA_ENABLE_DOCBUILD), 1) - @$(call PRINT_JULIA, $(JULIA_EXECUTABLE) doc/genstdlib.jl) -endif - -julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest julia-genstdlib +julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest debug release : % : julia-% From 743e4eb1de14a6295be88e4b3eed33b98dda0862 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Fri, 2 Oct 2015 15:59:13 -0400 Subject: [PATCH 0413/1938] Makefile: add back rules for julia-genstdlib, docs --- Makefile | 6 ++++++ doc/stdlib/base.rst | 1 + 2 files changed, 7 insertions(+) diff --git a/Makefile b/Makefile index be2d7157dac65..68bbcc0519f87 100644 --- a/Makefile +++ b/Makefile @@ -102,6 +102,12 @@ julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink ju debug release : % : julia-% +julia-genstdlib: julia-sysimg-$(JULIA_BUILD_MODE) + @$(call PRINT_JULIA, $(JULIA_EXECUTABLE) doc/genstdlib.jl) + +docs: julia-genstdlib + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/doc + check-whitespace: ifneq ($(NO_GIT), 1) @$(JULIAHOME)/contrib/check-whitespace.sh diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 31529bcda680d..43cf418e69b65 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -1384,3 +1384,4 @@ Internals .. Docstring generated from Julia source Compile the given function ``f`` for the argument tuple (of types) ``args``\ , but do not execute it. + From 2cc13716c7773109110ad376140b5fdfa764bcde Mon Sep 17 00:00:00 2001 From: Kiran Pamnany Date: Fri, 2 Oct 2015 13:47:15 -0700 Subject: [PATCH 0414/1938] Style: 4-spaced. --- test/perf/threads/lbm3d/lbm3d.jl | 298 +++++++++++++++---------------- 1 file changed, 149 insertions(+), 149 deletions(-) diff --git a/test/perf/threads/lbm3d/lbm3d.jl b/test/perf/threads/lbm3d/lbm3d.jl index beff7dcb009a8..5bf9088dab549 100644 --- a/test/perf/threads/lbm3d/lbm3d.jl +++ b/test/perf/threads/lbm3d/lbm3d.jl @@ -35,179 +35,179 @@ const fourths = tuple([ 6, 8, 9,12,13], [ 7,10,11,14,15], function relax!(F, UX, UY, UZ, nx, ny, nz, deltaU, t1D, t2D, t3D, sSQU) - tid = threadid() - outerrange = Base.splitrange(nx, nthreads()) - for i = outerrange[tid] - #@threads all for i = 1:nx - for j = 1:ny - for k = 1:nz - density = 0.0 - for l = 1:size(F,4) - density = density + F[i,j,k,l] + tid = threadid() + outerrange = Base.splitrange(nx, nthreads()) + for i = outerrange[tid] + #@threads all for i = 1:nx + for j = 1:ny + for k = 1:nz + density = 0.0 + for l = 1:size(F,4) + density = density + F[i,j,k,l] + end + fs = Array(Float64, 6) + for l = 1:6 + fs[l] = 0.0 + for m = 1:5 + fs[l] = fs[l] + F[i,j,k,fourths[l][m]] + end + end + UX[i,j,k] = (fs[1] - fs[2]) / density + UY[i,j,k] = (fs[3] - fs[4]) / density + UZ[i,j,k] = (fs[5] - fs[6]) / density + + if i == 1 + UX[i,j,k] = UX[i,j,k] + deltaU # Increase inlet pressure + end + + t1D[i,j,k] = t1 * density + t2D[i,j,k] = t2 * density + t3D[i,j,k] = t3 * density + sSQU[i,j,k] = 3/2 * (UX[i,j,k]^2 + UY[i,j,k]^2 + UZ[i,j,k]^2) + end end - fs = Array(Float64, 6) - for l = 1:6 - fs[l] = 0.0 - for m = 1:5 - fs[l] = fs[l] + F[i,j,k,fourths[l][m]] - end - end - UX[i,j,k] = (fs[1] - fs[2]) / density - UY[i,j,k] = (fs[3] - fs[4]) / density - UZ[i,j,k] = (fs[5] - fs[6]) / density - - if i == 1 - UX[i,j,k] = UX[i,j,k] + deltaU # Increase inlet pressure - end - - t1D[i,j,k] = t1 * density - t2D[i,j,k] = t2 * density - t3D[i,j,k] = t3 * density - sSQU[i,j,k] = 3/2 * (UX[i,j,k]^2 + UY[i,j,k]^2 + UZ[i,j,k]^2) - end end - end end function calc_equi!(F, FEQ, t1D, t2D, t3D, U, UX, UY, UZ, sSQU, nx, ny, nz, omega) - tid = threadid() - outerrange = Base.splitrange(nx, nthreads()) - for i = outerrange[tid] - #@threads all for i = 1:nx - #tid = threadid() - for j = 1:ny - for k = 1:nz - - FEQ[i,j,k,1] = t1D[i,j,k,1] * (1 - sSQU[i,j,k,1]) - - # nearest neighbors - FEQ[i,j,k,2] = t2D[i,j,k,1] * (1 + 3*UZ[i,j,k,1] + 9/2*UZ[i,j,k,1]^2 - sSQU[i,j,k,1]) - FEQ[i,j,k,3] = t2D[i,j,k,1] * (1 - 3*UZ[i,j,k,1] + 9/2*UZ[i,j,k,1]^2 - sSQU[i,j,k,1]) - FEQ[i,j,k,4] = t2D[i,j,k,1] * (1 + 3*UY[i,j,k,1] + 9/2*UY[i,j,k,1]^2 - sSQU[i,j,k,1]) - FEQ[i,j,k,5] = t2D[i,j,k,1] * (1 - 3*UY[i,j,k,1] + 9/2*UY[i,j,k,1]^2 - sSQU[i,j,k,1]) - FEQ[i,j,k,6] = t2D[i,j,k,1] * (1 + 3*UX[i,j,k,1] + 9/2*UX[i,j,k,1]^2 - sSQU[i,j,k,1]) - FEQ[i,j,k,7] = t2D[i,j,k,1] * (1 - 3*UX[i,j,k,1] + 9/2*UX[i,j,k,1]^2 - sSQU[i,j,k,1]) - - U[1,tid] = UX[i,j,k,1] + UY[i,j,k,1] - U[2,tid] = UX[i,j,k,1] - UY[i,j,k,1] - U[3,tid] = -UX[i,j,k,1] + UY[i,j,k,1] - U[4,tid] = -U[1,tid] - U[5,tid] = UX[i,j,k,1] + UZ[i,j,k,1] - U[6,tid] = UX[i,j,k,1] - UZ[i,j,k,1] - U[7,tid] = -U[6,tid] - U[8,tid] = -U[5,tid] - U[9,tid] = UY[i,j,k,1] + UZ[i,j,k,1] - U[10,tid] = UY[i,j,k,1] - UZ[i,j,k,1] - U[11,tid] = -U[10,tid] - U[12,tid] = -U[9,tid] - - # next-nearest neighbors - for l = 1:12 - FEQ[i,j,k,l+7] = t3D[i,j,k,1] * (1 + 3*U[l,tid] + 9/2*(U[l,tid]^2) - sSQU[i,j,k,1]) + tid = threadid() + outerrange = Base.splitrange(nx, nthreads()) + for i = outerrange[tid] + #@threads all for i = 1:nx + #tid = threadid() + for j = 1:ny + for k = 1:nz + + FEQ[i,j,k,1] = t1D[i,j,k,1] * (1 - sSQU[i,j,k,1]) + + # nearest neighbors + FEQ[i,j,k,2] = t2D[i,j,k,1] * (1 + 3*UZ[i,j,k,1] + 9/2*UZ[i,j,k,1]^2 - sSQU[i,j,k,1]) + FEQ[i,j,k,3] = t2D[i,j,k,1] * (1 - 3*UZ[i,j,k,1] + 9/2*UZ[i,j,k,1]^2 - sSQU[i,j,k,1]) + FEQ[i,j,k,4] = t2D[i,j,k,1] * (1 + 3*UY[i,j,k,1] + 9/2*UY[i,j,k,1]^2 - sSQU[i,j,k,1]) + FEQ[i,j,k,5] = t2D[i,j,k,1] * (1 - 3*UY[i,j,k,1] + 9/2*UY[i,j,k,1]^2 - sSQU[i,j,k,1]) + FEQ[i,j,k,6] = t2D[i,j,k,1] * (1 + 3*UX[i,j,k,1] + 9/2*UX[i,j,k,1]^2 - sSQU[i,j,k,1]) + FEQ[i,j,k,7] = t2D[i,j,k,1] * (1 - 3*UX[i,j,k,1] + 9/2*UX[i,j,k,1]^2 - sSQU[i,j,k,1]) + + U[1,tid] = UX[i,j,k,1] + UY[i,j,k,1] + U[2,tid] = UX[i,j,k,1] - UY[i,j,k,1] + U[3,tid] = -UX[i,j,k,1] + UY[i,j,k,1] + U[4,tid] = -U[1,tid] + U[5,tid] = UX[i,j,k,1] + UZ[i,j,k,1] + U[6,tid] = UX[i,j,k,1] - UZ[i,j,k,1] + U[7,tid] = -U[6,tid] + U[8,tid] = -U[5,tid] + U[9,tid] = UY[i,j,k,1] + UZ[i,j,k,1] + U[10,tid] = UY[i,j,k,1] - UZ[i,j,k,1] + U[11,tid] = -U[10,tid] + U[12,tid] = -U[9,tid] + + # next-nearest neighbors + for l = 1:12 + FEQ[i,j,k,l+7] = t3D[i,j,k,1] * (1 + 3*U[l,tid] + 9/2*(U[l,tid]^2) - sSQU[i,j,k,1]) + end + + for l = 1:19 + F[i,j,k,l] = omega * FEQ[i,j,k,l] + (1 - omega) * F[i,j,k,l] + end + + end end - - for l = 1:19 - F[i,j,k,l] = omega * FEQ[i,j,k,l] + (1 - omega) * F[i,j,k,l] - end - - end end - end end precompile(calc_equi!, (Array{Float64,4}, Array{Float64,4}, Array{Float64,3}, Array{Float64,3}, Array{Float64,3}, Array{Float64,2}, Array{Float64,3}, Array{Float64,3}, Array{Float64,3}, Array{Float64,3}, Int64, Int64, Int64, Float64)) function lbm3d(n) - const nx = n - const ny = nx - const nz = nx - const omega = 1.0 - const density = 1.0 - - tprop = 0 - trelax = 0 - tequi = 0 - - F = repeat([density/19], outer=[nx, ny, nz, 19]) - FEQ = F - matsize = nx*ny*nz - - CI = [0:matsize:matsize*19;]' - - BOUND = Array(Float64,nx,ny,nz) - - for i=1:nx, j=1:ny, k=1:nz - BOUND[i,j,k] = ((i-5)^2 + (j-6)^2 + (k-7)^2) < 6 - end - - BOUND[:,:,1] = 1 - BOUND[:,1,:] = 1 - - ON = find(BOUND); # matrix offset of each Occupied Node - - TO_REFLECT = [ON+CI[2] ON+CI[3] ON+CI[4] ON+CI[5] ON+CI[6] ON+CI[7] ON+CI[8] ON+CI[9] ON+CI[10] ON+CI[11] ON+CI[12] ON+CI[13] ON+CI[14] ON+CI[15] ON+CI[16] ON+CI[17] ON+CI[18] ON+CI[19]] - REFLECTED = [ON+CI[3] ON+CI[2] ON+CI[5] ON+CI[4] ON+CI[7] ON+CI[6] ON+CI[11] ON+CI[10] ON+CI[9] ON+CI[8] ON+CI[15] ON+CI[14] ON+CI[13] ON+CI[12] ON+CI[19] ON+CI[18] ON+CI[17] ON+CI[16]] - - UX = Array(Float64,nx,ny,nz) - UY = Array(Float64,nx,ny,nz) - UZ = Array(Float64,nx,ny,nz) - U = Array(Float64,12,nthreads()) - t1D = Array(Float64,nx,ny,nz) - t2D = Array(Float64,nx,ny,nz) - t3D = Array(Float64,nx,ny,nz) - sSQU = Array(Float64,nx,ny,nz) - - avu = 1 - prevavu = 1 - ts = 0 - deltaU = 1e-7 - numactivenodes = sum(1-BOUND) - - @time while (ts < 4000 && (1e-10 < abs((prevavu-avu)/avu))) || ts < 100 - tic() - # Propagate -- nearest and next-nearest neighbors - for i = 2:19 - circshift3d1!(F, i, prop_shifts[i-1]) - end - tprop = tprop + toq() + const nx = n + const ny = nx + const nz = nx + const omega = 1.0 + const density = 1.0 + + tprop = 0 + trelax = 0 + tequi = 0 - # Densities bouncing back at next timestep - BOUNCEDBACK = F[TO_REFLECT] + F = repeat([density/19], outer=[nx, ny, nz, 19]) + FEQ = F + matsize = nx*ny*nz - tic() + CI = [0:matsize:matsize*19;]' - # Relax; calculate equilibrium state (FEQ) with equivalent speed and density to F - @threads all relax!(F, UX, UY, UZ, nx, ny, nz, deltaU, t1D, t2D, t3D, sSQU) - for o in ON - UX[o] = UY[o] = UZ[o] = t1D[o] = t2D[o] = t3D[o] = sSQU[o] = 0.0 + BOUND = Array(Float64,nx,ny,nz) + + for i=1:nx, j=1:ny, k=1:nz + BOUND[i,j,k] = ((i-5)^2 + (j-6)^2 + (k-7)^2) < 6 end - trelax = trelax + toq() - tic() + BOUND[:,:,1] = 1 + BOUND[:,1,:] = 1 + + ON = find(BOUND); # matrix offset of each Occupied Node + + TO_REFLECT = [ON+CI[2] ON+CI[3] ON+CI[4] ON+CI[5] ON+CI[6] ON+CI[7] ON+CI[8] ON+CI[9] ON+CI[10] ON+CI[11] ON+CI[12] ON+CI[13] ON+CI[14] ON+CI[15] ON+CI[16] ON+CI[17] ON+CI[18] ON+CI[19]] + REFLECTED = [ON+CI[3] ON+CI[2] ON+CI[5] ON+CI[4] ON+CI[7] ON+CI[6] ON+CI[11] ON+CI[10] ON+CI[9] ON+CI[8] ON+CI[15] ON+CI[14] ON+CI[13] ON+CI[12] ON+CI[19] ON+CI[18] ON+CI[17] ON+CI[16]] + + UX = Array(Float64,nx,ny,nz) + UY = Array(Float64,nx,ny,nz) + UZ = Array(Float64,nx,ny,nz) + U = Array(Float64,12,nthreads()) + t1D = Array(Float64,nx,ny,nz) + t2D = Array(Float64,nx,ny,nz) + t3D = Array(Float64,nx,ny,nz) + sSQU = Array(Float64,nx,ny,nz) + + avu = 1 + prevavu = 1 + ts = 0 + deltaU = 1e-7 + numactivenodes = sum(1-BOUND) + + @time while (ts < 4000 && (1e-10 < abs((prevavu-avu)/avu))) || ts < 100 + tic() + # Propagate -- nearest and next-nearest neighbors + for i = 2:19 + circshift3d1!(F, i, prop_shifts[i-1]) + end + tprop = tprop + toq() + + # Densities bouncing back at next timestep + BOUNCEDBACK = F[TO_REFLECT] + + tic() - # Calculate equilibrium distribution: stationary - @threads all calc_equi!(F, FEQ, t1D, t2D, t3D, U, UX, UY, UZ, sSQU, nx, ny, nz, omega) + # Relax; calculate equilibrium state (FEQ) with equivalent speed and density to F + @threads all relax!(F, UX, UY, UZ, nx, ny, nz, deltaU, t1D, t2D, t3D, sSQU) + for o in ON + UX[o] = UY[o] = UZ[o] = t1D[o] = t2D[o] = t3D[o] = sSQU[o] = 0.0 + end + + trelax = trelax + toq() + tic() - tequi = tequi + toq() + # Calculate equilibrium distribution: stationary + @threads all calc_equi!(F, FEQ, t1D, t2D, t3D, U, UX, UY, UZ, sSQU, nx, ny, nz, omega) - F[REFLECTED] = BOUNCEDBACK + tequi = tequi + toq() - prevavu = avu - avu = sum(UX) / numactivenodes - ts = ts + 1 - #println(avu) - end + F[REFLECTED] = BOUNCEDBACK + + prevavu = avu + avu = sum(UX) / numactivenodes + ts = ts + 1 + #println(avu) + end - println("ts: $(ts)") - println("propagation time: $tprop") - println("relaxation time: $trelax") - println("equilibrium time: $tequi") + println("ts: $(ts)") + println("propagation time: $tprop") + println("relaxation time: $trelax") + println("equilibrium time: $tequi") - #zcut=5 - #quiver(UY[:,:,zcut], UX[:,:,zcut]) - #xlabel("y"); ylabel("x") - #title("Flow field at z = $(zcut)), after $(ts)") + #zcut=5 + #quiver(UY[:,:,zcut], UX[:,:,zcut]) + #xlabel("y"); ylabel("x") + #title("Flow field at z = $(zcut)), after $(ts)") end @time lbm3d(36) From 49aec7d14956e1fd4feab5baaf8411306394e582 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 2 Oct 2015 16:12:55 -0500 Subject: [PATCH 0415/1938] Changed format for named groupings In [Python named back references](https://docs.python.org/3/library/re.html) must take the form of `(?P...)` but in [Perl](http://perldoc.perl.org/perlretut.html#Named-backreferences) and Julia named back references can look like `(?...)`, `(?'name'...)`, `(?P...)`, etc. I think we should stick to recommending the `(?...)` syntax as it is clear and concise. --- doc/manual/strings.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index 926cc9cda9620..23e1d2eed8b43 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -743,7 +743,7 @@ use destructuring syntax to bind them to local variables:: Captures can also be accessed by indexing the :obj:`RegexMatch` object with the number or name of the capture group:: - julia> m=match(r"(?P\d+):(?P\d+)","12:45") + julia> m=match(r"(?\d+):(?\d+)","12:45") RegexMatch("12:45", hour="12", minute="45") julia> m[:minute] "45" @@ -756,7 +756,7 @@ subsitution string with ``s``. Capture group 0 refers to the entire match object Named capture groups can be referenced in the substitution with ``g``. For example:: - julia> replace("first second", r"(\w+) (?P\w+)", s"\g \1") + julia> replace("first second", r"(\w+) (?\w+)", s"\g \1") julia> "second first" Numbered capture groups can also be referenced as ``\g`` for disambiguation, From 5c2bbdf689c8763fa7b3538afe713a8b3cf173d9 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 11 Sep 2015 17:18:39 -0400 Subject: [PATCH 0416/1938] precedence fixes: arrows below ||, and `in` should be a comparison --- src/julia-parser.scm | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/julia-parser.scm b/src/julia-parser.scm index 0c43298b0da8d..b5ceaec0f1d03 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -6,12 +6,11 @@ (define prec-assignment '(= := += -= *= /= //= .//= .*= ./= |\\=| |.\\=| ^= .^= ÷= .÷= %= .%= |\|=| &= $= => <<= >>= >>>= ~ |.+=| |.-=|)) (define prec-conditional '(?)) +(define prec-arrow '(-- --> ← → ↔ ↚ ↛ ↠ ↣ ↦ ↮ ⇎ ⇏ ⇒ ⇔ ⇴ ⇶ ⇷ ⇸ ⇹ ⇺ ⇻ ⇼ ⇽ ⇾ ⇿ ⟵ ⟶ ⟷ ⟷ ⟹ ⟺ ⟻ ⟼ ⟽ ⟾ ⟿ ⤀ ⤁ ⤂ ⤃ ⤄ ⤅ ⤆ ⤇ ⤌ ⤍ ⤎ ⤏ ⤐ ⤑ ⤔ ⤕ ⤖ ⤗ ⤘ ⤝ ⤞ ⤟ ⤠ ⥄ ⥅ ⥆ ⥇ ⥈ ⥊ ⥋ ⥎ ⥐ ⥒ ⥓ ⥖ ⥗ ⥚ ⥛ ⥞ ⥟ ⥢ ⥤ ⥦ ⥧ ⥨ ⥩ ⥪ ⥫ ⥬ ⥭ ⥰ ⧴ ⬱ ⬰ ⬲ ⬳ ⬴ ⬵ ⬶ ⬷ ⬸ ⬹ ⬺ ⬻ ⬼ ⬽ ⬾ ⬿ ⭀ ⭁ ⭂ ⭃ ⭄ ⭇ ⭈ ⭉ ⭊ ⭋ ⭌ ￩ ￫)) (define prec-lazy-or '(|\|\||)) (define prec-lazy-and '(&&)) -(define prec-arrow '(-- --> ← → ↔ ↚ ↛ ↠ ↣ ↦ ↮ ⇎ ⇏ ⇒ ⇔ ⇴ ⇶ ⇷ ⇸ ⇹ ⇺ ⇻ ⇼ ⇽ ⇾ ⇿ ⟵ ⟶ ⟷ ⟷ ⟹ ⟺ ⟻ ⟼ ⟽ ⟾ ⟿ ⤀ ⤁ ⤂ ⤃ ⤄ ⤅ ⤆ ⤇ ⤌ ⤍ ⤎ ⤏ ⤐ ⤑ ⤔ ⤕ ⤖ ⤗ ⤘ ⤝ ⤞ ⤟ ⤠ ⥄ ⥅ ⥆ ⥇ ⥈ ⥊ ⥋ ⥎ ⥐ ⥒ ⥓ ⥖ ⥗ ⥚ ⥛ ⥞ ⥟ ⥢ ⥤ ⥦ ⥧ ⥨ ⥩ ⥪ ⥫ ⥬ ⥭ ⥰ ⧴ ⬱ ⬰ ⬲ ⬳ ⬴ ⬵ ⬶ ⬷ ⬸ ⬹ ⬺ ⬻ ⬼ ⬽ ⬾ ⬿ ⭀ ⭁ ⭂ ⭃ ⭄ ⭇ ⭈ ⭉ ⭊ ⭋ ⭌ ￩ ￫)) (define prec-comparison - '(> < >= ≥ <= ≤ == === ≡ != ≠ !== ≢ |.>| |.<| |.>=| |.≥| |.<=| |.≤| |.==| |.!=| |.≠| |.=| |.!| |<:| |>:| ∈ ∉ ∋ ∌ ⊆ ⊈ ⊂ ⊄ ⊊ ∝ ∊ ∍ ∥ ∦ ∷ ∺ ∻ ∽ ∾ ≁ ≃ ≄ ≅ ≆ ≇ ≈ ≉ ≊ ≋ ≌ ≍ ≎ ≐ ≑ ≒ ≓ ≔ ≕ ≖ ≗ ≘ ≙ ≚ ≛ ≜ ≝ ≞ ≟ ≣ ≦ ≧ ≨ ≩ ≪ ≫ ≬ ≭ ≮ ≯ ≰ ≱ ≲ ≳ ≴ ≵ ≶ ≷ ≸ ≹ ≺ ≻ ≼ ≽ ≾ ≿ ⊀ ⊁ ⊃ ⊅ ⊇ ⊉ ⊋ ⊏ ⊐ ⊑ ⊒ ⊜ ⊩ ⊬ ⊮ ⊰ ⊱ ⊲ ⊳ ⊴ ⊵ ⊶ ⊷ ⋍ ⋐ ⋑ ⋕ ⋖ ⋗ ⋘ ⋙ ⋚ ⋛ ⋜ ⋝ ⋞ ⋟ ⋠ ⋡ ⋢ ⋣ ⋤ ⋥ ⋦ ⋧ ⋨ ⋩ ⋪ ⋫ ⋬ ⋭ ⋲ ⋳ ⋴ ⋵ ⋶ ⋷ ⋸ ⋹ ⋺ ⋻ ⋼ ⋽ ⋾ ⋿ ⟈ ⟉ ⟒ ⦷ ⧀ ⧁ ⧡ ⧣ ⧤ ⧥ ⩦ ⩧ ⩪ ⩫ ⩬ ⩭ ⩮ ⩯ ⩰ ⩱ ⩲ ⩳ ⩴ ⩵ ⩶ ⩷ ⩸ ⩹ ⩺ ⩻ ⩼ ⩽ ⩾ ⩿ ⪀ ⪁ ⪂ ⪃ ⪄ ⪅ ⪆ ⪇ ⪈ ⪉ ⪊ ⪋ ⪌ ⪍ ⪎ ⪏ ⪐ ⪑ ⪒ ⪓ ⪔ ⪕ ⪖ ⪗ ⪘ ⪙ ⪚ ⪛ ⪜ ⪝ ⪞ ⪟ ⪠ ⪡ ⪢ ⪣ ⪤ ⪥ ⪦ ⪧ ⪨ ⪩ ⪪ ⪫ ⪬ ⪭ ⪮ ⪯ ⪰ ⪱ ⪲ ⪳ ⪴ ⪵ ⪶ ⪷ ⪸ ⪹ ⪺ ⪻ ⪼ ⪽ ⪾ ⪿ ⫀ ⫁ ⫂ ⫃ ⫄ ⫅ ⫆ ⫇ ⫈ ⫉ ⫊ ⫋ ⫌ ⫍ ⫎ ⫏ ⫐ ⫑ ⫒ ⫓ ⫔ ⫕ ⫖ ⫗ ⫘ ⫙ ⫷ ⫸ ⫹ ⫺ ⊢ ⊣)) -;; infix "in" goes here + '(> < >= ≥ <= ≤ == === ≡ != ≠ !== ≢ |.>| |.<| |.>=| |.≥| |.<=| |.≤| |.==| |.!=| |.≠| |.=| |.!| |<:| |>:| ∈ ∉ ∋ ∌ ⊆ ⊈ ⊂ ⊄ ⊊ ∝ ∊ ∍ ∥ ∦ ∷ ∺ ∻ ∽ ∾ ≁ ≃ ≄ ≅ ≆ ≇ ≈ ≉ ≊ ≋ ≌ ≍ ≎ ≐ ≑ ≒ ≓ ≔ ≕ ≖ ≗ ≘ ≙ ≚ ≛ ≜ ≝ ≞ ≟ ≣ ≦ ≧ ≨ ≩ ≪ ≫ ≬ ≭ ≮ ≯ ≰ ≱ ≲ ≳ ≴ ≵ ≶ ≷ ≸ ≹ ≺ ≻ ≼ ≽ ≾ ≿ ⊀ ⊁ ⊃ ⊅ ⊇ ⊉ ⊋ ⊏ ⊐ ⊑ ⊒ ⊜ ⊩ ⊬ ⊮ ⊰ ⊱ ⊲ ⊳ ⊴ ⊵ ⊶ ⊷ ⋍ ⋐ ⋑ ⋕ ⋖ ⋗ ⋘ ⋙ ⋚ ⋛ ⋜ ⋝ ⋞ ⋟ ⋠ ⋡ ⋢ ⋣ ⋤ ⋥ ⋦ ⋧ ⋨ ⋩ ⋪ ⋫ ⋬ ⋭ ⋲ ⋳ ⋴ ⋵ ⋶ ⋷ ⋸ ⋹ ⋺ ⋻ ⋼ ⋽ ⋾ ⋿ ⟈ ⟉ ⟒ ⦷ ⧀ ⧁ ⧡ ⧣ ⧤ ⧥ ⩦ ⩧ ⩪ ⩫ ⩬ ⩭ ⩮ ⩯ ⩰ ⩱ ⩲ ⩳ ⩴ ⩵ ⩶ ⩷ ⩸ ⩹ ⩺ ⩻ ⩼ ⩽ ⩾ ⩿ ⪀ ⪁ ⪂ ⪃ ⪄ ⪅ ⪆ ⪇ ⪈ ⪉ ⪊ ⪋ ⪌ ⪍ ⪎ ⪏ ⪐ ⪑ ⪒ ⪓ ⪔ ⪕ ⪖ ⪗ ⪘ ⪙ ⪚ ⪛ ⪜ ⪝ ⪞ ⪟ ⪠ ⪡ ⪢ ⪣ ⪤ ⪥ ⪦ ⪧ ⪨ ⪩ ⪪ ⪫ ⪬ ⪭ ⪮ ⪯ ⪰ ⪱ ⪲ ⪳ ⪴ ⪵ ⪶ ⪷ ⪸ ⪹ ⪺ ⪻ ⪼ ⪽ ⪾ ⪿ ⫀ ⫁ ⫂ ⫃ ⫄ ⫅ ⫆ ⫇ ⫈ ⫉ ⫊ ⫋ ⫌ ⫍ ⫎ ⫏ ⫐ ⫑ ⫒ ⫓ ⫔ ⫕ ⫖ ⫗ ⫘ ⫙ ⫷ ⫸ ⫹ ⫺ ⊢ ⊣)) ;; plus `in` (define prec-pipe '(|\|>| |<\||)) (define prec-colon '(: |..|)) (define prec-plus '(+ - ⊕ ⊖ ⊞ ⊟ |.+| |.-| |++| |\|| ∪ ∨ $ ⊔ ± ∓ ∔ ∸ ≂ ≏ ⊎ ⊻ ⊽ ⋎ ⋓ ⧺ ⧻ ⨈ ⨢ ⨣ ⨤ ⨥ ⨦ ⨧ ⨨ ⨩ ⨪ ⨫ ⨬ ⨭ ⨮ ⨹ ⨺ ⩁ ⩂ ⩅ ⩊ ⩌ ⩏ ⩐ ⩒ ⩔ ⩖ ⩗ ⩛ ⩝ ⩡ ⩢ ⩣)) @@ -46,6 +45,11 @@ (eval `(define ,(symbol (string "is-" name "?")) (Set ,name)))) prec-names) +(define is-prec-comparison? + (let ((compare-ops (Set prec-comparison))) + (lambda (t) + (or (compare-ops t) (eq? t 'in))))) + ;; hash table of binary operators -> precedence (define prec-table (let ((t (table))) (define (pushprec L prec) @@ -538,7 +542,7 @@ (if (not (,ops t)) ex (begin (take-token ,s) - (if (or (syntactic-op? t) (eq? t 'in) (eq? t '|::|)) + (if (or (syntactic-op? t) (eq? t '|::|)) (loop (list t ex (,down ,s)) (peek-token ,s)) (loop (list 'call t ex (,down ,s)) (peek-token ,s))))))) @@ -569,7 +573,7 @@ (list 'call t ex (parse-RtoL s down ops)))))))) (define (parse-cond s) - (let ((ex (parse-or s))) + (let ((ex (parse-arrow s))) (cond ((eq? (peek-token s) '?) (begin (take-token s) (let ((then (without-range-colon (parse-eq* s)))) @@ -582,7 +586,7 @@ (if (or (eof-object? next) (closing-token? next) (newline? next)) `(call (top string) ,@(reverse args)) - (loop (cons (parse-or s) args)))))) + (loop (cons (parse-arrow s) args)))))) (else ex)))) (define (invalid-initial-token? tok) @@ -727,9 +731,9 @@ ; parse-comma is needed for commas outside parens, for example a = b,c (define (parse-comma s) (parse-Nary s parse-cond '(#\,) 'tuple '() #f)) -(define (parse-or s) (parse-LtoR s parse-and is-prec-lazy-or?)) -(define (parse-and s) (parse-LtoR s parse-arrow is-prec-lazy-and?)) -(define (parse-arrow s) (parse-RtoL s parse-comparison is-prec-arrow?)) +(define (parse-arrow s) (parse-RtoL s parse-or is-prec-arrow?)) +(define (parse-or s) (parse-RtoL s parse-and is-prec-lazy-or?)) +(define (parse-and s) (parse-RtoL s parse-comparison is-prec-lazy-and?)) ;; parse left to right chains of a certain binary operator ;; returns a list of arguments @@ -780,19 +784,16 @@ (define (parse-pipes s) (parse-LtoR s parse-range is-prec-pipe?)) -(define is-in? (Set '(in))) -(define (parse-in s) (parse-LtoR s parse-pipes is-in?)) - (define (parse-comparison s) - (let loop ((ex (parse-in s)) + (let loop ((ex (parse-pipes s)) (first #t)) (let ((t (peek-token s))) (if (not (is-prec-comparison? t)) ex (begin (take-token s) (if first - (loop (list 'comparison ex t (parse-in s)) #f) - (loop (append ex (list t (parse-in s))) #f))))))) + (loop (list 'comparison ex t (parse-pipes s)) #f) + (loop (append ex (list t (parse-pipes s))) #f))))))) ; flag an error for tokens that cannot begin an expression (define (closing-token? tok) @@ -1412,8 +1413,8 @@ r) ((eq? r ':) r) - ((and (pair? r) (eq? (car r) 'in)) - `(= ,(cadr r) ,(caddr r))) + ((and (length= r 4) (eq? (car r) 'comparison) (eq? (caddr r) 'in)) + `(= ,(cadr r) ,(cadddr r))) (else (error "invalid iteration specification"))))) (case (peek-token s) From f8308f91fa548fa4a0402af6195ba210825aa2df Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 2 Oct 2015 18:21:56 -0400 Subject: [PATCH 0417/1938] add tests for precedence of `in` and arrows --- test/parse.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/parse.jl b/test/parse.jl index 333bdc4dc2e15..3dcf6c4924866 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -304,3 +304,9 @@ let p = parse("try @test p.args[2] === false @test p.args[3].args[end] == parse("b,c = t") end + +# pr #13078 +@test parse("a in b in c") == Expr(:comparison, :a, :in, :b, :in, :c) +@test parse("a||b→c&&d") == Expr(:call, :→, + Expr(symbol("||"), :a, :b), + Expr(symbol("&&"), :c, :d)) From f2fe46bdfcae7680b2b25d577be506fefb468fc3 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Fri, 2 Oct 2015 12:31:01 -0700 Subject: [PATCH 0418/1938] Bundle CA certs on linux if we have a single cert.pem file (we do on our buildbots) --- Makefile | 4 ++++ base/libgit2.jl | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/Makefile b/Makefile index 68bbcc0519f87..ae22c8169610c 100644 --- a/Makefile +++ b/Makefile @@ -448,6 +448,10 @@ ifneq ($(OS), WINNT) endif ifeq ($(OS), Linux) -$(JULIAHOME)/contrib/fixup-libstdc++.sh $(DESTDIR)$(private_libdir) + # We need to bundle ca certs on linux now that we're using libgit2 with ssl +ifneq ($(shell cat $(shell openssl version -d | cut -d '"' -f 2)/cert.pem),) + -cp $(shell openssl version -d | cut -d '"' -f 2)/cert.pem $(DESTDIR)$(datarootdir)/julia/ +endif endif # Copy in juliarc.jl files per-platform for binary distributions as well # Note that we don't install to sysconfdir: we always install to $(DESTDIR)$(prefix)/etc. diff --git a/base/libgit2.jl b/base/libgit2.jl index 1fa5ceffecfa6..c2779489fac90 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -484,6 +484,11 @@ function transact(f::Function, repo::GitRepo) end end +function set_ssl_cert_locations(cert_file) + GIT_OPT_SET_SSL_CERT_LOCATIONS = 12 + ccall((:git_libgit2_opts, :libgit2), Cint, (Cint, Cstring, Ptr{Void}), + GIT_OPT_SET_SSL_CERT_LOCATIONS, cert_file, C_NULL) +end function __init__() err = ccall((:git_libgit2_init, :libgit2), Cint, ()) @@ -491,6 +496,13 @@ function __init__() atexit() do ccall((:git_libgit2_shutdown, :libgit2), Cint, ()) end + + # If we have a bundled ca cert file, point libgit2 at that so SSL connections work. + cert_file = abspath(ccall(:jl_get_julia_home, Any, ()),Base.DATAROOTDIR,"julia","cert.pem") + if isfile(cert_file) + set_ssl_cert_locations(cert_file) + end end + end # module From 99ee2ffe229842f8713b5b6436eecf1d360c362b Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Fri, 2 Oct 2015 23:40:26 -0400 Subject: [PATCH 0419/1938] Fix inlinedAt debug locations ref #922 and 35c5234a --- src/codegen.cpp | 24 +++++++++++++----------- test/backtrace.jl | 2 ++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 365c6163f250f..9160670246262 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4414,6 +4414,7 @@ static Function *emit_function(jl_lambda_info_t *lam) } } ctx.lineno = lno; + int toplineno = lno; DIBuilder dbuilder(*m); ctx.dbuilder = &dbuilder; @@ -4994,23 +4995,24 @@ static Function *emit_function(jl_lambda_info_t *lam) MDNode *funcscope = (MDNode*)dbuilder.createLexicalBlockFile(SP, topfile); MDNode *scope; if ((dfil == topfile || dfil == NULL) && - lno >= ctx.lineno) // if we are in the top-level file - // and the current lineno is less than - // the last, must be same-file inline - // TODO: improve this heuristic... + lno >= toplineno) { - // set location to the current top-level line + // for sequentially-defined code, + // set location to line in top file. + // TODO: improve handling of nested inlines loc = DebugLoc::get(lno, 1, SP, NULL); } else { - // otherwise, we are compiling code from another file, - // so create a location for the top-level line, and - // set the DebugLoc "inlinedAt" parameter. + // otherwise, we are compiling inlined code, + // so set the DebugLoc "inlinedAt" parameter + // to the current line, then use source loc. #ifdef LLVM37 scope = (MDNode*)dbuilder.createLexicalBlockFile(SP,dfil); - MDNode *inlineLocMd = DebugLoc::get(ctx.lineno, 1, funcscope, NULL).getAsMDNode(); + MDNode *inlineLocMd = DebugLoc::get(toplineno, 1, funcscope, NULL). + getAsMDNode(); #else scope = (MDNode*)dbuilder.createLexicalBlockFile(SP,DIFile(dfil)); - MDNode *inlineLocMd = DebugLoc::get(ctx.lineno, 1, funcscope, NULL).getAsMDNode(jl_LLVMContext); + MDNode *inlineLocMd = DebugLoc::get(toplineno, 1, funcscope, NULL). + getAsMDNode(jl_LLVMContext); #endif loc = DebugLoc::get(lno, 1, scope, inlineLocMd); } @@ -5018,7 +5020,7 @@ static Function *emit_function(jl_lambda_info_t *lam) } if (do_coverage) coverageVisitLine(filename, lno); - ctx.lineno = lno; + ctx.lineno = lno; // NOO TOUCHIE; NO TOUCH! See #922 } if (jl_is_labelnode(stmt)) { if (prevlabel) continue; diff --git a/test/backtrace.jl b/test/backtrace.jl index 19c0f4f17eeeb..16bfec0f626d1 100644 --- a/test/backtrace.jl +++ b/test/backtrace.jl @@ -34,6 +34,7 @@ eval(Expr(:function, Expr(:call, :test_inline_1), # different-file inline eval(Expr(:function, Expr(:call, :test_inline_2), Expr(:block, LineNumberNode(symbol("backtrace.jl"), 99), + LineNumberNode(symbol("foobar.jl"), 666), LineNumberNode(symbol("/foo/bar/baz.jl"), 111), Expr(:call, :throw, "foo")))) @@ -74,6 +75,7 @@ loc = functionloc(f12977) @test endswith(loc[1], "backtrace.jl") @test loc[2] == linenum +# issue #922: SimplifyCFG pass merges throws code_loc(p, skipC=true) = ccall(:jl_lookup_code_address, Any, (Ptr{Void},Cint), p, skipC) @noinline function test_throw_commoning(x) From 3561f4b1afc4687f140160fd43b25a290cf04d5b Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 2 Oct 2015 23:34:24 -0400 Subject: [PATCH 0420/1938] fix type-inference regression when encountering a lambda with the wrong number of arguments --- base/inference.jl | 8 +++++--- test/core.jl | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 00acb4392723c..5bb10ab807999 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1490,8 +1490,8 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV for i=laty+1:la s[1][args[i]] = VarState(lastatype,false) end - else - @assert la == 0 + elseif la != 0 + return ((), Bottom, false) # wrong number of arguments end # types of closed vars @@ -1804,7 +1804,9 @@ function eval_annotate(e::ANY, vtypes::ANY, sv::StaticVarInfo, decls, clo, undef argtypes = Tuple{[abstract_eval(a, vtypes, sv) for a in fargs]...} # recur inside inner functions once we have all types tr,ty = typeinf(called, argtypes, called.sparams, called, false, true) - called.ast = tr + if tr !== () + called.ast = tr + end end return e end diff --git a/test/core.jl b/test/core.jl index 6fabf8c54076a..54ab15328e26c 100644 --- a/test/core.jl +++ b/test/core.jl @@ -2180,6 +2180,21 @@ f9520c(::Any, ::Any, ::Any, ::Any, ::Any, ::Any, args...) = 46 @test invoke(f9520c, (Any, Any, Any, Any, Any, Any), 1, 2, 3, 4, 5, 6) == 46 @test invoke(f9520c, (Any, Any, Any, Any, Any, Any, Any), 1, 2, 3, 4, 5, 6, 7) == 46 +call_lambda1() = (()->x)(1) +call_lambda2() = ((x)->x)() +call_lambda3() = ((x)->x)(1,2) +call_lambda4() = ((x,y...)->x)() +@test (try call_lambda1(); false; catch e; (e::ErrorException).msg; end) == "wrong number of arguments" +@test (try call_lambda2(); false; catch e; (e::ErrorException).msg; end) == "wrong number of arguments" +@test (try call_lambda3(); false; catch e; (e::ErrorException).msg; end) == "wrong number of arguments" +@test (try call_lambda4(); false; catch e; (e::ErrorException).msg; end) == "too few arguments" +call_lambda5() = ((x...)->x)() +call_lambda6() = ((x...)->x)(1) +call_lambda7() = ((x...)->x)(1,2) +@test call_lambda5() == () +@test call_lambda6() == (1,) +@test call_lambda7() == (1,2) + # jl_new_bits testing let x = [1,2,3] @test ccall(:jl_new_bits, Any, (Any,Ptr{Void},), Int, x) === 1 From fba99fdbb3730fa4bfc3e3da2d518d473139e60c Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 2 Oct 2015 23:58:58 -0400 Subject: [PATCH 0421/1938] enables inlining of anonymous functions --- base/inference.jl | 231 ++++++++++++++++++++++++---------------------- 1 file changed, 120 insertions(+), 111 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 5bb10ab807999..4086c97d72c80 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2118,90 +2118,97 @@ end # static parameters are ok if all the static parameter values are leaf types, # meaning they are fully known. function inlineable(f::ANY, e::Expr, atype::ANY, sv::StaticVarInfo, enclosing_ast::Expr) - if !(isa(f,Function) || isa(f,IntrinsicFunction)) - return NF - end - atypes = atype.parameters - argexprs = e.args[2:end] + local linfo, + metharg::Type, + methsp::SimpleVector, + atypes = atype.parameters, + argexprs = e.args[2:end], + incompletematch = false, + isgf = false + if isa(f, LambdaStaticData) + linfo = f + metharg = Tuple + methsp = svec() + else + if !(isa(f,Function) || isa(f,IntrinsicFunction)) + return NF + end - if is(f, typeassert) && length(atypes)==2 - # typeassert(x::S, T) => x, when S<:T - if isType(atypes[2]) && isleaftype(atypes[2]) && - atypes[1] <: atypes[2].parameters[1] - return (e.args[2],()) + if is(f, typeassert) && length(atypes)==2 + # typeassert(x::S, T) => x, when S<:T + if isType(atypes[2]) && isleaftype(atypes[2]) && + atypes[1] <: atypes[2].parameters[1] + return (e.args[2],()) + end end - end - if length(atypes)==2 && is(f,unbox) && isa(atypes[2],DataType) && !atypes[2].mutable && atypes[2].pointerfree - # remove redundant unbox - return (e.args[3],()) - end - topmod = _topmod() - if istopfunction(topmod, f, :isbits) && length(atypes)==1 && isType(atypes[1]) && - effect_free(argexprs[1],sv,true) && isleaftype(atypes[1].parameters[1]) - return (isbits(atypes[1].parameters[1]),()) - end - # special-case inliners for known pure functions that compute types - if isType(e.typ) - if (is(f,apply_type) || is(f,fieldtype) || - istopfunction(topmod, f, :typejoin) || - istopfunction(topmod, f, :promote_type)) && - isleaftype(e.typ.parameters[1]) - return (e.typ.parameters[1],()) + if length(atypes)==2 && is(f,unbox) && isa(atypes[2],DataType) && !atypes[2].mutable && atypes[2].pointerfree + # remove redundant unbox + return (e.args[3],()) + end + topmod = _topmod() + if istopfunction(topmod, f, :isbits) && length(atypes)==1 && isType(atypes[1]) && + effect_free(argexprs[1],sv,true) && isleaftype(atypes[1].parameters[1]) + return (isbits(atypes[1].parameters[1]),()) + end + # special-case inliners for known pure functions that compute types + if isType(e.typ) + if (is(f,apply_type) || is(f,fieldtype) || + istopfunction(topmod, f, :typejoin) || + istopfunction(topmod, f, :promote_type)) && + isleaftype(e.typ.parameters[1]) + return (e.typ.parameters[1],()) + end + end + if isa(f,IntrinsicFunction) + return NF end - end - if isa(f,IntrinsicFunction) - return NF - end - meth = _methods(f, atype, 1) - if meth === false || length(meth) != 1 - return NF - end - meth = meth[1]::SimpleVector + local methfunc::Function + if isgeneric(f) + isgf = true + meth = _methods(f, atype, 1) + if meth === false || length(meth) != 1 + return NF + end + meth = meth[1]::SimpleVector + linfo = try + func_for_method(meth[3],atype,meth[2]) + catch + NF + end + if linfo === NF + return NF + end + metharg = meth[1] + methsp = meth[2] + methfunc = meth[3].func + methsig = meth[3].sig + if !(atype <: metharg) + incompletematch = true + if !inline_incompletematch_allowed || !isdefined(Main,:Base) + # provide global disable if this optimization is not desirable + # need Main.Base defined for MethodError + return NF + end + end + else + if !isdefined(f, :code) + return NF + end + linfo = f.code + metharg = Tuple + methsp = svec() + methfunc = f + end - local linfo - linfo = try - func_for_method(meth[3],atype,meth[2]) - catch - NF - end - if linfo === NF - return NF + if length(methfunc.env) > 0 + # can't inline something with an env + return NF + end end linfo = linfo::LambdaStaticData - ## This code tries to limit the argument list length only when it is - ## growing due to recursion. - ## It might be helpful for some things, but turns out not to be - ## necessary to get max performance from recursive varargs functions. - # if length(atypes) > MAX_TUPLETYPE_LEN - # # check call stack to see if this argument list is growing - # st = inference_stack - # while !isa(st, EmptyCallStack) - # if st.ast === linfo.def.ast && length(atypes) > length(st.types) - # atypes = limit_tuple_type(atypes) - # meth = _methods(f, atypes, 1) - # if meth === false || length(meth) != 1 - # return NF - # end - # meth = meth[1]::Tuple - # linfo2 = meth[3].func.code - # if linfo2 !== linfo - # return NF - # end - # linfo = linfo2 - # break - # end - # st = st.prev - # end - # end - - if !isa(linfo,LambdaStaticData) || length(meth[3].func.env) > 0 - return NF - end - - sp = meth[2]::SimpleVector - sp = svec(sp..., linfo.sparams...) + sp = svec(methsp..., linfo.sparams...) spvals = Any[ sp[i] for i in 2:2:length(sp) ] for i=1:length(spvals) si = spvals[i] @@ -2213,21 +2220,10 @@ function inlineable(f::ANY, e::Expr, atype::ANY, sv::StaticVarInfo, enclosing_as end end - metharg = meth[1]::Type methargs = metharg.parameters nm = length(methargs) - if !(atype <: metharg) - incompletematch = true - if !inline_incompletematch_allowed || !isdefined(Main,:Base) - # provide global disable if this optimization is not desirable - # need Main.Base defined for MethodError - return NF - end - else - incompletematch = false - end - (ast, ty) = typeinf(linfo, metharg, meth[2], linfo, true, true) + (ast, ty) = typeinf(linfo, metharg, methsp, linfo, true, true) if is(ast,()) return NF end @@ -2255,8 +2251,8 @@ function inlineable(f::ANY, e::Expr, atype::ANY, sv::StaticVarInfo, enclosing_as if is(f, next) || is(f, done) || is(f, unsafe_convert) || is(f, cconvert) cost ÷= 4 end - inline_op = (f===(+) || f===(*) || f===min || f===max) && (3 <= length(argexprs) <= 9) && - meth[3].sig == Tuple{Any,Any,Any,Vararg{Any}} + inline_op = isgeneric(f) && (f===(+) || f===(*) || f===min || f===max) && (3 <= length(argexprs) <= 9) && + methsig == Tuple{Any,Any,Any,Vararg{Any}} if !inline_op && !inline_worthy(body, cost) if incompletematch # inline a typeassert-based call-site, rather than a @@ -2264,9 +2260,9 @@ function inlineable(f::ANY, e::Expr, atype::ANY, sv::StaticVarInfo, enclosing_as # all the fiddly details numarg = length(argexprs) newnames = unique_names(ast,numarg) - sp = () + methsp = sp + sp = svec() spvals = [] - meth = svec(metharg, sp) locals = [] newcall = Expr(:call, e.args[1]) newcall.typ = ty @@ -2297,7 +2293,10 @@ function inlineable(f::ANY, e::Expr, atype::ANY, sv::StaticVarInfo, enclosing_as na = length(args) isva = false - if na>0 && is_rest_arg(ast.args[1][na]) + if na > 0 && is_rest_arg(ast.args[1][na]) + if length(argexprs) < na - 1 + return (Expr(:call, TopNode(:error), "too few arguments"), []) + end vaname = args[na] len_argexprs = length(argexprs) valen = len_argexprs-na+1 @@ -2334,12 +2333,15 @@ function inlineable(f::ANY, e::Expr, atype::ANY, sv::StaticVarInfo, enclosing_as isva = true end elseif na != length(argexprs) - # we have a method match only because an earlier - # inference step shortened our call args list, even - # though we have too many arguments to actually - # call this function - @assert isvarargtype(atypes[na]) - return NF + if isgf + # we have a method match only because an earlier + # inference step shortened our call args list, even + # though we have too many arguments to actually + # call this function + @assert isvarargtype(atypes[na]) + return NF + end + return (Expr(:call, TopNode(:error), "wrong number of arguments"), []) end @assert na == length(argexprs) @@ -2558,7 +2560,7 @@ function inlineable(f::ANY, e::Expr, atype::ANY, sv::StaticVarInfo, enclosing_as lastexpr = pop!(body.args) if isa(lastexpr,LabelNode) push!(body.args, lastexpr) - push!(body.args, Expr(:call,:error,"fatal error in type inference")) + push!(body.args, Expr(:call, TopNode(:error), "fatal error in type inference")) lastexpr = nothing else @assert isa(lastexpr,Expr) "inference.jl:1774" @@ -2745,18 +2747,25 @@ function inlining_pass(e::Expr, sv, ast) end end end - f1 = f = isconstantfunc(arg1, sv) - if !is(f,false) - f = _ieval(f) - end - if (!isa(f,Function) && !isa(f,IntrinsicFunction) && - (f1 !== false || typeintersect(exprtype(arg1,sv), Function) === Bottom)) - modu = (inference_stack::CallStack).mod - if !_iisdefined(:call) - return (e,stmts) + + if isa(e.args[1], LambdaStaticData) + f = e.args[1] + else + f1 = f = isconstantfunc(arg1, sv) + if !is(f,false) + f = _ieval(f) + end + if (!isa(f,Function) && !isa(f,IntrinsicFunction) && + (f1 !== false || typeintersect(exprtype(arg1,sv), Function) === Bottom)) + modu = (inference_stack::CallStack).mod + if !_iisdefined(:call) + return (e,stmts) + end + f = _ieval(:call) + e.args = Any[is_global(sv,:call) ? (:call) : GlobalRef(modu, :call), e.args...] + elseif f1 === false + return (e, stmts) end - f = _ieval(:call) - e.args = Any[is_global(sv,:call) ? (:call) : GlobalRef(modu, :call), e.args...] end if isdefined(Main, :Base) && From a27ae96882911e3054c60120396429a971e1ad53 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 2 Oct 2015 23:58:20 -0400 Subject: [PATCH 0422/1938] add hook into inference for anonymous functions to provide custom tfunc --- base/inference.jl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 4086c97d72c80..fc190395ebb30 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -467,7 +467,7 @@ function tuple_tfunc(argtype::ANY) argtype end -function builtin_tfunction(f::ANY, args::ANY, argtype::ANY) +function builtin_tfunction(f::ANY, args::ANY, argtype::ANY, vtypes::ObjectIdDict, sv::StaticVarInfo) isva = isvatuple(argtype) argtypes = argtype.parameters if is(f,tuple) @@ -521,6 +521,12 @@ function builtin_tfunction(f::ANY, args::ANY, argtype::ANY) # wrong # of args return Bottom end + if isdefined(f, :code) + try # in case the user tfunc fails + return tf[3](argtypes, args, vtypes, sv) + end + return Any + end if is(f,typeassert) || is(f,getfield) || is(f,apply_type) || is(f,fieldtype) # TODO: case of apply(), where we do not have the args return tf[3](args, argtypes...) @@ -896,7 +902,7 @@ function abstract_call(f, fargs, argtypes::Vector{Any}, vtypes, sv::StaticVarInf return Any end end - rt = builtin_tfunction(f, fargs, Tuple{argtypes...}) + rt = builtin_tfunction(f, fargs, Tuple{argtypes...}, vtypes, sv) #print("=> ", rt, "\n") return rt end From 6bcb315d8a73a5d3523173e3d2eb88e796f5e3ba Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Fri, 2 Oct 2015 22:35:37 -0700 Subject: [PATCH 0423/1938] Mention license of SSL ca certs --- LICENSE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/LICENSE.md b/LICENSE.md index d1b76975eeac8..488126fe32c42 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -87,3 +87,5 @@ Julia bundles the following external programs and libraries on some platforms: - [GIT](http://git-scm.com/about/free-and-open-source) - [ZLIB](http://zlib.net/zlib_license.html) - [LIBEXPAT](http://expat.cvs.sourceforge.net/viewvc/expat/expat/README) + +On some platforms, distributions of Julia contain SSL certificate authority certificates, released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License). \ No newline at end of file From 4dd2b2593147cd67e88a91fc6bbd11a91660d546 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Fri, 2 Oct 2015 22:44:42 -0700 Subject: [PATCH 0424/1938] Add openssl to the list --- LICENSE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/LICENSE.md b/LICENSE.md index 488126fe32c42..f3a3c3e63d3b2 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -87,5 +87,6 @@ Julia bundles the following external programs and libraries on some platforms: - [GIT](http://git-scm.com/about/free-and-open-source) - [ZLIB](http://zlib.net/zlib_license.html) - [LIBEXPAT](http://expat.cvs.sourceforge.net/viewvc/expat/expat/README) +- [OPENSSL](https://github.com/openssl/openssl/blob/master/LICENSE) On some platforms, distributions of Julia contain SSL certificate authority certificates, released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License). \ No newline at end of file From 9d16f96732309e6699ccb14779ccd9933142af99 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 3 Oct 2015 01:26:22 -0400 Subject: [PATCH 0425/1938] correct the codegen type marking of constants and type-intersection test in emit_f_is. fixes #13432 --- src/alloc.c | 4 ++-- src/codegen.cpp | 17 +++++++++-------- test/core.jl | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 5b4dd0d550ee7..9e975dcaa9019 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -159,7 +159,7 @@ jl_value_t *jl_new_bits(jl_value_t *bt, void *data) return jl_new_bits_internal(bt, data, &len); } -// run time version of pointerref intrinsic +// run time version of pointerref intrinsic (warning: i is not rooted) DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i) { JL_TYPECHK(pointerref, pointer, p); @@ -192,7 +192,7 @@ void jl_assign_bits(void *dest, jl_value_t *bits) } } -// run time version of pointerset intrinsic +// run time version of pointerset intrinsic (warning: x is not gc-rooted) DLLEXPORT void jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t *i) { JL_TYPECHK(pointerset, pointer, p); diff --git a/src/codegen.cpp b/src/codegen.cpp index 365c6163f250f..aff636e9d7bdd 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1972,9 +1972,13 @@ static jl_cgval_t emit_getfield(jl_value_t *expr, jl_sym_t *name, jl_codectx_t * Value *bp = global_binding_pointer((jl_module_t*)expr, name, &bnd, false, ctx); // TODO: refactor. this partially duplicates code in emit_var if (bnd && bnd->value != NULL) { - if (bnd->constp && jl_isbits(jl_typeof(bnd->value))) - return emit_unboxed(bnd->value, ctx); - return mark_julia_type(builder.CreateLoad(bp), true, bnd->constp ? jl_typeof(bnd->value) : (jl_value_t*)jl_any_type); + if (bnd->constp) { + if (jl_isbits(jl_typeof(bnd->value))) + return emit_unboxed(bnd->value, ctx); + else + return mark_julia_const(bnd->value); + } + return mark_julia_type(builder.CreateLoad(bp), true, (jl_value_t*)jl_any_type); } // todo: use type info to avoid undef check return emit_checked_var(bp, name, ctx); @@ -2110,13 +2114,10 @@ static Value *emit_f_is(const jl_cgval_t &arg1, const jl_cgval_t &arg2, jl_codec } } - bool sub1 = jl_subtype(rt1, rt2, 0); - bool sub2 = jl_subtype(rt2, rt1, 0); - bool isteq = sub1 && sub2; - if (!sub1 && !sub2) // types are disjoint (exhaustive test) + if (jl_type_intersection(rt1, rt2) == (jl_value_t*)jl_bottom_type) // types are disjoint (exhaustive test) return ConstantInt::get(T_int1, 0); - bool isbits = isleaf && isteq && jl_isbits(rt1); + bool isbits = isleaf && jl_isbits(rt1) && jl_types_equal(rt1, rt2); if (isbits) { // whether this type is unique'd by value return emit_bits_compare(arg1, arg2, ctx); } diff --git a/test/core.jl b/test/core.jl index 6fabf8c54076a..66f200ea116ef 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3401,3 +3401,18 @@ b8932 = Vec3_8932(2,2,2) f13261() = (x = (error("oops"),); +(x...)) g13261() = f13261() @test_throws ErrorException g13261() + +# issue 13432 +@noinline function f13432(x) + offset = x ? Base.Bottom : 1 + return is(offset, Base.Bottom) +end +@test f13432(true) == true +@test f13432(false) == false +@noinline function f13432b(x) + a = x ? 1 : 1.0 + b = x ? 1 : 1.0f0 + return is(a, b) +end +@test f13432b(true) == true +@test f13432b(false) == false From 29f074aac80d9678d4731e312fb5faaf51d9ab1d Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 3 Oct 2015 00:53:23 -0400 Subject: [PATCH 0426/1938] Fix read!(s::IO, a::Vector{UInt8}) to return a lost in #12839 --- base/io.jl | 1 + test/core.jl | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/base/io.jl b/base/io.jl index ca653d8970dd9..817c1bcaad7a3 100644 --- a/base/io.jl +++ b/base/io.jl @@ -129,6 +129,7 @@ function read!(s::IO, a::Vector{UInt8}) for i in 1:length(a) a[i] = read(s, UInt8) end + return a end function read!{T}(s::IO, a::Array{T}) diff --git a/test/core.jl b/test/core.jl index 66f200ea116ef..c0c91b8795044 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3416,3 +3416,8 @@ end end @test f13432b(true) == true @test f13432b(false) == false + +#13433, read!(::IO, a::Vector{UInt8}) should return a +type IO13433 <: IO end +Base.read(::IO13433, ::Type{UInt8}) = 0x01 +@test read!(IO13433(), Array(UInt8, 4)) == [0x01, 0x01, 0x01, 0x01] From e4582bc77d687af783fdb92887c009843363897c Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Sat, 3 Oct 2015 14:40:00 -0400 Subject: [PATCH 0427/1938] Revert "Cleanup triangular multiplication and solves and remove redundant copies" This reverts commit 1b443ff05c3a20200762c4905fbcd2fe39699c23. --- base/linalg/triangular.jl | 52 +++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/base/linalg/triangular.jl b/base/linalg/triangular.jl index fe0a34cf95722..507cc0a1bffb8 100644 --- a/base/linalg/triangular.jl +++ b/base/linalg/triangular.jl @@ -979,25 +979,41 @@ for (f, g) in ((:*, :A_mul_B!), (:Ac_mul_B, :Ac_mul_B!), (:At_mul_B, :At_mul_B!) @eval begin function ($f){TA,TB}(A::AbstractTriangular{TA}, B::StridedVecOrMat{TB}) TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(convert(AbstractArray{TAB}, A), copy_oftype(B, TAB)) + ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) end end end ### Left division with triangle to the left hence rhs cannot be transposed. No quotients. for (f, g) in ((:\, :A_ldiv_B!), (:Ac_ldiv_B, :Ac_ldiv_B!), (:At_ldiv_B, :At_ldiv_B!)) @eval begin - function ($f){TA,TB,S}(A::AbstractTriangular{TA,S}, B::StridedVecOrMat{TB}) + function ($f){TA,TB,S}(A::UnitUpperTriangular{TA,S}, B::StridedVecOrMat{TB}) TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(convert(AbstractArray{TAB}, A), copy_oftype(B, TAB)) + ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + end + end +end +for (f, g) in ((:\, :A_ldiv_B!), (:Ac_ldiv_B, :Ac_ldiv_B!), (:At_ldiv_B, :At_ldiv_B!)) + @eval begin + function ($f){TA,TB,S}(A::UnitLowerTriangular{TA,S}, B::StridedVecOrMat{TB}) + TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) + ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) end end end ### Left division with triangle to the left hence rhs cannot be transposed. Quotients. for (f, g) in ((:\, :A_ldiv_B!), (:Ac_ldiv_B, :Ac_ldiv_B!), (:At_ldiv_B, :At_ldiv_B!)) @eval begin - function ($f){TA,TB,S}(A::AbstractTriangular{TA,S}, B::StridedVecOrMat{TB}) + function ($f){TA,TB,S}(A::UpperTriangular{TA,S}, B::StridedVecOrMat{TB}) + TAB = typeof((zero(TA)*zero(TB) + zero(TA)*zero(TB))/one(TA)) + ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + end + end +end +for (f, g) in ((:\, :A_ldiv_B!), (:Ac_ldiv_B, :Ac_ldiv_B!), (:At_ldiv_B, :At_ldiv_B!)) + @eval begin + function ($f){TA,TB,S}(A::LowerTriangular{TA,S}, B::StridedVecOrMat{TB}) TAB = typeof((zero(TA)*zero(TB) + zero(TA)*zero(TB))/one(TA)) - ($g)(convert(AbstractArray{TAB}, A), copy_oftype(B, TAB)) + ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) end end end @@ -1006,25 +1022,41 @@ for (f, g) in ((:*, :A_mul_B!), (:A_mul_Bc, :A_mul_Bc!), (:A_mul_Bt, :A_mul_Bt!) @eval begin function ($f){TA,TB}(A::StridedVecOrMat{TA}, B::AbstractTriangular{TB}) TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(copy_oftype(A, TAB), convert(AbstractArray{TAB}, B)) + ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) end end end ### Right division with triangle to the right hence lhs cannot be transposed. No quotients. for (f, g) in ((:/, :A_rdiv_B!), (:A_rdiv_Bc, :A_rdiv_Bc!), (:A_rdiv_Bt, :A_rdiv_Bt!)) @eval begin - function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::AbstractTriangular{TB,S}) + function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::UnitUpperTriangular{TB,S}) TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(copy_oftype(A, TAB), convert(AbstractArray{TAB}, B)) + ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + end + end +end +for (f, g) in ((:/, :A_rdiv_B!), (:A_rdiv_Bc, :A_rdiv_Bc!), (:A_rdiv_Bt, :A_rdiv_Bt!)) + @eval begin + function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::UnitLowerTriangular{TB,S}) + TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) + ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) end end end ### Right division with triangle to the right hence lhs cannot be transposed. Quotients. for (f, g) in ((:/, :A_rdiv_B!), (:A_rdiv_Bc, :A_rdiv_Bc!), (:A_rdiv_Bt, :A_rdiv_Bt!)) @eval begin - function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::AbstractTriangular{TB,S}) + function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::UpperTriangular{TB,S}) + TAB = typeof((zero(TA)*zero(TB) + zero(TA)*zero(TB))/one(TA)) + ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + end + end +end +for (f, g) in ((:/, :A_rdiv_B!), (:A_rdiv_Bc, :A_rdiv_Bc!), (:A_rdiv_Bt, :A_rdiv_Bt!)) + @eval begin + function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::LowerTriangular{TB,S}) TAB = typeof((zero(TA)*zero(TB) + zero(TA)*zero(TB))/one(TA)) - ($g)(copy_oftype(A, TAB), convert(AbstractArray{TAB}, B)) + ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) end end end From 127ee86b89de9e968e1f90cf43483b1368dd3d6c Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Sat, 3 Oct 2015 19:33:41 -0400 Subject: [PATCH 0428/1938] Cleanup triangular multiplication and solves and remove redundant copies. Take two. --- base/linalg/triangular.jl | 52 ++++++++------------------------------- 1 file changed, 10 insertions(+), 42 deletions(-) diff --git a/base/linalg/triangular.jl b/base/linalg/triangular.jl index 507cc0a1bffb8..2996dbcbdcaf8 100644 --- a/base/linalg/triangular.jl +++ b/base/linalg/triangular.jl @@ -979,41 +979,25 @@ for (f, g) in ((:*, :A_mul_B!), (:Ac_mul_B, :Ac_mul_B!), (:At_mul_B, :At_mul_B!) @eval begin function ($f){TA,TB}(A::AbstractTriangular{TA}, B::StridedVecOrMat{TB}) TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + ($g)(convert(AbstractArray{TAB}, A), copy_oftype(B, TAB)) end end end ### Left division with triangle to the left hence rhs cannot be transposed. No quotients. for (f, g) in ((:\, :A_ldiv_B!), (:Ac_ldiv_B, :Ac_ldiv_B!), (:At_ldiv_B, :At_ldiv_B!)) @eval begin - function ($f){TA,TB,S}(A::UnitUpperTriangular{TA,S}, B::StridedVecOrMat{TB}) + function ($f){TA,TB,S}(A::Union{UnitUpperTriangular{TA,S},UnitLowerTriangular{TA,S}}, B::StridedVecOrMat{TB}) TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) - end - end -end -for (f, g) in ((:\, :A_ldiv_B!), (:Ac_ldiv_B, :Ac_ldiv_B!), (:At_ldiv_B, :At_ldiv_B!)) - @eval begin - function ($f){TA,TB,S}(A::UnitLowerTriangular{TA,S}, B::StridedVecOrMat{TB}) - TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + ($g)(convert(AbstractArray{TAB}, A), copy_oftype(B, TAB)) end end end ### Left division with triangle to the left hence rhs cannot be transposed. Quotients. for (f, g) in ((:\, :A_ldiv_B!), (:Ac_ldiv_B, :Ac_ldiv_B!), (:At_ldiv_B, :At_ldiv_B!)) @eval begin - function ($f){TA,TB,S}(A::UpperTriangular{TA,S}, B::StridedVecOrMat{TB}) - TAB = typeof((zero(TA)*zero(TB) + zero(TA)*zero(TB))/one(TA)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) - end - end -end -for (f, g) in ((:\, :A_ldiv_B!), (:Ac_ldiv_B, :Ac_ldiv_B!), (:At_ldiv_B, :At_ldiv_B!)) - @eval begin - function ($f){TA,TB,S}(A::LowerTriangular{TA,S}, B::StridedVecOrMat{TB}) + function ($f){TA,TB,S}(A::Union{UpperTriangular{TA,S},LowerTriangular{TA,S}}, B::StridedVecOrMat{TB}) TAB = typeof((zero(TA)*zero(TB) + zero(TA)*zero(TB))/one(TA)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + ($g)(convert(AbstractArray{TAB}, A), copy_oftype(B, TAB)) end end end @@ -1022,41 +1006,25 @@ for (f, g) in ((:*, :A_mul_B!), (:A_mul_Bc, :A_mul_Bc!), (:A_mul_Bt, :A_mul_Bt!) @eval begin function ($f){TA,TB}(A::StridedVecOrMat{TA}, B::AbstractTriangular{TB}) TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + ($g)(copy_oftype(A, TAB), convert(AbstractArray{TAB}, B)) end end end ### Right division with triangle to the right hence lhs cannot be transposed. No quotients. for (f, g) in ((:/, :A_rdiv_B!), (:A_rdiv_Bc, :A_rdiv_Bc!), (:A_rdiv_Bt, :A_rdiv_Bt!)) @eval begin - function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::UnitUpperTriangular{TB,S}) + function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::Union{UnitUpperTriangular{TB,S},UnitLowerTriangular{TB,S}}) TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) - end - end -end -for (f, g) in ((:/, :A_rdiv_B!), (:A_rdiv_Bc, :A_rdiv_Bc!), (:A_rdiv_Bt, :A_rdiv_Bt!)) - @eval begin - function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::UnitLowerTriangular{TB,S}) - TAB = typeof(zero(TA)*zero(TB) + zero(TA)*zero(TB)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + ($g)(copy_oftype(A, TAB), convert(AbstractArray{TAB}, B)) end end end ### Right division with triangle to the right hence lhs cannot be transposed. Quotients. for (f, g) in ((:/, :A_rdiv_B!), (:A_rdiv_Bc, :A_rdiv_Bc!), (:A_rdiv_Bt, :A_rdiv_Bt!)) @eval begin - function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::UpperTriangular{TB,S}) - TAB = typeof((zero(TA)*zero(TB) + zero(TA)*zero(TB))/one(TA)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) - end - end -end -for (f, g) in ((:/, :A_rdiv_B!), (:A_rdiv_Bc, :A_rdiv_Bc!), (:A_rdiv_Bt, :A_rdiv_Bt!)) - @eval begin - function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::LowerTriangular{TB,S}) + function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::Union{UnitUpperTriangular{TB,S},UnitLowerTriangular{TB,S}}) TAB = typeof((zero(TA)*zero(TB) + zero(TA)*zero(TB))/one(TA)) - ($g)(TA == TAB ? copy(A) : convert(AbstractArray{TAB}, A), TB == TAB ? copy(B) : convert(AbstractArray{TAB}, B)) + ($g)(copy_oftype(A, TAB), convert(AbstractArray{TAB}, B)) end end end From 199ebeffc436e644f1ce57647f1d1730f75a66cb Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 3 Oct 2015 19:11:50 -0700 Subject: [PATCH 0429/1938] fix appveyor 64 for openblas suffix --- contrib/windows/msys_build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index 27caae65a75a7..64db00243f3df 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -56,12 +56,16 @@ if [ "$ARCH" = x86_64 ]; then exc=seh echo "override MARCH = x86-64" >> Make.user echo 'USE_BLAS64 = 1' >> Make.user + echo 'LIBBLAS = -L$(JULIAHOME)/usr/bin -lopenblas64_' >> Make.user + echo 'LIBBLASNAME = libopenblas64_' >> Make.user else bits=32 archsuffix=86 exc=sjlj echo "override MARCH = i686" >> Make.user echo "override JULIA_CPU_TARGET = pentium4" >> Make.user + echo 'LIBBLAS = -L$(JULIAHOME)/usr/bin -lopenblas' >> Make.user + echo 'LIBBLASNAME = libopenblas' >> Make.user fi # Set XC_HOST if in Cygwin or Linux @@ -185,8 +189,6 @@ for lib in SUITESPARSE ARPACK BLAS LAPACK FFTW \ GMP MPFR PCRE LIBUNWIND RMATH OPENSPECFUN; do echo "USE_SYSTEM_$lib = 1" >> Make.user done -echo 'LIBBLAS = -L$(JULIAHOME)/usr/bin -lopenblas' >> Make.user -echo 'LIBBLASNAME = libopenblas' >> Make.user echo 'override LIBLAPACK = $(LIBBLAS)' >> Make.user echo 'override LIBLAPACKNAME = $(LIBBLASNAME)' >> Make.user echo 'JULIA_SYSIMG_BUILD_FLAGS=--output-ji ../usr/lib/julia/sys.ji' >> Make.user From b3a2e82cd78045ede281104992d5c2fb5e79e6d0 Mon Sep 17 00:00:00 2001 From: Jim Garrison Date: Sat, 3 Oct 2015 22:44:53 -0700 Subject: [PATCH 0430/1938] Implement complex(::Type) --- base/complex.jl | 3 +++ test/complex.jl | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/base/complex.jl b/base/complex.jl index 99cc4d2cd29fd..7b8d7a73f64ce 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -37,6 +37,9 @@ reim(z) = (real(z), imag(z)) real{T<:Real}(::Type{T}) = T real{T<:Real}(::Type{Complex{T}}) = T +complex{T<:Real}(::Type{T}) = Complex{T} +complex{T<:Real}(::Type{Complex{T}}) = Complex{T} + isreal(x::Real) = true isreal(z::Complex) = imag(z) == 0 isimag(z::Number) = real(z) == 0 diff --git a/test/complex.jl b/test/complex.jl index fe3c1188001cb..228124e5d8afa 100644 --- a/test/complex.jl +++ b/test/complex.jl @@ -2,6 +2,13 @@ @test reim(2 + 3im) == (2, 3) +for T in (Int64, Float64) + @test real(T) == T + @test real(Complex{T}) == T + @test complex(T) == Complex{T} + @test complex(Complex{T}) == Complex{T} +end + # Basic arithmetic for T in (Float16, Float32, Float64, BigFloat) t = true From db562bce72919f3ede02ef8dd95a08f82460365d Mon Sep 17 00:00:00 2001 From: ScottPJones Date: Sun, 4 Oct 2015 11:14:27 -0400 Subject: [PATCH 0431/1938] Fixed #13442 Warning in codegen.cpp when assertions off --- src/codegen.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 8b34d443c78cf..c1ca8adb2f401 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3973,7 +3973,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t } else { Value *arg; - Type *at = theFptr->getFunctionType()->getParamType(FParamIndex++); + FParamIndex++; if (isboxed) { arg = boxed(inputarg, &ctx); make_gcroot(arg, &ctx); @@ -3982,6 +3982,9 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t arg = emit_unbox(t, inputarg, jargty); assert(!isa(arg)); if (t->isAggregateType()) { +#ifndef NDEBUG + Type *at = theFptr->getFunctionType()->getParamType(FParamIndex-1); +#endif assert(at->isPointerTy() && at->getContainedType(0) == t); // aggregate types are passed by pointer Value *loc = emit_static_alloca(t, &ctx); From d9b443c49dc4ca07c770e0c4cb60e39338667fa0 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sun, 4 Oct 2015 20:41:00 -0400 Subject: [PATCH 0432/1938] Bump libgit2 to 0.23.2 --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - deps/libgit2.version | 4 ++-- 5 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 deps/checksums/libgit2-2de198b4cec26c2b54c06da4baf88b3f57b9ca86.tar.gz/md5 create mode 100644 deps/checksums/libgit2-2de198b4cec26c2b54c06da4baf88b3f57b9ca86.tar.gz/sha512 delete mode 100644 deps/checksums/libgit2-f6dedf2c2eb806e2a6fdd4cf31f68386efc2ee0b.tar.gz/md5 delete mode 100644 deps/checksums/libgit2-f6dedf2c2eb806e2a6fdd4cf31f68386efc2ee0b.tar.gz/sha512 diff --git a/deps/checksums/libgit2-2de198b4cec26c2b54c06da4baf88b3f57b9ca86.tar.gz/md5 b/deps/checksums/libgit2-2de198b4cec26c2b54c06da4baf88b3f57b9ca86.tar.gz/md5 new file mode 100644 index 0000000000000..077b2ea83bae9 --- /dev/null +++ b/deps/checksums/libgit2-2de198b4cec26c2b54c06da4baf88b3f57b9ca86.tar.gz/md5 @@ -0,0 +1 @@ +c78031212e56b80b3b9c6e06990d2396 diff --git a/deps/checksums/libgit2-2de198b4cec26c2b54c06da4baf88b3f57b9ca86.tar.gz/sha512 b/deps/checksums/libgit2-2de198b4cec26c2b54c06da4baf88b3f57b9ca86.tar.gz/sha512 new file mode 100644 index 0000000000000..a6d2effef8e2b --- /dev/null +++ b/deps/checksums/libgit2-2de198b4cec26c2b54c06da4baf88b3f57b9ca86.tar.gz/sha512 @@ -0,0 +1 @@ +8a716e16145cfe9319fdb812f754a473096d643e38f67887301d6a90c25d0b888f367e3e80f8eac6396a908e654ad333cd19455eaec9f6d489dfa167982f7558 diff --git a/deps/checksums/libgit2-f6dedf2c2eb806e2a6fdd4cf31f68386efc2ee0b.tar.gz/md5 b/deps/checksums/libgit2-f6dedf2c2eb806e2a6fdd4cf31f68386efc2ee0b.tar.gz/md5 deleted file mode 100644 index ed9ac40afad0c..0000000000000 --- a/deps/checksums/libgit2-f6dedf2c2eb806e2a6fdd4cf31f68386efc2ee0b.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -5ec6d4caf6218c27b73f7c00226de7aa diff --git a/deps/checksums/libgit2-f6dedf2c2eb806e2a6fdd4cf31f68386efc2ee0b.tar.gz/sha512 b/deps/checksums/libgit2-f6dedf2c2eb806e2a6fdd4cf31f68386efc2ee0b.tar.gz/sha512 deleted file mode 100644 index d3fe19ddf87fd..0000000000000 --- a/deps/checksums/libgit2-f6dedf2c2eb806e2a6fdd4cf31f68386efc2ee0b.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -76d85f5fce5c7a9ee142a7eb216253618845efd60962f2140865a2953faffc95d3855e5b87fd5cb1e0ef8d46cc93e9f51582adc60530ef1e86b0493013e16f49 diff --git a/deps/libgit2.version b/deps/libgit2.version index 466d0faca846f..7c0893dbd81cd 100644 --- a/deps/libgit2.version +++ b/deps/libgit2.version @@ -1,2 +1,2 @@ -LIBGIT2_BRANCH=v0.23.1 -LIBGIT2_SHA1=f6dedf2c2eb806e2a6fdd4cf31f68386efc2ee0b +LIBGIT2_BRANCH=v0.23.2 +LIBGIT2_SHA1=2de198b4cec26c2b54c06da4baf88b3f57b9ca86 From ad3ebba8df7a51ad92835cb9138050731c4b3ef6 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sun, 4 Oct 2015 20:44:56 -0400 Subject: [PATCH 0433/1938] Fail libgit2 configuration if openssl not found on Linux --- deps/Makefile | 3 +++ deps/libgit2-require-openssl.patch | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 deps/libgit2-require-openssl.patch diff --git a/deps/Makefile b/deps/Makefile index c00fedf06576f..7f935ac5f2d1f 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -2026,9 +2026,12 @@ else LIBGIT2_OPTS += -DBUILD_CLAR=OFF -DCMAKE_RC_COMPILER=`which $(CROSS_COMPILE)windres` -DDLLTOOL=`which $(CROSS_COMPILE)dlltool` LIBGIT2_OPTS += -DCMAKE_FIND_ROOT_PATH=/usr/$(XC_HOST) -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY endif +else ifneq ($(OS), Darwin) +LIBGIT2_OPTS += -DREQUIRE_OPENSSL=ON endif $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt + -cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/libgit2-require-openssl.patch mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(LIBGIT2_OPTS) diff --git a/deps/libgit2-require-openssl.patch b/deps/libgit2-require-openssl.patch new file mode 100644 index 0000000000000..fce6e10635397 --- /dev/null +++ b/deps/libgit2-require-openssl.patch @@ -0,0 +1,13 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 73c9630..3dedd0a 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -217,7 +217,7 @@ ELSE () + ENDIF () + + IF (NOT AMIGA AND USE_OPENSSL) +- FIND_PACKAGE(OpenSSL) ++ FIND_PACKAGE(OpenSSL REQUIRED) + ENDIF () + + IF (CURL_FOUND) From 218ac58c5c4a1770136c7fefac6fd1605bdfc5a6 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sun, 4 Oct 2015 20:55:33 -0400 Subject: [PATCH 0434/1938] Document libssl-dev build dependency closes #13389 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f3a64eaeac863..3aef7d3c1f00c 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,7 @@ Building Julia requires that the following software be installed: - **[m4]** — needed to build GMP. - **[patch]** — for modifying source code. - **[cmake]** — needed to build `libgit2`. +- **[openssl]** — needed for HTTPS support in `libgit2` on Linux, install via `apt-get install libssl-dev` or `yum install openssl-devel`. Julia uses the following external libraries, which are automatically downloaded (or in a few cases, included in the Julia source repository) and then compiled from source the first time you run `make`: @@ -317,6 +318,7 @@ For a longer overview of Julia's dependencies, see these [slides](https://github [libosxunwind]: https://github.com/JuliaLang/libosxunwind [libunwind]: http://www.nongnu.org/libunwind [Rmath-julia]: https://github.com/JuliaLang/Rmath-julia +[openssl]: https://www.openssl.org ### System Provided Libraries From 927e530f8e51de2a18573caf29d773733c00f2b3 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sun, 4 Oct 2015 21:39:20 -0700 Subject: [PATCH 0435/1938] Fix warning messages by using file existence check --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ae22c8169610c..c8abbf768adf5 100644 --- a/Makefile +++ b/Makefile @@ -449,7 +449,7 @@ endif ifeq ($(OS), Linux) -$(JULIAHOME)/contrib/fixup-libstdc++.sh $(DESTDIR)$(private_libdir) # We need to bundle ca certs on linux now that we're using libgit2 with ssl -ifneq ($(shell cat $(shell openssl version -d | cut -d '"' -f 2)/cert.pem),) +ifneq ($(shell [ -e $(shell openssl version -d | cut -d '"' -f 2)/cert.pem ] && echo exists),exists) -cp $(shell openssl version -d | cut -d '"' -f 2)/cert.pem $(DESTDIR)$(datarootdir)/julia/ endif endif From 916f0e0a1deae865b9e578e44ac857a87091efd6 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sun, 4 Oct 2015 22:18:29 -0700 Subject: [PATCH 0436/1938] ifneq -> ifeq --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c8abbf768adf5..fa98ad496afda 100644 --- a/Makefile +++ b/Makefile @@ -449,7 +449,7 @@ endif ifeq ($(OS), Linux) -$(JULIAHOME)/contrib/fixup-libstdc++.sh $(DESTDIR)$(private_libdir) # We need to bundle ca certs on linux now that we're using libgit2 with ssl -ifneq ($(shell [ -e $(shell openssl version -d | cut -d '"' -f 2)/cert.pem ] && echo exists),exists) +ifeq ($(shell [ -e $(shell openssl version -d | cut -d '"' -f 2)/cert.pem ] && echo exists),exists) -cp $(shell openssl version -d | cut -d '"' -f 2)/cert.pem $(DESTDIR)$(datarootdir)/julia/ endif endif From 53c2a2c9d5e9e7ccff3f30e83011d6959e79bf9f Mon Sep 17 00:00:00 2001 From: wildart Date: Fri, 2 Oct 2015 02:05:32 -0400 Subject: [PATCH 0437/1938] added credentials callback with payload to all high livel network functions: clone, fetch, push credential callback can handle payload added `fetch_refspecs` & `push_refspecs` funcs for remote reference added function for github credentials refined repo initialization tests fixed branch head switching added signature parameter for `tag_create` added checkout options as a parameter improved libgit2 tests & separated online reqired provide signature to avoid reading from git global configuration fixed tests to handle testing system with(out) a default git configuration fixed typos --- .travis.yml | 2 +- appveyor.yml | 2 +- base/libgit2.jl | 56 +++-- base/libgit2/callbacks.jl | 51 ++-- base/libgit2/config.jl | 50 ++-- base/libgit2/consts.jl | 8 + base/libgit2/index.jl | 4 + base/libgit2/merge.jl | 15 +- base/libgit2/remote.jl | 22 ++ base/libgit2/tag.jl | 10 +- base/libgit2/types.jl | 19 ++ base/libgit2/utils.jl | 13 +- base/pkg/entry.jl | 2 +- base/pkg/github.jl | 17 +- test/libgit2-online.jl | 59 +++++ test/libgit2.jl | 506 +++++++++++++++++++++++++------------- test/pkg.jl | 2 +- 17 files changed, 589 insertions(+), 249 deletions(-) create mode 100644 test/libgit2-online.jl diff --git a/.travis.yml b/.travis.yml index f1137de6d3c50..043df379cc610 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,7 +66,7 @@ script: - cd .. && mv julia julia2 - cp /tmp/julia/lib/julia/sys.ji local.ji && /tmp/julia/bin/julia -J local.ji -e 'true' && /tmp/julia/bin/julia-debug -J local.ji -e 'true' && rm local.ji - /tmp/julia/bin/julia -e 'versioninfo()' - - export JULIA_CPU_CORES=2 && cd /tmp/julia/share/julia/test && /tmp/julia/bin/julia --check-bounds=yes runtests.jl all && /tmp/julia/bin/julia --check-bounds=yes runtests.jl pkg + - export JULIA_CPU_CORES=2 && cd /tmp/julia/share/julia/test && /tmp/julia/bin/julia --check-bounds=yes runtests.jl all && /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online pkg - cd - && mv julia2 julia - sudo dmesg - echo "Ready for packaging..." diff --git a/appveyor.yml b/appveyor.yml index 1634ea19f6f6a..6173d881d7544 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -49,4 +49,4 @@ build_script: test_script: - usr\bin\julia -e "versioninfo()" - copy usr\lib\julia\sys.ji local.ji && usr\bin\julia -J local.ji -e "true" && del local.ji - - cd test && ..\usr\bin\julia runtests.jl all && ..\usr\bin\julia runtests.jl pkg + - cd test && ..\usr\bin\julia runtests.jl all && ..\usr\bin\julia runtests.jl libgit2-online pkg diff --git a/base/libgit2.jl b/base/libgit2.jl index c2779489fac90..b2b4dd141aa2c 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -9,6 +9,7 @@ export with, GitRepo, GitConfig const GITHUB_REGEX = r"^(?:git@|git://|https://(?:[\w\.\+\-]+@)?)github.com[:/](([^/].+)/(.+?))(?:\.git)?$"i +include("libgit2/utils.jl") include("libgit2/consts.jl") include("libgit2/types.jl") include("libgit2/error.jl") @@ -30,7 +31,6 @@ include("libgit2/rebase.jl") include("libgit2/status.jl") include("libgit2/tree.jl") include("libgit2/callbacks.jl") -include("libgit2/utils.jl") immutable State head::Oid @@ -141,17 +141,18 @@ function set_remote_url(path::AbstractString, url::AbstractString; remote::Abstr end """ git fetch [|] []""" -function fetch{T<:AbstractString}(repo::GitRepo; +function fetch{T<:AbstractString, P<:AbstractPayload}(repo::GitRepo; remote::AbstractString="origin", remoteurl::AbstractString="", - refspecs::Vector{T}=AbstractString[]) + refspecs::Vector{T}=AbstractString[], + payload::Nullable{P}=Nullable{AbstractPayload}()) rmt = if isempty(remoteurl) get(GitRemote, repo, remote) else GitRemoteAnon(repo, remoteurl) end try - fo = FetchOptions(callbacks=RemoteCallbacks(credentials=credentials_cb())) + fo = FetchOptions(callbacks=RemoteCallbacks(credentials_cb(), payload)) fetch(rmt, refspecs, msg="from $(url(rmt))", options = fo) catch err warn("fetch: $err") @@ -161,19 +162,20 @@ function fetch{T<:AbstractString}(repo::GitRepo; end """ git push [|] []""" -function push{T<:AbstractString}(repo::GitRepo; +function push{T<:AbstractString, P<:AbstractPayload}(repo::GitRepo; remote::AbstractString="origin", remoteurl::AbstractString="", refspecs::Vector{T}=AbstractString[], - force::Bool=false) + force::Bool=false, + payload::Nullable{P}=Nullable{AbstractPayload}()) rmt = if isempty(remoteurl) get(GitRemote, repo, remote) else GitRemoteAnon(repo, remoteurl) end try - po = PushOptions(callbacks=RemoteCallbacks(credentials=credentials_cb())) - push(rmt, refspecs, force=force, options=po) + push_opts=PushOptions(callbacks=RemoteCallbacks(credentials_cb(), payload)) + push(rmt, refspecs, force=force, options=push_opts) finally finalize(rmt) end @@ -232,13 +234,15 @@ function branch!(repo::GitRepo, branch_name::AbstractString, end end - # checout selected branch - with(peel(GitTree, branch_ref)) do btree - checkout_tree(repo, btree) - end + if set_head + # checkout selected branch + with(peel(GitTree, branch_ref)) do btree + checkout_tree(repo, btree) + end - # switch head to the branch - set_head && head!(repo, branch_ref) + # switch head to the branch + head!(repo, branch_ref) + end finally finalize(branch_ref) end @@ -289,17 +293,20 @@ function checkout!(repo::GitRepo, commit::AbstractString = ""; end """ git clone [-b ] [--bare] """ -function clone(repo_url::AbstractString, repo_path::AbstractString; +function clone{P<:AbstractPayload}(repo_url::AbstractString, repo_path::AbstractString; branch::AbstractString="", isbare::Bool = false, - remote_cb::Ptr{Void} = C_NULL) - # setup colne options + remote_cb::Ptr{Void} = C_NULL, + payload::Nullable{P}=Nullable{AbstractPayload}()) + # setup clone options + fetch_opts=FetchOptions(callbacks = RemoteCallbacks(credentials_cb(), payload)) clone_opts = CloneOptions( - bare = Int32(isbare), - checkout_branch = isempty(branch) ? Cstring_NULL : - convert(Cstring, pointer(branch)), - remote_cb = remote_cb - ) + bare = Int32(isbare), + checkout_branch = isempty(branch) ? Cstring_NULL : + convert(Cstring, pointer(branch)), + fetch_opts=fetch_opts, + remote_cb = remote_cb + ) return clone(repo_url, repo_path, clone_opts) end @@ -357,7 +364,8 @@ function merge!(repo::GitRepo; committish::AbstractString = "", branch::AbstractString = "", fastforward::Bool = false, - options = MergeOptions()) + merge_opts::MergeOptions = MergeOptions(), + checkout_opts::CheckoutOptions = CheckoutOptions()) # merge into head branch with(head(repo)) do head_ref upst_anns = if !isempty(committish) # merge committish into HEAD @@ -383,7 +391,7 @@ function merge!(repo::GitRepo; end try - merge!(repo, upst_anns, fastforward, options) + merge!(repo, upst_anns, fastforward, merge_opts, checkout_opts=checkout_opts) finally for ann in upst_anns finalize(ann) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index fe1899f8a0c11..78f6536736800 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -1,14 +1,7 @@ -function prompt(msg::AbstractString; default::AbstractString="", password::Bool=false) - msg = length(default) > 0 ? msg*" [$default]:" : msg*":" - uinput = if password - bytestring(ccall(:getpass, Cstring, (Cstring,), msg)) - else - print(msg) - chomp(readline(STDIN)) - end - length(uinput) == 0 ? default : uinput -end +"""Mirror callback function +Function sets `+refs/*:refs/*` refspecs and `mirror` flag for remote reference. +""" function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, name::Cstring, url::Cstring, payload::Ptr{Void}) # Create the remote with a mirroring url @@ -29,20 +22,45 @@ function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, return Cint(0) end +"""Credentials callback function + +Function provides different credential acquisition functionality w.r.t. a connection protocol. +If payload is provided then `payload_ptr` should contain `LibGit2.AbstractPayload` object. + +For `LibGit2.Consts.CREDTYPE_USERPASS_PLAINTEXT` type, if payload contains fields: `user` & `pass` they are used to create authentication credentials. +In addition, if payload has field `used` it can be set to `true` to indicate that payload was used and abort callback with error. This behavior is required to avoid repeated authentication calls with incorrect credentials. +""" function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, username_ptr::Cstring, - allowed_types::Cuint, payload::Ptr{Void}) + allowed_types::Cuint, payload_ptr::Ptr{Void}) err = 1 url = bytestring(url_ptr) if isset(allowed_types, Cuint(Consts.CREDTYPE_USERPASS_PLAINTEXT)) - # use keyboard-interactive prompt - username = prompt("Username for '$url'") - pass = prompt("Password for '$url'", password=true) + username = password = "" + if payload_ptr != C_NULL + payload = unsafe_pointer_to_objref(payload_ptr) + if isa(payload, AbstractPayload) + username = isdefined(payload, :user) ? getfield(payload, :user) : "" + password = isdefined(payload, :pass) ? getfield(payload, :pass) : "" + if isdefined(payload, :used) + getfield(payload, :used) && return Cint(-1) + setfield!(payload, :used, true) + end + end + end + if isempty(username) + username = prompt("Username for '$url'") + end + if isempty(password) + password = prompt("Password for '$username@$url'", password=true) + end + + length(username) == 0 && length(password) == 0 && return Cint(-1) err = ccall((:git_cred_userpass_plaintext_new, :libgit2), Cint, (Ptr{Ptr{Void}}, Cstring, Cstring), - cred, username, pass) + cred, username, password) err == 0 && return Cint(0) elseif isset(allowed_types, Cuint(Consts.CREDTYPE_SSH_KEY)) && err > 0 # use ssh-agent @@ -91,6 +109,9 @@ function fetchhead_foreach_callback(ref_name::Cstring, remote_url::Cstring, return Cint(0) end +"C function pointer for `mirror_callback`" mirror_cb() = cfunction(mirror_callback, Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring, Ptr{Void})) +"C function pointer for `credentials_callback`" credentials_cb() = cfunction(credentials_callback, Cint, (Ptr{Ptr{Void}}, Cstring, Cstring, Cuint, Ptr{Void})) +"C function pointer for `fetchhead_foreach_callback`" fetchhead_foreach_cb() = cfunction(fetchhead_foreach_callback, Cint, (Cstring, Cstring, Ptr{Oid}, Cuint, Ptr{Void})) \ No newline at end of file diff --git a/base/libgit2/config.jl b/base/libgit2/config.jl index 532e0b49d0000..b8aa0b33b2c34 100644 --- a/base/libgit2/config.jl +++ b/base/libgit2/config.jl @@ -1,36 +1,56 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -function GitConfig(path::AbstractString) +function GitConfig(path::AbstractString, + level::Consts.GIT_CONFIG = Consts.CONFIG_LEVEL_APP, + force::Bool=false) + # create new config object cfg_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - err = ccall((:git_config_open_ondisk, :libgit2), Cint, - (Ptr{Ptr{Void}}, Cstring), cfg_ptr_ptr, path) - err !=0 && return nothing - return GitConfig(cfg_ptr_ptr[]) + @check ccall((:git_config_new, :libgit2), Cint, (Ptr{Ptr{Void}},), cfg_ptr_ptr) + cfg = GitConfig(cfg_ptr_ptr[]) + try + addfile(cfg, path, level, force) + catch ex + finalize(cfg) + throw(ex) + end + return cfg end function GitConfig(r::GitRepo) cfg_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - err = ccall((:git_repository_config, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}), cfg_ptr_ptr, r.ptr) - err !=0 && return nothing + @check ccall((:git_repository_config, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}), cfg_ptr_ptr, r.ptr) return GitConfig(cfg_ptr_ptr[]) end -function GitConfig(isglobal::Bool = true) +function GitConfig(level::Consts.GIT_CONFIG = Consts.CONFIG_LEVEL_DEFAULT) cfg_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_config_open_default, :libgit2), Cint, (Ptr{Ptr{Void}}, ), cfg_ptr_ptr) cfg = GitConfig(cfg_ptr_ptr[]) - if isglobal + if level != Consts.CONFIG_LEVEL_DEFAULT glb_cfg_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - @check ccall((:git_config_open_global, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}), glb_cfg_ptr_ptr, cfg.ptr) - finalize(cfg) - cfg = GitConfig(glb_cfg_ptr_ptr[]) + tmpcfg = cfg + try + @check ccall((:git_config_open_level, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ptr{Void}, Cint), + glb_cfg_ptr_ptr, cfg.ptr, Cint(level)) + cfg = GitConfig(glb_cfg_ptr_ptr[]) + finally + finalize(tmpcfg) + end end return cfg end +function addfile(cfg::GitConfig, path::AbstractString, + level::Consts.GIT_CONFIG = Consts.CONFIG_LEVEL_APP, + force::Bool=false) + @check ccall((:git_config_add_file_ondisk, :libgit2), Cint, + (Ptr{Ptr{Void}}, Cstring, Cint, Cint), + cfg.ptr, path, Cint(level), Cint(force)) +end + function get{T<:AbstractString}(::Type{T}, c::GitConfig, name::AbstractString) buf_ptr = Ref(Buffer()) @check ccall((:git_config_get_string_buf, :libgit2), Cint, @@ -57,7 +77,7 @@ end function get(::Type{Int64}, c::GitConfig, name::AbstractString) val_ptr = Ref(Cintmax_t(0)) @check ccall((:git_config_get_int64, :libgit2), Cint, - (Ptr{Cint}, Ptr{Void}, Cstring), val_ptr, c.ptr, name) + (Ptr{Cintmax_t}, Ptr{Void}, Cstring), val_ptr, c.ptr, name) return val_ptr[] end diff --git a/base/libgit2/consts.jl b/base/libgit2/consts.jl index eac2933d02dac..ee24ec86c9367 100644 --- a/base/libgit2/consts.jl +++ b/base/libgit2/consts.jl @@ -263,4 +263,12 @@ module Consts CREDTYPE_SSH_INTERACTIVE = Cuint(1 << 4), CREDTYPE_USERNAME = Cuint(1 << 5), CREDTYPE_SSH_MEMORY = Cuint(1 << 6)) + + @enum(GIT_CONFIG, CONFIG_LEVEL_DEFAULT = 0, + CONFIG_LEVEL_SYSTEM = 1, # System-wide configuration file; /etc/gitconfig on Linux systems + CONFIG_LEVEL_XDG = 2, # XDG compatible configuration file; typically ~/.config/git/config + CONFIG_LEVEL_GLOBAL = 3, # User-specific configuration file (also called Global configuration file); typically ~/.gitconfig + CONFIG_LEVEL_LOCAL = 4, # Repository specific configuration file; $WORK_DIR/.git/config on non-bare repos + CONFIG_LEVEL_APP = 5, # Application specific configuration file; freely defined by applications + CONFIG_HIGHEST_LEVEL =-1) # Represents the highest level available config file (i.e. the most specific config file available that actually is loaded) end diff --git a/base/libgit2/index.jl b/base/libgit2/index.jl index c38d3257e5834..e5b349024328a 100644 --- a/base/libgit2/index.jl +++ b/base/libgit2/index.jl @@ -81,6 +81,7 @@ function add!{T<:AbstractString}(repo::GitRepo, files::T...; add!(idx, files..., flags = flags) write!(idx) end + return end function update!{T<:AbstractString}(repo::GitRepo, files::T...) @@ -88,6 +89,7 @@ function update!{T<:AbstractString}(repo::GitRepo, files::T...) update!(idx, files...) write!(idx) end + return end function remove!{T<:AbstractString}(repo::GitRepo, files::T...) @@ -95,12 +97,14 @@ function remove!{T<:AbstractString}(repo::GitRepo, files::T...) remove!(idx, files...) write!(idx) end + return end function read!(repo::GitRepo, force::Bool = false) with(GitIndex, repo) do idx read!(idx, force) end + return end function Base.count(idx::GitIndex) diff --git a/base/libgit2/merge.jl b/base/libgit2/merge.jl index 6a3ea80c700de..d779bbe41c411 100644 --- a/base/libgit2/merge.jl +++ b/base/libgit2/merge.jl @@ -20,7 +20,7 @@ function GitAnnotated(repo::GitRepo, fh::FetchHead) ann_ref_ref = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_annotated_commit_from_fetchhead, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Cstring, Ptr{Oid}), - ann_ref_ref, repo.ptr, fh.name, fh.url, fh.oid) + ann_ref_ref, repo.ptr, fh.name, fh.url, Ref(fh.oid)) return GitAnnotated(ann_ref_ref[]) end @@ -74,8 +74,9 @@ function ffmerge!(repo::GitRepo, ann::GitAnnotated) end """ Merge changes into current head """ -function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, merge_opts::MergeOptions, - checkout_opts = CheckoutOptions(checkout_strategy = Consts.CHECKOUT_SAFE)) +function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, + merge_opts::MergeOptions = MergeOptions(); + checkout_opts::CheckoutOptions = CheckoutOptions()) anns_size = Csize_t(length(anns)) @check ccall((:git_merge, :libgit2), Cint, (Ptr{Void}, Ptr{Ptr{Void}}, Csize_t, @@ -89,7 +90,9 @@ end """Internal implementation of merge. Returns `true` if merge was successful, otherwise `false` """ -function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, fastforward::Bool, options::MergeOptions) +function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, fastforward::Bool, + merge_opts::MergeOptions = MergeOptions(); + checkout_opts::CheckoutOptions = CheckoutOptions()) ma, mp = merge_analysis(repo, anns) if isset(ma, Cint(Consts.MERGE_ANALYSIS_UP_TO_DATE)) return true # no merge - everything is up to date @@ -118,7 +121,7 @@ function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, fastforward::Bool, op ffmerge!(repo, anns[1]) end elseif isset(ma, Cint(Consts.MERGE_ANALYSIS_NORMAL)) - merge!(repo, anns, options) + merge!(repo, anns, merge_opts, checkout_opts=checkout_opts) end elseif ffPref == Consts.MERGE_PREFERENCE_FASTFORWARD_ONLY if isset(ma, Cint(Consts.MERGE_ANALYSIS_FASTFORWARD)) @@ -134,7 +137,7 @@ function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, fastforward::Bool, op end elseif ffPref == Consts.MERGE_PREFERENCE_NO_FASTFORWARD if isset(ma, Cint(Consts.MERGE_ANALYSIS_NORMAL)) - merge!(repo, anns, options) + merge!(repo, anns, merge_opts, checkout_opts=checkout_opts) end end diff --git a/base/libgit2/remote.jl b/base/libgit2/remote.jl index 51f324529988c..7fff7d2fd3502 100644 --- a/base/libgit2/remote.jl +++ b/base/libgit2/remote.jl @@ -38,6 +38,28 @@ function url(rmt::GitRemote) return bytestring(url_ptr) end +function fetch_refspecs(rmt::GitRemote) + sa_ref = Ref{StrArrayStruct}() + try + @check ccall((:git_remote_get_fetch_refspecs, :libgit2), Cint, + (Ptr{LibGit2.StrArrayStruct}, Ptr{Void}), sa_ref, rmt.ptr) + convert(Vector{AbstractString}, sa_ref[]) + finally + finalize(sa_ref[]) + end +end + +function push_refspecs(rmt::GitRemote) + sa_ref = Ref{StrArrayStruct}() + try + @check ccall((:git_remote_get_push_refspecs, :libgit2), Cint, + (Ptr{LibGit2.StrArrayStruct}, Ptr{Void}), sa_ref, rmt.ptr) + convert(Vector{AbstractString}, sa_ref[]) + finally + finalize(sa_ref[]) + end +end + function fetch{T<:AbstractString}(rmt::GitRemote, refspecs::Vector{T}; options::FetchOptions = FetchOptions(), msg::AbstractString="") diff --git a/base/libgit2/tag.jl b/base/libgit2/tag.jl index b20642689cec6..efaeb5feb1e57 100644 --- a/base/libgit2/tag.jl +++ b/base/libgit2/tag.jl @@ -14,15 +14,17 @@ function tag_delete(repo::GitRepo, tag::AbstractString) (Ptr{Void}, Cstring, ), repo.ptr, tag) end -function tag_create(repo::GitRepo, tag::AbstractString, commit::AbstractString; +function tag_create(repo::GitRepo, tag::AbstractString, commit::Union{AbstractString,Oid}; msg::AbstractString = "", - force::Bool = false) + force::Bool = false, + sig::Signature = Signature(repo)) oid_ptr = Ref(Oid()) with(get(GitCommit, repo, commit)) do commit_obj - with(default_signature(repo)) do sig + commit_obj === nothing && return oid_ptr[] # return empty oid + with(convert(GitSignature, sig)) do git_sig @check ccall((:git_tag_create, :libgit2), Cint, (Ptr{Oid}, Ptr{Void}, Cstring, Ptr{Void}, Ptr{SignatureStruct}, Cstring, Cint), - oid_ptr, repo.ptr, tag, commit_obj.ptr, sig.ptr, msg, Cint(force)) + oid_ptr, repo.ptr, tag, commit_obj.ptr, git_sig.ptr, msg, Cint(force)) end end return oid_ptr[] diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index 027fb16ba0219..7fb092d5ce857 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -50,6 +50,9 @@ function Base.finalize(buf::Buffer) return buf_ptr[] end +"Abstract payload type for callback functions" +abstract AbstractPayload + immutable CheckoutOptions version::Cuint @@ -161,6 +164,15 @@ RemoteCallbacks(; sideband_progress::Ptr{Void} = C_NULL, transport, payload) +function RemoteCallbacks{P<:AbstractPayload}(credentials::Ptr{Void}, payload::Nullable{P}) + if isnull(payload) + RemoteCallbacks(credentials=credentials_cb()) + else + payload_ptr = pointer_from_objref(Base.get(payload)) + RemoteCallbacks(credentials=credentials_cb(), payload=payload_ptr) + end +end + immutable FetchOptions version::Cuint callbacks::RemoteCallbacks @@ -511,3 +523,10 @@ function getobjecttype(obj_type::Cint) throw(GitError(Error.Object, Error.ENOTFOUND, "Object type $obj_type is not supported")) end end + +type UserPasswordCredentials <: AbstractPayload + user::AbstractString + pass::AbstractString + used::Bool + UserPasswordCredentials(u::AbstractString,p::AbstractString) = new(u,p,false) +end \ No newline at end of file diff --git a/base/libgit2/utils.jl b/base/libgit2/utils.jl index bd6cfc72806b3..e5c42f5db6cad 100644 --- a/base/libgit2/utils.jl +++ b/base/libgit2/utils.jl @@ -7,4 +7,15 @@ function version() return VersionNumber(major[], minor[], patch[]) end -isset(val::Integer, flag::Integer) = (val & flag == flag) \ No newline at end of file +isset(val::Integer, flag::Integer) = (val & flag == flag) + +function prompt(msg::AbstractString; default::AbstractString="", password::Bool=false) + msg = length(default) > 0 ? msg*" [$default]:" : msg*":" + uinput = if password + bytestring(ccall(:getpass, Cstring, (Cstring,), msg)) + else + print(msg) + chomp(readline(STDIN)) + end + length(uinput) == 0 ? default : uinput +end diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index ef6d561b99d56..8e25da91524b1 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -651,7 +651,7 @@ function tag(pkg::AbstractString, ver::Union{Symbol,VersionNumber}, force::Bool= existing = VersionNumber[keys(Read.available(pkg))...] ancestors = filter(v->LibGit2.is_ancestor_of(avail[v].sha1, commit, repo), existing) else - tags = filter(t->startswith(t,"v"), Pkg.LibGit2.tag_list(repo)) + tags = filter(t->startswith(t,"v"), LibGit2.tag_list(repo)) filter!(tag->ismatch(Base.VERSION_REGEX,tag), tags) existing = VersionNumber[tags...] filter!(tags) do tag diff --git a/base/pkg/github.jl b/base/pkg/github.jl index 69a58624b21e4..1eced0e9aa99c 100644 --- a/base/pkg/github.jl +++ b/base/pkg/github.jl @@ -60,13 +60,14 @@ function delete_token() info("Could not authenticate with existing token. Deleting token and trying again.") end +readtoken(tokfile=Dir.path(".github","token")) = isfile(tokfile) ? strip(readchomp(tokfile)) : "" + function token(user::AbstractString=user()) tokfile = Dir.path(".github","token") - if isfile(tokfile) - tok = strip(readchomp(tokfile)) - !isempty(tok) && return tok - end - params = merge(AUTH_DATA, ["fingerprint" => randstring(40)]) + tok = readtoken(tokfile) + !isempty(tok) && return tok + + params = merge(AUTH_DATA, Dict("fingerprint" => randstring(40))) status, header, content = curl("https://api.github.com/authorizations",params,`-u $user`) tfa = false @@ -134,4 +135,10 @@ function normalize_url(url::AbstractString) m === nothing ? url : "https://github.com/$(m.captures[1]).git" end +function credentials() + username = try user() catch "" end + password = readtoken() + return Nullable(LibGit2.UserPasswordCredentials(username, password)) +end + end # module diff --git a/test/libgit2-online.jl b/test/libgit2-online.jl new file mode 100644 index 0000000000000..169f2cab3ec25 --- /dev/null +++ b/test/libgit2-online.jl @@ -0,0 +1,59 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + +@testset "libgit2-online" begin + +######### +# Setup # +######### + +function temp_dir(fn::Function, remove_tmp_dir::Bool=true) + tmpdir = joinpath(tempdir(),randstring()) + @test !isdir(tmpdir) + try + mkdir(tmpdir) + @test isdir(tmpdir) + fn(tmpdir) + finally + remove_tmp_dir && rm(tmpdir, recursive=true) + end +end + +######### +# TESTS # +######### +# init & clone +temp_dir() do dir + repo_url = "github.com/JuliaLang/Example.jl" + https_prefix = "https://" + ssh_prefix = "git@" + @testset "Cloning repository" begin + @testset "with 'https' protocol" begin + repo_path = joinpath(dir, "Example1") + repo = LibGit2.clone(https_prefix*repo_url, repo_path) + try + @test isdir(repo_path) + @test isdir(joinpath(repo_path, ".git")) + finally + finalize(repo) + end + end + @testset "with incorrect url" begin + repo_path = joinpath(dir, "Example2") + # credential are required because github try authenticate on uknown repo + x = Nullable(LibGit2.UserPasswordCredentials("X","X")) + @test_throws LibGit2.Error.GitError LibGit2.clone(https_prefix*repo_url*randstring(10), repo_path, payload=x) + end + + try + @testset "with 'ssh' protocol (by default is not supported)" begin + repo_path = joinpath(dir, "Example3") + @test_throws LibGit2.Error.GitError LibGit2.clone(ssh_prefix*repo_url, repo_path) + end + catch ex + # but we cloned succesfully, so check that repo was created + ex.fail == 1 && @test isdir(joinpath(path, ".git")) + end + end +end + +end diff --git a/test/libgit2.jl b/test/libgit2.jl index ca19cb1f5ff1f..7e96689df85b9 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -1,21 +1,50 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -# check that libgit2 has been installed correctly +@testset "libgit2" begin const LIBGIT2_VER = v"0.23.0" -function check_version() - v = LibGit2.version() - if v.major == LIBGIT2_VER.major && v.minor >= LIBGIT2_VER.minor - return true - else - return false +######### +# Setup # +######### + +function temp_dir(fn::Function, remove_tmp_dir::Bool=true) + tmpdir = joinpath(tempdir(),randstring()) + @test !isdir(tmpdir) + try + mkdir(tmpdir) + @test isdir(tmpdir) + fn(tmpdir) + finally + remove_tmp_dir && rm(tmpdir, recursive=true) end end -@test check_version() -# strarray -begin + +######### +# TESTS # +######### + +@testset "Check library verison" begin + v = LibGit2.version() + @test v.major == LIBGIT2_VER.major && v.minor >= LIBGIT2_VER.minor +end + +@testset "OID" begin + z = LibGit2.Oid() + r = rand(LibGit2.Oid) + @test LibGit2.iszero(z) + @test z == zero(LibGit2.Oid) + rs = string(r) + rr = LibGit2.raw(r) + @test r == LibGit2.Oid(rr) + @test r == LibGit2.Oid(rs) + @test r == LibGit2.Oid(pointer(rr)) + for i in 11:length(rr); rr[i] = 0; end + @test LibGit2.Oid(rr) == LibGit2.Oid(rs[1:20]) +end + +@testset "StrArrayStruct" begin p1 = "XXX" p2 = "YYY" sa1 = LibGit2.StrArrayStruct(p1) @@ -41,206 +70,333 @@ begin end end -function credentials!(cfg::LibGit2.GitConfig, usr="Test User", usr_email="Test@User.com") - git_user = LibGit2.get(cfg, "user.name", usr) - usr==git_user && LibGit2.set!(cfg, "user.name", usr) - git_user_email = LibGit2.get(cfg, "user.email", usr_email) - usr_email==git_user_email && LibGit2.set!(cfg, "user.email", usr_email) -end - -#TODO: tests need 'user.name' & 'user.email' in config ??? -LibGit2.with(LibGit2.GitConfig) do cfg - credentials!(cfg) -end - -function temp_dir(fn::Function, remove_tmp_dir::Bool=true) - tmpdir = joinpath(tempdir(),randstring()) - @test !isdir(tmpdir) - try - mkdir(tmpdir) - @test isdir(tmpdir) - fn(tmpdir) - finally - remove_tmp_dir && rm(tmpdir, recursive=true) - end +@testset "Signature" begin + sig = LibGit2.Signature("AAA", "AAA@BBB.COM", round(time(), 0), 0) + git_sig = convert(LibGit2.GitSignature, sig) + sig2 = LibGit2.Signature(git_sig) + finalize(git_sig) + @test sig.name == sig2.name + @test sig.email == sig2.email + @test sig.time == sig2.time end -# clone bare temp_dir() do dir + + # test parameters repo_url = "https://github.com/JuliaLang/Example.jl" - repo_path = joinpath(dir, "Example.Bare") - repo = LibGit2.clone(repo_url, repo_path, isbare = true, remote_cb = LibGit2.mirror_cb()) - finalize(repo) - @test isdir(repo_path) - @test isfile(joinpath(repo_path, LibGit2.Consts.HEAD_FILE)) -end + ssh_prefix = "git@" + cache_repo = joinpath(dir, "Example") + test_repo = joinpath(dir, "Example.Test") + test_sig = LibGit2.Signature("TEST", "TEST@TEST.COM", round(time(), 0), 0) + test_file = "testfile" + config_file = "testconfig" + commit_msg1 = randstring(10) + commit_msg2 = randstring(10) + commit_oid1 = LibGit2.Oid() + commit_oid2 = LibGit2.Oid() + test_branch = "test_branch" + tag1 = "tag1" + tag2 = "tag2" -# clone -temp_dir() do dir - url = "https://github.com/JuliaLang/Example.jl" - path = joinpath(dir, "Example") - repo = LibGit2.clone(url, path) - finalize(repo) - @test isdir(path) - @test isdir(joinpath(path, ".git")) -end + @testset "Configuration" begin + cfg = LibGit2.GitConfig(joinpath(dir, config_file), LibGit2.Consts.CONFIG_LEVEL_APP) + try + @test_throws LibGit2.Error.GitError LibGit2.get(AbstractString, cfg, "tmp.str") + @test isempty(LibGit2.get(cfg, "tmp.str", "")) == true -# init -temp_dir() do dir - path = joinpath(dir, "Example") - repo = LibGit2.init(path) + LibGit2.set!(cfg, "tmp.str", "AAAA") + LibGit2.set!(cfg, "tmp.int32", Int32(1)) + LibGit2.set!(cfg, "tmp.int64", Int64(1)) + LibGit2.set!(cfg, "tmp.bool", true) - @test isdir(path) - @test isdir(joinpath(path, ".git")) + @test LibGit2.get(cfg, "tmp.str", "") == "AAAA" + @test LibGit2.get(cfg, "tmp.int32", Int32(0)) == Int32(1) + @test LibGit2.get(cfg, "tmp.int64", Int64(0)) == Int64(1) + @test LibGit2.get(cfg, "tmp.bool", false) == true + finally + finalize(cfg) + end + end - branch = "upstream" - url = "https://github.com/JuliaLang/julia.git" - remote = LibGit2.GitRemote(repo, branch, url) - finalize(remote) + @testset "Initializing repository" begin + @testset "with remote branch" begin + repo = LibGit2.init(cache_repo) + try + @test isdir(cache_repo) + @test isdir(joinpath(cache_repo, ".git")) - config = joinpath(path, ".git", "config") - lines = split(open(readall, config, "r"), "\n") - @test any(map(x->x == "[remote \"upstream\"]", lines)) + # set a remote branch + branch = "upstream" + LibGit2.GitRemote(repo, branch, repo_url) |> finalize - remote = LibGit2.get(LibGit2.GitRemote, repo, branch) - @test LibGit2.url(remote) == url + config = joinpath(cache_repo, ".git", "config") + lines = split(open(readall, config, "r"), "\n") + @test any(map(x->x == "[remote \"upstream\"]", lines)) - finalize(repo) -end + remote = LibGit2.get(LibGit2.GitRemote, repo, branch) + @test LibGit2.url(remote) == repo_url + finalize(remote) + finally + finalize(repo) + end + end -# fetch -temp_dir() do dir_cache - # create cache - url = "https://github.com/JuliaLang/Example.jl" - path_cache = joinpath(dir_cache, "Example.Bare") - repo = LibGit2.clone(url, path_cache, isbare = true, remote_cb = LibGit2.mirror_cb()) - LibGit2.with(LibGit2.GitConfig, repo) do cfg - credentials!(cfg) + @testset "bare" begin + path = joinpath(dir, "Example.Bare") + repo = LibGit2.init(path, Cuint(1)) + try + @test isdir(path) + @test isfile(joinpath(path, LibGit2.Consts.HEAD_FILE)) + finally + finalize(repo) + end + end end - finalize(repo) - - # fetch - temp_dir() do dir - # clone repo - path = joinpath(dir, "Example") - repo = LibGit2.clone(path_cache, path) - LibGit2.with(LibGit2.GitConfig, repo) do cfg - credentials!(cfg) + + @testset "Cloning repository" begin + + @testset "bare" begin + repo_path = joinpath(dir, "Example.Bare1") + repo = LibGit2.clone(cache_repo, repo_path, isbare = true) + try + @test isdir(repo_path) + @test isfile(joinpath(repo_path, LibGit2.Consts.HEAD_FILE)) + finally + finalize(repo) + end + end + @testset "bare with remote callback" begin + repo_path = joinpath(dir, "Example.Bare2") + repo = LibGit2.clone(cache_repo, repo_path, isbare = true, remote_cb = LibGit2.mirror_cb()) + try + @test isdir(repo_path) + @test isfile(joinpath(repo_path, LibGit2.Consts.HEAD_FILE)) + rmt = LibGit2.get(LibGit2.GitRemote, repo, "origin") + try + @test LibGit2.fetch_refspecs(rmt)[1] == "+refs/*:refs/*" + finally + finalize(rmt) + end + finally + finalize(repo) + end end + @testset "normal" begin + repo = LibGit2.clone(cache_repo, test_repo, remote_cb = LibGit2.mirror_cb()) + try + @test isdir(test_repo) + @test isdir(joinpath(test_repo, ".git")) + finally + finalize(repo) + end + end + end - LibGit2.fetch(repo) - refs1 = parse(Int, readchomp(pipeline(`find $(joinpath(path, ".git/refs"))`,`wc -l`))) + @testset "Update cache repository" begin - LibGit2.fetch(repo, remoteurl=path_cache, refspecs =["+refs/*:refs/remotes/cache/*"]) - refs2 = parse(Int, readchomp(pipeline(`find $(joinpath(path, ".git/refs"))`,`wc -l`))) + @testset "with commits" begin + repo = LibGit2.GitRepo(cache_repo) + repo_file = open(joinpath(cache_repo,test_file), "a") + try + # create commits + println(repo_file, commit_msg1) + flush(repo_file) + LibGit2.add!(repo, test_file) + @test LibGit2.iszero(commit_oid1) + commit_oid1 = LibGit2.commit(repo, commit_msg1; author=test_sig, committer=test_sig) + @test !LibGit2.iszero(commit_oid1) - finalize(repo) - @test refs1 > 0 - @test refs2 > 0 - @test refs2 > refs1 - end + println(repo_file, randstring(10)) + flush(repo_file) + LibGit2.add!(repo, test_file) + LibGit2.commit(repo, randstring(10); author=test_sig, committer=test_sig) - # signature - temp_dir() do dir - path = joinpath(dir, "Example") - LibGit2.with(LibGit2.clone(path_cache, path)) do repo - oid = "129eb39c8e0817c616296d1ac5f2cd1cf4f8b312" - c = LibGit2.get(LibGit2.GitCommit, repo, oid) - auth = LibGit2.author(c) - cmtr = LibGit2.committer(c) - cmsg = LibGit2.message(c) - @test isa(auth, LibGit2.Signature) - @test length(auth.name) > 0 - @test isa(cmtr, LibGit2.Signature) - @test length(cmtr.email) > 0 - @test length(cmsg) > 0 - - sig = LibGit2.Signature("AAA", "AAA@BBB.COM", round(time(), 0), 0) - git_sig = convert(LibGit2.GitSignature, sig) - sig2 = LibGit2.Signature(git_sig) - finalize(git_sig) - @test sig.name == sig2.name - @test sig.email == sig2.email - @test sig.time == sig2.time + println(repo_file, commit_msg2) + flush(repo_file) + LibGit2.add!(repo, test_file) + @test LibGit2.iszero(commit_oid2) + commit_oid2 = LibGit2.commit(repo, commit_msg2; author=test_sig, committer=test_sig) + @test !LibGit2.iszero(commit_oid2) + + # lookup commits + cmt = LibGit2.get(LibGit2.GitCommit, repo, commit_oid1) + try + @test commit_oid1 == LibGit2.Oid(cmt) + auth = LibGit2.author(cmt) + @test isa(auth, LibGit2.Signature) + @test auth.name == test_sig.name + @test auth.time == test_sig.time + @test auth.email == test_sig.email + cmtr = LibGit2.committer(cmt) + @test isa(cmtr, LibGit2.Signature) + @test cmtr.name == test_sig.name + @test cmtr.time == test_sig.time + @test cmtr.email == test_sig.email + @test LibGit2.message(cmt) == commit_msg1 + finally + finalize(cmt) + end + finally + finalize(repo) + close(repo_file) + end end - end - # revwalk - temp_dir() do dir - path = joinpath(dir, "Example") - repo = LibGit2.clone(path_cache, path) - LibGit2.with(LibGit2.GitConfig, repo) do cfg - credentials!(cfg) + @testset "with branch" begin + repo = LibGit2.GitRepo(cache_repo) + try + brnch = LibGit2.branch(repo) + @test brnch == "master" + + LibGit2.branch!(repo, test_branch, string(commit_oid1), set_head=false) + + branches = map(b->LibGit2.shortname(b[1]), LibGit2.GitBranchIter(repo)) + @test "master" in branches + @test test_branch in branches + finally + finalize(repo) + end end - oids = LibGit2.with(LibGit2.GitRevWalker(repo)) do walker - LibGit2.map((oid,repo)->string(oid), walker, by = LibGit2.Consts.SORT_TIME) + + @testset "with default configuration" begin + repo = LibGit2.GitRepo(cache_repo) + try + try + LibGit2.Signature(repo) + catch ex + # these test configure repo with new signature + # in case when global one does not exsist + @test isa(ex, LibGit2.Error.GitError) == true + + cfg = LibGit2.GitConfig(repo) + LibGit2.set!(cfg, "user.name", "AAAA") + LibGit2.set!(cfg, "user.email", "BBBB@BBBB.COM") + sig = LibGit2.Signature(repo) + @test sig.name == "AAAA" + @test sig.email == "BBBB@BBBB.COM" + end + finally + finalize(repo) + end end - @test length(oids) > 0 - finalize(repo) - LibGit2.with(LibGit2.GitRepo, path) do repo - oid = LibGit2.Oid(oids[end]) - oids = LibGit2.with(LibGit2.GitRevWalker(repo)) do walker - LibGit2.map((oid,repo)->(oid,repo), walker, oid=oid, by=LibGit2.Consts.SORT_TIME) + + @testset "with tags" begin + repo = LibGit2.GitRepo(cache_repo) + try + tags = LibGit2.tag_list(repo) + @test length(tags) == 0 + + tag_oid1 = LibGit2.tag_create(repo, tag1, commit_oid1, sig=test_sig) + @test !LibGit2.iszero(tag_oid1) + tags = LibGit2.tag_list(repo) + @test length(tags) == 1 + @test tag1 in tags + + tag_oid2 = LibGit2.tag_create(repo, tag2, commit_oid2) + @test !LibGit2.iszero(tag_oid2) + tags = LibGit2.tag_list(repo) + @test length(tags) == 2 + @test tag2 in tags + + LibGit2.tag_delete(repo, tag1) + tags = LibGit2.tag_list(repo) + @test length(tags) == 1 + @test tag2 ∈ tags + @test tag1 ∉ tags + finally + finalize(repo) end - @test length(oids) > 0 end end - # transact - temp_dir() do dir - # clone repo - path = joinpath(dir, "Example") - LibGit2.with(LibGit2.clone(path_cache, path)) do repo - LibGit2.with(LibGit2.GitConfig, repo) do cfg - credentials!(cfg) + @testset "Fetch from cache repository" begin + repo = LibGit2.GitRepo(test_repo) + try + # fetch changes + @test LibGit2.fetch(repo) == 0 + @test !isfile(joinpath(test_repo, test_file)) + + # ff merge them + @test LibGit2.merge!(repo, fastforward=true) + + # because there was not any file we need to reset branch + head_oid = LibGit2.head_oid(repo) + LibGit2.reset!(repo, head_oid, LibGit2.Consts.RESET_HARD) + @test isfile(joinpath(test_repo, test_file)) + finally + finalize(repo) + end + end + + @testset "Examine test repository" begin + @testset "files" begin + @test readall(joinpath(test_repo, test_file)) == readall(joinpath(cache_repo, test_file)) + end + + @testset "tags & branches" begin + repo = LibGit2.GitRepo(test_repo) + try + # all tag in place + tags = LibGit2.tag_list(repo) + @test length(tags) == 1 + @test tag2 in tags + + # all tag in place + branches = map(b->LibGit2.shortname(b[1]), LibGit2.GitBranchIter(repo)) + @test "master" in branches + @test test_branch in branches + finally + finalize(repo) end - cp(joinpath(path, "REQUIRE"), joinpath(path, "CCC")) - cp(joinpath(path, "REQUIRE"), joinpath(path, "AAA")) - LibGit2.add!(repo, "AAA") - @test_throws ErrorException LibGit2.transact(repo) do repo - mv(joinpath(path, "REQUIRE"), joinpath(path, "BBB")) - LibGit2.add!(repo, "BBB") - oid = LibGit2.commit(repo, "test commit") - error("Force recovery") + end + + @testset "commits with revwalk" begin + repo = LibGit2.GitRepo(test_repo) + cache = LibGit2.GitRepo(cache_repo) + try + oids = LibGit2.with(LibGit2.GitRevWalker(repo)) do walker + LibGit2.map((oid,repo)->(oid,repo), walker, oid=commit_oid1, by=LibGit2.Consts.SORT_TIME) + end + @test length(oids) == 1 + + test_oids = LibGit2.with(LibGit2.GitRevWalker(repo)) do walker + LibGit2.map((oid,repo)->string(oid), walker, by = LibGit2.Consts.SORT_TIME) + end + cache_oids = LibGit2.with(LibGit2.GitRevWalker(cache)) do walker + LibGit2.map((oid,repo)->string(oid), walker, by = LibGit2.Consts.SORT_TIME) + end + @testloop for i in eachindex(oids) + @test cache_oids[i] == test_oids[i] + end + finally + finalize(repo) + finalize(cache) end - @test isfile(joinpath(path, "AAA")) - @test isfile(joinpath(path, "CCC")) - @test !isfile(joinpath(path, "BBB")) - @test isfile(joinpath(path, "REQUIRE")) end end - # branch - temp_dir() do dir - path = joinpath(dir, "Example") - LibGit2.with(LibGit2.clone(path_cache, path)) do repo - LibGit2.with(LibGit2.GitConfig, repo) do cfg - credentials!(cfg) + @testset "Transact test repository" begin + repo = LibGit2.GitRepo(test_repo) + try + cp(joinpath(test_repo, test_file), joinpath(test_repo, "CCC")) + cp(joinpath(test_repo, test_file), joinpath(test_repo, "AAA")) + LibGit2.add!(repo, "AAA") + @test_throws ErrorException LibGit2.transact(repo) do trepo + mv(joinpath(test_repo, test_file), joinpath(test_repo, "BBB")) + LibGit2.add!(trepo, "BBB") + oid = LibGit2.commit(trepo, "test commit"; author=test_sig, committer=test_sig) + error("Force recovery") end - brnch = LibGit2.branch(repo) - @test brnch == "master" - - brnch_name = "test" - LibGit2.branch!(repo, brnch_name) - brnch = LibGit2.branch(repo) - @test brnch == brnch_name + @test isfile(joinpath(test_repo, "AAA")) + @test isfile(joinpath(test_repo, "CCC")) + @test !isfile(joinpath(test_repo, "BBB")) + @test isfile(joinpath(test_repo, test_file)) + finally + finalize(repo) end end -end -# Oid -begin - z = LibGit2.Oid() - r = rand(LibGit2.Oid) - @test LibGit2.iszero(z) - @test z == zero(LibGit2.Oid) - rs = string(r) - rr = LibGit2.raw(r) - @test r == LibGit2.Oid(rr) - @test r == LibGit2.Oid(rs) - @test r == LibGit2.Oid(pointer(rr)) - for i in 11:length(rr); rr[i] = 0; end - @test LibGit2.Oid(rr) == LibGit2.Oid(rs[1:20]) end +end diff --git a/test/pkg.jl b/test/pkg.jl index a40fc31dd315f..f0c56efc82d41 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -232,7 +232,7 @@ end # issue #13374 temp_pkg_dir() do - Pkg.generate("Foo", "MIT") + Pkg.generate("Foo", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) Pkg.tag("Foo") Pkg.tag("Foo") end From 9c1f69c2c23ce3642ab529d9574c9cf0af45016b Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 5 Oct 2015 09:56:54 -0500 Subject: [PATCH 0438/1938] Prefer the parent module when serializing functions (fixes #13452) --- base/serialize.jl | 2 ++ test/serialize.jl | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/base/serialize.jl b/base/serialize.jl index 5681286d1d6fb..8c2009f1c6a71 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -281,6 +281,8 @@ function serialize(s::SerializationState, f::Function) mod = () if isa(f.env,Symbol) mod = Core + elseif isdefined(f.env, :module) && isa(f.env.module, Module) + mod = f.env.module elseif !is(f.env.defs, ()) mod = f.env.defs.func.code.module end diff --git a/test/serialize.jl b/test/serialize.jl index 1b63b7fe25734..b24e08fe2ffa9 100644 --- a/test/serialize.jl +++ b/test/serialize.jl @@ -319,3 +319,26 @@ create_serialization_stream() do s r2 = deserialize(s) @test r1 == r2 end + +# issue #13452 +module Test13452 +using Base.Test + +module Shell +export foo +foo(x) = error("Instances must implement foo") +end + +module Instance1 +using ..Shell +Shell.foo(x::Int) = "This is an Int" +end + +using .Shell, .Instance1 +io = IOBuffer() +serialize(io, foo) +str = takebuf_string(io) +@test isempty(search(str, "Instance1")) +@test !isempty(search(str, "Shell")) + +end # module Test13452 From 06b52dd12c80ec458a66024f902e94365acc4942 Mon Sep 17 00:00:00 2001 From: Matthieu Gomez Date: Mon, 5 Oct 2015 10:17:45 -0400 Subject: [PATCH 0439/1938] Add dispatch on parametric types --- doc/manual/types.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/manual/types.rst b/doc/manual/types.rst index 5c0945c48d736..09572088b8616 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -711,6 +711,21 @@ pointers to individually allocated :obj:`Real` objects — which may well be complex objects, which are declared to be implementations of the :obj:`Real` abstract type. +Since ``Point{Float64}`` is not a subtype of ``Point{Real}``, the following method can't be applied to arguments of type ``Point{Float64}``:: + + function norm(p::Point{Real}) + sqrt(p.x^2 + p.y^2) + end + +The correct way to define a method that accepts all arguments of type ``Point{T}`` where ``T`` is a subtype of ``Real`` is:: + + function norm{T<:Real}(p::Point{T}) + sqrt(p.x^2 + p.y^2) + end + +More examples will be discussed later in :ref:`man-methods`. + + How does one construct a ``Point`` object? It is possible to define custom constructors for composite types, which will be discussed in detail in :ref:`man-constructors`, but in the absence of any From e6a8b1efab1392ef71333bfd24d5fe3f50d4eafd Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 5 Oct 2015 14:13:19 -0400 Subject: [PATCH 0440/1938] work around a gcc register allocation error see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67814 --- src/alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index 5b4dd0d550ee7..3106ffa6ecc73 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -581,7 +581,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) } if (al != 0) { size_t alsz = LLT_ALIGN(sz, al); - if (alsz > sz) + if (sz & (al - 1)) st->haspadding = 1; sz = alsz; if (al > alignm) From db59e7a55df85512cf030ca2b2384e015fe3c7b4 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 5 Oct 2015 15:17:06 -0400 Subject: [PATCH 0441/1938] fix minor makefile grievances --- Makefile | 2 +- deps/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index aeb7b237632aa..c91cc7f80929f 100644 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ julia-sysimg-debug : julia-inference julia-ui-debug JULIA_ENABLE_DOCBUILD ?= 1 julia-genstdlib : julia-sysimg-$(JULIA_BUILD_MODE) ifeq ($(JULIA_ENABLE_DOCBUILD), 1) - @$(call PRINT_JULIA, $(JULIA_EXECUTABLE) doc/genstdlib.jl) + @$(call PRINT_JULIA, $(JULIA_EXECUTABLE) $(JULIAHOME)/doc/genstdlib.jl) endif julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest julia-genstdlib diff --git a/deps/Makefile b/deps/Makefile index 319de7803d3de..82bc7e65ab9a8 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -1906,7 +1906,7 @@ $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER)/configure: $(SRCDIR)/srccache/patche cd $(dir $<) && $(TAR) zxf $< touch -c $@ $(BUILDDIR)/patchelf-$(PATCHELF_VER)/config.status: $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER)/configure - mkdir $(dir $@) + mkdir -p $(dir $@) cd $(dir $@) && \ $< $(CONFIGURE_COMMON) touch -c $@ From 592620f39eaf53508509a209f6e79a8560501d86 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 4 Oct 2015 22:26:45 -0400 Subject: [PATCH 0442/1938] fix regression in --compile=all function caused by improved ccall error message --- src/ccall.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index b66b236ac8dbd..819818f1bb9f9 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1006,15 +1006,16 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) static_rt = true; } } - if (jl_exception_in_transit && jl_typeis(jl_exception_in_transit, - jl_undefvarerror_type)) { - std::string msg = "ccall return type undefined: " + - std::string(((jl_sym_t*)args[2])->name ); - emit_error(msg.c_str(), ctx); - JL_GC_POP(); - return jl_cgval_t(); - } if (rt == NULL) { + if (jl_exception_in_transit && jl_typeis(jl_exception_in_transit, + jl_undefvarerror_type) + && jl_is_symbol(args[2])) { + std::string msg = "ccall return type undefined: " + + std::string(((jl_sym_t*)args[2])->name); + emit_error(msg.c_str(), ctx); + JL_GC_POP(); + return jl_cgval_t(); + } emit_error("error interpreting ccall return type", ctx); JL_GC_POP(); return jl_cgval_t(); From fa6c97706a5a799f4ec9ea71bd4aec9d708c76e9 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 5 Oct 2015 17:49:50 -0500 Subject: [PATCH 0443/1938] Use eachindex in generic_scale! --- base/linalg/generic.jl | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/base/linalg/generic.jl b/base/linalg/generic.jl index bf2bcd1917cde..c8948f30c3eff 100644 --- a/base/linalg/generic.jl +++ b/base/linalg/generic.jl @@ -8,8 +8,8 @@ scale(s::Number, X::AbstractArray) = s*X # For better performance when input and output are the same array # See https://github.com/JuliaLang/julia/issues/8415#issuecomment-56608729 function generic_scale!(X::AbstractArray, s::Number) - for i = 1:length(X) - @inbounds X[i] *= s + for I in eachindex(X) + @inbounds X[I] *= s end X end @@ -18,8 +18,14 @@ function generic_scale!(C::AbstractArray, X::AbstractArray, s::Number) if length(C) != length(X) throw(DimensionMismatch("first array has length $(length(C)) which does not match the length of the second, $(length(X)).")) end - for i = 1:length(X) - @inbounds C[i] = X[i]*s + if size(C) == size(X) + for I in eachindex(C, X) + @inbounds C[I] = X[I]*s + end + else + for (IC, IX) in zip(eachindex(C), eachindex(X)) + @inbounds C[IC] = X[IX]*s + end end C end From 987d42fe6978f9c9fc9df0ac8584c2cd0926e7e9 Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Mon, 5 Oct 2015 21:39:22 -0400 Subject: [PATCH 0444/1938] Show file:line with binding deprecation warning --- src/module.c | 7 +++++++ test/cmdlineargs.jl | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 3520de1015d9f..44bff90e5a3c2 100644 --- a/src/module.c +++ b/src/module.c @@ -479,6 +479,9 @@ DLLEXPORT int jl_is_binding_deprecated(jl_module_t *m, jl_sym_t *var) return b && b->deprecated; } +extern const char *jl_filename; +extern int jl_lineno; + void jl_binding_deprecation_warning(jl_binding_t *b) { if (b->deprecated && jl_options.depwarn) { @@ -503,6 +506,10 @@ void jl_binding_deprecation_warning(jl_binding_t *b) jl_printf(JL_STDERR, " instead"); } jl_printf(JL_STDERR, ".\n"); + + if (jl_options.depwarn != JL_OPTIONS_DEPWARN_ERROR) + jl_printf(JL_STDERR, " likely near %s:%d\n", jl_filename, jl_lineno); + if (jl_options.depwarn == JL_OPTIONS_DEPWARN_ERROR) { if (b->owner) jl_errorf("deprecated binding: %s.%s", b->owner->name->name, b->name->name); diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 2ed49d782c316..9a8e9396aed1a 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -173,7 +173,7 @@ let exename = `$(joinpath(JULIA_HOME, Base.julia_exename())) --precompiled=yes` wait(proc) close(out.in) @test success(proc) - @test readchomp(out) == "WARNING: Foo.Deprecated is deprecated." + @test readchomp(out) == "WARNING: Foo.Deprecated is deprecated.\n likely near no file:5" end @unix_only let out = Pipe(), From fd135674ba2bbca5ccbc214033a9ba577c38b0b2 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Mon, 5 Oct 2015 23:16:42 -0400 Subject: [PATCH 0445/1938] Remove an unnecessary semicolon to suppress a GCC 5 warning --- src/intrinsics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 552a1dd0cfdd2..a1bcb335872f0 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -45,7 +45,7 @@ namespace JL_I { // c interface ccall, cglobal, llvmcall }; -}; +} using namespace JL_I; From b53ae9d7b7357745912f577940563a33ca73d041 Mon Sep 17 00:00:00 2001 From: Michael Hatherly Date: Tue, 6 Oct 2015 08:53:04 +0200 Subject: [PATCH 0446/1938] Replace `Docs.isexpr` with `Meta.isexpr` This fixes the bounds error reported in #13467 by removing the custom `isexpr` function that was defined in the `Docs` module. Instead use the standard `Meta.isexpr` function. Fixes #13467. --- base/docs/Docs.jl | 27 ++++++++++++--------------- base/docs/bindings.jl | 2 +- test/docs.jl | 3 +++ 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index 9988ed016d7ff..acad83bc90515 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -57,7 +57,7 @@ function. include("bindings.jl") import Base.Markdown: @doc_str, MD -import Base.Meta: quot +import Base.Meta: quot, isexpr export doc @@ -247,7 +247,9 @@ catdoc(xs...) = vcat(xs...) # Type Documentation -isdoc(x) = isexpr(x, :string, AbstractString) || +isdoc(s::AbstractString) = true + +isdoc(x) = isexpr(x, :string) || (isexpr(x, :macrocall) && x.args[1] == symbol("@doc_str")) || (isexpr(x, :call) && x.args[1] == Expr(:., Base.Markdown, QuoteNode(:doc_str))) @@ -259,7 +261,7 @@ function field_meta(def) for l in def.args[3].args if isdoc(l) doc = mdify(l) - elseif doc !== nothing && isexpr(l, Symbol, :(::)) + elseif doc !== nothing && (isa(l, Symbol) || isexpr(l, :(::))) meta[namify(l)] = doc doc = nothing end @@ -358,8 +360,8 @@ function typesummary(T::DataType) end isfield(x) = isexpr(x, :.) && - (isexpr(x.args[1], Symbol) || isfield(x.args[1])) && - isexpr(x.args[2], QuoteNode, :quote) + (isa(x.args[1], Symbol) || isfield(x.args[1])) && + (isa(x.args[2], QuoteNode) || isexpr(x.args[2], :quote)) function fielddoc(T, k) for mod in modules @@ -390,11 +392,6 @@ const keywords = Dict{Symbol,Any}() # Usage macros -isexpr(x::Expr) = true -isexpr(x) = false -isexpr(x::Expr, ts...) = x.head in ts -isexpr(x, ts...) = any(T->isa(T, Type) && isa(x, T), ts) - function unblock(ex) isexpr(ex, :block) || return ex exs = filter(ex -> !(isa(ex, LineNumberNode) || isexpr(ex, :line)), ex.args) @@ -409,7 +406,7 @@ namify(ex::QuoteNode) = ex.value namify(sy::Symbol) = sy function mdify(ex) - if isexpr(ex, AbstractString, :string) + if isa(ex, AbstractString) || isexpr(ex, :string) :(Markdown.doc_str($(esc(ex)), @__FILE__, current_module())) else esc(ex) @@ -496,7 +493,7 @@ more than one expression is marked then the same docstring is applied to each ex :(Base.@__doc__) function __doc__!(meta, def::Expr) - if isexpr(def, :block) && length(def.args) == 2 && def.args[1] == symbol("#doc#") + if isexpr(def, :block, 2) && def.args[1] == symbol("#doc#") # Convert `Expr(:block, :#doc#, ...)` created by `@__doc__` to an `@doc`. def.head = :macrocall def.args = [symbol("@doc"), meta, def.args[end]] @@ -511,7 +508,7 @@ function __doc__!(meta, def::Expr) end __doc__!(meta, def) = false -fexpr(ex) = isexpr(ex, :function, :stagedfunction, :(=)) && isexpr(ex.args[1], :call) +fexpr(ex) = isexpr(ex, [:function, :stagedfunction, :(=)]) && isexpr(ex.args[1], :call) function docm(meta, def, define = true) @@ -539,8 +536,8 @@ function docm(meta, def, define = true) isexpr(def′, :bitstype) ? namedoc(meta, def, namify(def′.args[2])) : isexpr(def′, :typealias) ? vardoc(meta, def, namify(def′)) : isexpr(def′, :module) ? moddoc(meta, def, def′.args[2]) : - isexpr(def′, :(=), :const, - :global) ? vardoc(meta, def, namify(def′)) : + isexpr(def′, [:(=), :const, + :global]) ? vardoc(meta, def, namify(def′)) : isvar(def′) ? objdoc(meta, def′) : isexpr(def′, :tuple) ? multidoc(meta, def′.args) : __doc__!(meta, def′) ? esc(def′) : diff --git a/base/docs/bindings.jl b/base/docs/bindings.jl index bc799b7ff0eba..1d5decbfa0550 100644 --- a/base/docs/bindings.jl +++ b/base/docs/bindings.jl @@ -16,7 +16,7 @@ end splitexpr(s::Symbol) = :(current_module()), quot(s) splitexpr(other) = error("Invalid @var syntax `$other`.") -isvar(x) = isexpr(x, :macrocall, :.) +isvar(x) = isexpr(x, [:macrocall, :.]) isvar(::Symbol) = true macro var(x) diff --git a/test/docs.jl b/test/docs.jl index 5bd3c47073d8d..5abb6b059b5c3 100644 --- a/test/docs.jl +++ b/test/docs.jl @@ -411,6 +411,9 @@ f12593_2() = 1 # test that macro documentation works @test (Docs.@repl @assert) !== nothing +# Issue #13467. +@test (Docs.@repl @r_str) !== nothing + # Simple tests for apropos: @test contains(sprint(apropos, "pearson"), "cov") @test contains(sprint(apropos, r"ind(exes|ices)"), "eachindex") From bb247cfef7e4fd920c04398262684790d9de17da Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Mon, 5 Oct 2015 18:50:39 -0400 Subject: [PATCH 0447/1938] Avoid introducing local variable (and GC frame store) when the inliner is confused about the return point. --- base/array.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/base/array.jl b/base/array.jl index a1a17d57e24d2..f67a2185fbb06 100644 --- a/base/array.jl +++ b/base/array.jl @@ -303,7 +303,10 @@ end setindex!{T}(A::Array{T}, x, i1::Real) = arrayset(A, convert(T,x)::T, to_index(i1)) setindex!{T}(A::Array{T}, x, i1::Real, i2::Real, I::Real...) = arrayset(A, convert(T,x)::T, to_index(i1), to_index(i2), to_indexes(I...)...) -unsafe_setindex!{T}(A::Array{T}, x, i1::Real, I::Real...) = @inbounds return arrayset(A, convert(T,x)::T, to_index(i1), to_indexes(I...)...) +# Type inference is confused by `@inbounds return ...` and introduces a +# !ispointerfree local variable and a GC frame +unsafe_setindex!{T}(A::Array{T}, x, i1::Real, I::Real...) = + (@inbounds arrayset(A, convert(T,x)::T, to_index(i1), to_indexes(I...)...); A) # These are redundant with the abstract fallbacks but needed for bootstrap function setindex!(A::Array, x, I::AbstractVector{Int}) From efc02ce43ac6989135125adad55e680fb96aef68 Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Fri, 2 Oct 2015 18:02:14 -0400 Subject: [PATCH 0448/1938] now finish(::DefaultTestSet) returns the testset and @testloop returns an array of finish return values --- base/test.jl | 56 +++++++++++++++++++++++++++++----------------------- test/test.jl | 19 ++++++++++++++++-- 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/base/test.jl b/base/test.jl index 22fb41d99da9f..e085ac55e5234 100644 --- a/base/test.jl +++ b/base/test.jl @@ -275,7 +275,7 @@ function record(ts::FallbackTestSet, t::Union{Fail,Error}) t end # We don't need to do anything as we don't record anything -finish(ts::FallbackTestSet) = nothing +finish(ts::FallbackTestSet) = ts #----------------------------------------------------------------------- @@ -367,6 +367,9 @@ function finish(ts::DefaultTestSet) if total != total_pass throw(TestSetException(total_pass,total_fail,total_error)) end + + # return the testset so it is returned from the @testset macro + ts end # Recursive function that finds the column that the result counts @@ -516,36 +519,39 @@ string accepts interpolation from the loop indices. If no description is provided, one is constructed based on the variables. """ macro testloop(args...) + length(args) > 0 || error("no arguments to @testloop") + testloop = args[end] + isa(testloop,Expr) && testloop.head == :for || error("Unexpected argument to @testloop") + # pull out the loop variables. We might need them for generating the + # description and we'll definitely need them for generating the + # comprehension expression at the end + loopvars = Expr[] + if testloop.args[1].head == :(=) + push!(loopvars, testloop.args[1]) + elseif testloop.args[1].head == :block + for loopvar in testloop.args[1].args + push!(loopvars, loopvar) + end + else + error("Unexpected argument to @testloop") + end + # Parse arguments to do determine if any options passed in if length(args) == 2 # Looks like description format - desc, testloop = args + desc = args[1] isa(desc,AbstractString) || (isa(desc,Expr) && desc.head == :string) || error("Unexpected argument to @testloop") - isa(testloop,Expr) && testloop.head == :for || error("Unexpected argument to @testloop") - elseif length(args) == 1 - # No description provided - testloop = args[1] - isa(testloop,Expr) && testloop.head == :for || error("Unexpected argument to @testloop") - loopvars = testloop.args[1] - if loopvars.head == :(=) - # 1 variable - v = loopvars.args[1] - desc = Expr(:string,"$v = ",v) - else - # multiple variables - v = loopvars.args[1].args[1] - desc = Expr(:string,"$v = ",v) # first variable - for l = loopvars.args[2:end] - v = l.args[1] - push!(desc.args,", $v = ") - push!(desc.args,v) - end + # No description provided. Generate from the loop variable names + v = loopvars[1].args[1] + desc = Expr(:string,"$v = ",v) # first variable + for l = loopvars[2:end] + v = l.args[1] + push!(desc.args,", $v = ") + push!(desc.args,v) end - elseif length(args) >= 3 - error("Too many arguments to @testloop") else - error("Too few arguments to @testloop") + error("Too many arguments to @testloop") end # Uses a similar block as for `@testset`, except that it is @@ -565,7 +571,7 @@ macro testloop(args...) pop_testset() finish($ts) end - Expr(:for,esc(testloop.args[1]),blk) + Expr(:comprehension, blk, [esc(v) for v in loopvars]...) end #----------------------------------------------------------------------- diff --git a/test/test.jl b/test/test.jl index 67822f048ef7f..bb4920f7de2cd 100644 --- a/test/test.jl +++ b/test/test.jl @@ -83,7 +83,7 @@ try end # should add 3 errors and 3 passing tests @testloop for i in 1:6 - i % 2 == 0 || error("error outside of test") + iseven(i) || error("error outside of test") @test true # only gets run if the above passed end end @@ -92,7 +92,6 @@ end redirect_stdout(OLD_STDOUT) error("No exception was thrown!") catch ex - redirect_stdout(OLD_STDOUT) @test isa(ex, Test.TestSetException) @test ex.pass == 24 @@ -110,3 +109,19 @@ end # Test @test_approx_eq_eps # TODO + +ts = @testset "@testset should return the testset" begin + @test true +end +@test typeof(ts) == Base.Test.DefaultTestSet +@test typeof(ts.results[1]) == Base.Test.Pass + +tss = @testloop "@testloop should return an array of testsets: $i" for i in 1:3 + @test true +end +@test length(tss) == 3 +@test typeof(tss[1]) == Base.Test.DefaultTestSet +@test typeof(tss[1].results[1]) == Base.Test.Pass + + +redirect_stdout(OLD_STDOUT) From e4c5faf404e78c1472b335d4db4609cd5074b079 Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Fri, 2 Oct 2015 19:26:35 -0400 Subject: [PATCH 0449/1938] WIP trying to get nested custom testset inheritance to work --- base/test.jl | 62 ++++++++++++++++++++++++++++++-------------------- test/test.jl | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 26 deletions(-) diff --git a/base/test.jl b/base/test.jl index e085ac55e5234..3f0fd54ab9194 100644 --- a/base/test.jl +++ b/base/test.jl @@ -476,27 +476,38 @@ are any `Fail`s or `Error`s, an exception will be thrown only at the end, along with a summary of the test results. """ macro testset(args...) - # Parse arguments to do determine if any options passed in - if length(args) == 2 - # Looks like description format - desc, tests = args - !isa(desc, AbstractString) && error("Unexpected argument to @testset") - elseif length(args) == 1 - # No description provided - desc, tests = "test set", args[1] - elseif length(args) >= 3 - error("Too many arguments to @testset") - else - error("Too few arguments to @testset") + length(args) > 0 || error("Too few arguments to @testset") + + options = Dict{Symbol, Any}() + desc = "test set" + tests = args[end] + # if we're at the top level we'll default to DefaultTestSet. Otherwise + # default to the type of the parent testset + testsettype = :(get_testset_depth() == 0 ? DefaultTestSet : typeof(get_testset())) + + for arg in args[1:end-1] + # a standalone symbol is assumed to be the test set we should use + if isa(arg, Symbol) + testsettype = arg + # a string is the description + elseif isa(arg, String) + desc = arg + # an assignment is an option + elseif isa(arg, Expr) && arg.head == :(=) + options[arg.args[1]] = eval(arg.args[2]) + else + error("Unexpected argument $arg to @testset") + end end + # Generate a block of code that initializes a new testset, adds # it to the task local storage, evaluates the test(s), before # finally removing the testset and giving it a change to take # action (such as reporting the results) ts = gensym() quote - $ts = DefaultTestSet($desc) - add_testset($ts) + $ts = $(esc(testsettype))($desc; $options...) + push_testset($ts) try $(esc(tests)) catch err @@ -560,7 +571,7 @@ macro testloop(args...) tests = testloop.args[2] blk = quote $ts = DefaultTestSet($(esc(desc))) - add_testset($ts) + push_testset($ts) try $(esc(tests)) catch err @@ -580,34 +591,35 @@ end """ get_testset() -Retrieve the active test set from the task's local storage. If no -test set is active, use the fallback default test set. +Retrieve the active test set from the task's local storage and the options used +to create it. If no test set is active, use the fallback default test set. """ function get_testset() testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) - return length(testsets) == 0 ? fallback_testset : testsets[end] + return length(testsets) == 0 ? (fallback_testset, Dict()) : testsets[end] end """ - add_testset(ts::AbstractTestSet) + push_testset(ts::AbstractTestSet[, options::Dict]) -Adds the test set to the task_local_storage. +Adds the test set to the task_local_storage, as well as the options used to +create it. """ -function add_testset(ts::AbstractTestSet) +function push_testset(ts::AbstractTestSet, options::Dict = Dict()) testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) - push!(testsets, ts) + push!(testsets, (ts, options)) setindex!(task_local_storage(), testsets, :__BASETESTNEXT__) end """ pop_testset() -Pops the last test set added to the task_local_storage. If there are no -active test sets, returns the default test set. +Pops the last test set added to the task_local_storage along with the options +used to create it. If there are no active test sets, returns the default test set. """ function pop_testset() testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) - ret = length(testsets) == 0 ? fallback_testset : pop!(testsets) + ret = length(testsets) == 0 ? (fallback_testset, Dict()) : pop!(testsets) setindex!(task_local_storage(), testsets, :__BASETESTNEXT__) return ret end diff --git a/test/test.jl b/test/test.jl index bb4920f7de2cd..90a896ff39327 100644 --- a/test/test.jl +++ b/test/test.jl @@ -123,5 +123,67 @@ end @test typeof(tss[1]) == Base.Test.DefaultTestSet @test typeof(tss[1].results[1]) == Base.Test.Pass - +# now we're done running tests with DefaultTestSet so we can go back to STDOUT redirect_stdout(OLD_STDOUT) + +immutable CustomTestSet <: Base.Test.AbstractTestSet + description::AbstractString + foo::Int + results::Vector + CustomTestSet(desc; foo=1) = new(desc, foo, []) +end + +Base.Test.record(ts::CustomTestSet, child::Base.Test.AbstractTestSet) = push!(ts.results, child) +Base.Test.record(ts::CustomTestSet, res::Base.Test.Result) = push!(ts.results, res) +Base.Test.finish(ts::CustomTestSet) = ts + +ts = @testset CustomTestSet "Testing custom testsets" begin + # this testset should inherit the parent testset type + @testset "custom testset inner 1" begin + @test true + @test false + @test error("this error will be reported as an error") + @test_throws ErrorException nothing + @test_throws ErrorException error("this error is a success") + end + # this testset has its own testset type + @testset CustomTestSet foo=4 "custom testset inner 2" begin + # this testset should inherit the type and arguments + @testset "custom testset inner 2 inner" begin + @test true + end + end +end + +@test typeof(ts) == CustomTestSet +@test ts.foo == 1 +@test ts.description == "Testing custom testsets" +@test typeof(ts.results[1]) == CustomTestSet +@test ts.results[1].description == "custom testset inner 1" +@test ts.results[1].foo == 1 +@test typeof(ts.results[1].results[1]) == Base.Test.Pass +@test typeof(ts.results[1].results[2]) == Base.Test.Fail +@test typeof(ts.results[1].results[3]) == Base.Test.Error +@test typeof(ts.results[1].results[4]) == Base.Test.Fail +@test typeof(ts.results[1].results[5]) == Base.Test.Pass + +@test typeof(ts.results[2]) == CustomTestSet +@test ts.results[2].description == "custom testset inner 1" +@test ts.results[2].foo == 4 +@test typeof(ts.results[2].results[1]) == CustomTestSet +@test ts.results[2].results[1].foo == 4 +@test typeof(ts.results[2].results[1].results[1]) == Base.Test.Pass + +# # test custom testset types on testloops +# tss = @testloop CustomTestSet "custom testloop $i" for i in 1:6 +# @test iseven(i) +# end +# +# for i in 1:6 +# @test typeof(tss[i]) == CustomTestSet +# if iseven(i) +# @test typeof(tss[i].results[1]) == Base.Test.Pass +# else +# @test typeof(tss[i].results[1]) == Base.Test.Fail +# end +# end From 9e3938700ed1350ba1023d2f85c3f0f38002ff53 Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Mon, 5 Oct 2015 19:18:04 -0400 Subject: [PATCH 0450/1938] implements testset type inheritance and option setting --- base/test.jl | 96 +++++++++++++++++++++++++++++++--------------------- test/test.jl | 83 ++++++++++++++++++++++++++++++--------------- 2 files changed, 113 insertions(+), 66 deletions(-) diff --git a/base/test.jl b/base/test.jl index 3f0fd54ab9194..183a1d0ae26ab 100644 --- a/base/test.jl +++ b/base/test.jl @@ -476,47 +476,36 @@ are any `Fail`s or `Error`s, an exception will be thrown only at the end, along with a summary of the test results. """ macro testset(args...) - length(args) > 0 || error("Too few arguments to @testset") + length(args) > 0 || error("No arguments to @testset") - options = Dict{Symbol, Any}() - desc = "test set" tests = args[end] # if we're at the top level we'll default to DefaultTestSet. Otherwise # default to the type of the parent testset - testsettype = :(get_testset_depth() == 0 ? DefaultTestSet : typeof(get_testset())) - for arg in args[1:end-1] - # a standalone symbol is assumed to be the test set we should use - if isa(arg, Symbol) - testsettype = arg - # a string is the description - elseif isa(arg, String) - desc = arg - # an assignment is an option - elseif isa(arg, Expr) && arg.head == :(=) - options[arg.args[1]] = eval(arg.args[2]) - else - error("Unexpected argument $arg to @testset") - end + desc, testsettype, options = parse_testset_args(args[1:end-1]) + if desc == nothing + desc = "test set" + end + if testsettype == nothing + testsettype = :(get_testset_depth() == 0 ? DefaultTestSet : typeof(get_testset())) end # Generate a block of code that initializes a new testset, adds # it to the task local storage, evaluates the test(s), before # finally removing the testset and giving it a change to take # action (such as reporting the results) - ts = gensym() quote - $ts = $(esc(testsettype))($desc; $options...) - push_testset($ts) + ts = $(testsettype)($desc; $options...) + push_testset(ts) try $(esc(tests)) catch err # something in the test block threw an error. Count that as an # error in this test set - record($ts, Error(:nontest_error, :(), err, catch_backtrace())) + record(ts, Error(:nontest_error, :(), err, catch_backtrace())) end pop_testset() - finish($ts) + finish(ts) end end @@ -531,6 +520,7 @@ is provided, one is constructed based on the variables. """ macro testloop(args...) length(args) > 0 || error("no arguments to @testloop") + testloop = args[end] isa(testloop,Expr) && testloop.head == :for || error("Unexpected argument to @testloop") # pull out the loop variables. We might need them for generating the @@ -547,12 +537,9 @@ macro testloop(args...) error("Unexpected argument to @testloop") end - # Parse arguments to do determine if any options passed in - if length(args) == 2 - # Looks like description format - desc = args[1] - isa(desc,AbstractString) || (isa(desc,Expr) && desc.head == :string) || error("Unexpected argument to @testloop") - elseif length(args) == 1 + desc, testsettype, options = parse_testset_args(args[1:end-1]) + + if desc == nothing # No description provided. Generate from the loop variable names v = loopvars[1].args[1] desc = Expr(:string,"$v = ",v) # first variable @@ -561,30 +548,61 @@ macro testloop(args...) push!(desc.args,", $v = ") push!(desc.args,v) end - else - error("Too many arguments to @testloop") + end + + if testsettype == nothing + testsettype = :(get_testset_depth() == 0 ? DefaultTestSet : typeof(get_testset())) end # Uses a similar block as for `@testset`, except that it is # wrapped in the outer loop provided by the user - ts = gensym() tests = testloop.args[2] blk = quote - $ts = DefaultTestSet($(esc(desc))) - push_testset($ts) + ts = $(testsettype)($desc; $options...) + push_testset(ts) try $(esc(tests)) catch err # something in the test block threw an error. Count that as an # error in this test set - record($ts, Error(:nontest_error, :(), err, catch_backtrace())) + record(ts, Error(:nontest_error, :(), err, catch_backtrace())) end pop_testset() - finish($ts) + finish(ts) end Expr(:comprehension, blk, [esc(v) for v in loopvars]...) end +""" +Parse the arguments to the `@testset` or `@testloop` macro to pull out +the description, Testset Type, and options. Generally this should be called +with all the macro arguments except the last one, which is the test expression +itself. +""" +function parse_testset_args(args) + desc = nothing + testsettype = nothing + options = :(Dict{Symbol, Any}()) + for arg in args[1:end] + # a standalone symbol is assumed to be the test set we should use + if isa(arg, Symbol) + testsettype = esc(arg) + # a string is the description + elseif isa(arg, AbstractString) || (isa(arg, Expr) && arg.head == :string) + desc = esc(arg) + # an assignment is an option + elseif isa(arg, Expr) && arg.head == :(=) + # we're building up a Dict literal here + key = Expr(:quote, arg.args[1]) + push!(options.args, Expr(:(=>), key, arg.args[2])) + else + error("Unexpected argument $arg to @testset") + end + end + + (desc, testsettype, options) +end + #----------------------------------------------------------------------- # Various helper methods for test sets @@ -596,7 +614,7 @@ to create it. If no test set is active, use the fallback default test set. """ function get_testset() testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) - return length(testsets) == 0 ? (fallback_testset, Dict()) : testsets[end] + return length(testsets) == 0 ? (fallback_testset) : testsets[end] end """ @@ -605,9 +623,9 @@ end Adds the test set to the task_local_storage, as well as the options used to create it. """ -function push_testset(ts::AbstractTestSet, options::Dict = Dict()) +function push_testset(ts::AbstractTestSet) testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) - push!(testsets, (ts, options)) + push!(testsets, ts) setindex!(task_local_storage(), testsets, :__BASETESTNEXT__) end @@ -619,7 +637,7 @@ used to create it. If there are no active test sets, returns the default test se """ function pop_testset() testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) - ret = length(testsets) == 0 ? (fallback_testset, Dict()) : pop!(testsets) + ret = length(testsets) == 0 ? fallback_testset : pop!(testsets) setindex!(task_local_storage(), testsets, :__BASETESTNEXT__) return ret end diff --git a/test/test.jl b/test/test.jl index 90a896ff39327..3417adea10b77 100644 --- a/test/test.jl +++ b/test/test.jl @@ -126,16 +126,26 @@ end # now we're done running tests with DefaultTestSet so we can go back to STDOUT redirect_stdout(OLD_STDOUT) +# import the methods needed for defining our own testset type +import Base.Test: record, finish, get_testset_depth, get_testset +import Base.Test: AbstractTestSet, Result, Pass, Fail, Error immutable CustomTestSet <: Base.Test.AbstractTestSet description::AbstractString foo::Int results::Vector + # constructor takes a description string and options keyword arguments CustomTestSet(desc; foo=1) = new(desc, foo, []) end -Base.Test.record(ts::CustomTestSet, child::Base.Test.AbstractTestSet) = push!(ts.results, child) -Base.Test.record(ts::CustomTestSet, res::Base.Test.Result) = push!(ts.results, res) -Base.Test.finish(ts::CustomTestSet) = ts +record(ts::CustomTestSet, child::AbstractTestSet) = push!(ts.results, child) +record(ts::CustomTestSet, res::Result) = push!(ts.results, res) +function finish(ts::CustomTestSet) + # just record if we're not the top-level parent + if get_testset_depth() > 0 + record(get_testset(), ts) + end + ts +end ts = @testset CustomTestSet "Testing custom testsets" begin # this testset should inherit the parent testset type @@ -148,8 +158,15 @@ ts = @testset CustomTestSet "Testing custom testsets" begin end # this testset has its own testset type @testset CustomTestSet foo=4 "custom testset inner 2" begin - # this testset should inherit the type and arguments - @testset "custom testset inner 2 inner" begin + # this testset should inherit the type, but not the argument. If a particular + # testset type wants inheritance behavior they should implement it themselves + # using get_testset() in the constructor + @testset "custom testset inner 2 inner 1" begin + @test true + end + # make sure the RHS can use computed values, also tests options without + # specifying the testset type + @testset foo=(1+2) "custom testset inner 2 inner 2" begin @test true end end @@ -161,29 +178,41 @@ end @test typeof(ts.results[1]) == CustomTestSet @test ts.results[1].description == "custom testset inner 1" @test ts.results[1].foo == 1 -@test typeof(ts.results[1].results[1]) == Base.Test.Pass -@test typeof(ts.results[1].results[2]) == Base.Test.Fail -@test typeof(ts.results[1].results[3]) == Base.Test.Error -@test typeof(ts.results[1].results[4]) == Base.Test.Fail -@test typeof(ts.results[1].results[5]) == Base.Test.Pass +@test typeof(ts.results[1].results[1]) == Pass +@test typeof(ts.results[1].results[2]) == Fail +@test typeof(ts.results[1].results[3]) == Error +@test typeof(ts.results[1].results[4]) == Fail +@test typeof(ts.results[1].results[5]) == Pass @test typeof(ts.results[2]) == CustomTestSet -@test ts.results[2].description == "custom testset inner 1" +@test ts.results[2].description == "custom testset inner 2" @test ts.results[2].foo == 4 @test typeof(ts.results[2].results[1]) == CustomTestSet -@test ts.results[2].results[1].foo == 4 -@test typeof(ts.results[2].results[1].results[1]) == Base.Test.Pass - -# # test custom testset types on testloops -# tss = @testloop CustomTestSet "custom testloop $i" for i in 1:6 -# @test iseven(i) -# end -# -# for i in 1:6 -# @test typeof(tss[i]) == CustomTestSet -# if iseven(i) -# @test typeof(tss[i].results[1]) == Base.Test.Pass -# else -# @test typeof(tss[i].results[1]) == Base.Test.Fail -# end -# end +@test ts.results[2].results[1].foo == 1 +@test typeof(ts.results[2].results[1].results[1]) == Pass +@test typeof(ts.results[2].results[2]) == CustomTestSet +@test ts.results[2].results[2].foo == 3 + +# test custom testset types on testloops +tss = @testloop CustomTestSet foo=3 "custom testloop $i" for i in 1:6 + @testloop "inner testloop $i-$j" for j in 1:3 + @test iseven(i + j) + end + # make sure a testset within a testloop works + @testset "inner testset $i" begin + @test iseven(i) + end +end + + +for i in 1:6 + @test typeof(tss[i]) == CustomTestSet + @test tss[i].foo == 3 + for j in 1:3 + @test typeof(tss[i].results[j]) == CustomTestSet + @test tss[i].results[j].foo == 1 + @test typeof(tss[i].results[j].results[1]) == (iseven(i+j) ? Pass : Fail) + end + @test typeof(tss[i].results[4]) == CustomTestSet + @test typeof(tss[i].results[4].results[1]) == (iseven(i) ? Pass : Fail) +end From f7c2f84fe2e4973f4560d69dda390e636348f798 Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Tue, 6 Oct 2015 13:30:15 -0400 Subject: [PATCH 0451/1938] all tests passing for new testset type functionality --- base/test.jl | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/base/test.jl b/base/test.jl index 183a1d0ae26ab..c9c376a3b0f26 100644 --- a/base/test.jl +++ b/base/test.jl @@ -479,13 +479,13 @@ macro testset(args...) length(args) > 0 || error("No arguments to @testset") tests = args[end] - # if we're at the top level we'll default to DefaultTestSet. Otherwise - # default to the type of the parent testset desc, testsettype, options = parse_testset_args(args[1:end-1]) if desc == nothing desc = "test set" end + # if we're at the top level we'll default to DefaultTestSet. Otherwise + # default to the type of the parent testset if testsettype == nothing testsettype = :(get_testset_depth() == 0 ? DefaultTestSet : typeof(get_testset())) end @@ -542,11 +542,11 @@ macro testloop(args...) if desc == nothing # No description provided. Generate from the loop variable names v = loopvars[1].args[1] - desc = Expr(:string,"$v = ",v) # first variable + desc = Expr(:string,"$v = ", esc(v)) # first variable for l = loopvars[2:end] v = l.args[1] push!(desc.args,", $v = ") - push!(desc.args,v) + push!(desc.args, esc(v)) end end @@ -609,19 +609,18 @@ end """ get_testset() -Retrieve the active test set from the task's local storage and the options used -to create it. If no test set is active, use the fallback default test set. +Retrieve the active test set from the task's local storage. If no +test set is active, use the fallback default test set. """ function get_testset() testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) - return length(testsets) == 0 ? (fallback_testset) : testsets[end] + return length(testsets) == 0 ? fallback_testset : testsets[end] end """ - push_testset(ts::AbstractTestSet[, options::Dict]) + push_testset(ts::AbstractTestSet) -Adds the test set to the task_local_storage, as well as the options used to -create it. +Adds the test set to the task_local_storage. """ function push_testset(ts::AbstractTestSet) testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) @@ -632,8 +631,8 @@ end """ pop_testset() -Pops the last test set added to the task_local_storage along with the options -used to create it. If there are no active test sets, returns the default test set. +Pops the last test set added to the task_local_storage. If there are no +active test sets, returns the default test set. """ function pop_testset() testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) From f90a976e6fd1093bcaf33a39802d6a63024208d9 Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Tue, 6 Oct 2015 16:38:42 -0400 Subject: [PATCH 0452/1938] switches to `using` for methods/types we're calling but not extending --- test/test.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test.jl b/test/test.jl index 3417adea10b77..6496919aefa92 100644 --- a/test/test.jl +++ b/test/test.jl @@ -127,8 +127,9 @@ end redirect_stdout(OLD_STDOUT) # import the methods needed for defining our own testset type -import Base.Test: record, finish, get_testset_depth, get_testset -import Base.Test: AbstractTestSet, Result, Pass, Fail, Error +import Base.Test: record, finish +using Base.Test: get_testset_depth, get_testset +using Base.Test: AbstractTestSet, Result, Pass, Fail, Error immutable CustomTestSet <: Base.Test.AbstractTestSet description::AbstractString foo::Int From 8492087ad2e9e7a1761477242d5d56ad2bf70184 Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Tue, 6 Oct 2015 17:07:59 -0400 Subject: [PATCH 0453/1938] updates docstrings for at-testset and at-testloop macros --- base/test.jl | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/base/test.jl b/base/test.jl index c9c376a3b0f26..d3a2ab0a33f99 100644 --- a/base/test.jl +++ b/base/test.jl @@ -468,12 +468,19 @@ end #----------------------------------------------------------------------- """ - @testset "description" begin ... end - @testset begin ... end + @testset [CustomTestSet] [option=val ...] ["description"] begin ... end Starts a new test set. The test results will be recorded, and if there -are any `Fail`s or `Error`s, an exception will be thrown only at the end, -along with a summary of the test results. +are any `Fail`s or `Error`s, an exception will be thrown at the end of the +top-level (non-nested) test set, along with a summary of the test results. + +Any custom testset type (subtype of `AbstractTestSet`) can be given and it will +also be used for any nested `@testset` or `@testloop` invocations. The given +options are only applied to the test set where they are given. The default test +set type does not take any options. + +By default the `@testset` macro will return the testset object itself, though +this behavior can be customized in other testset types. """ macro testset(args...) length(args) > 0 || error("No arguments to @testset") @@ -511,12 +518,20 @@ end """ - @testloop "description \$v" for v in (...) ... end - @testloop for x in (...), y in (...) ... end + @testloop [CustomTestSet] [option=val ...] ["description \$v"] for v in (...) ... end + @testloop [CustomTestSet] [option=val ...] ["description \$v, \$w"] for v in (...), w in (...) ... end + +Starts a new test set for each iteration of the loop. The description string +accepts interpolation from the loop indices. If no description is provided, one +is constructed based on the variables. + +Any custom testset type (subtype of `AbstractTestSet`) can be given and it will +also be used for any nested `@testset` or `@testloop` invocations. The given +options are only applied to the test sets where they are given. The default test +set type does not take any options. -Starts a new test set for each iteration of the loop. The description -string accepts interpolation from the loop indices. If no description -is provided, one is constructed based on the variables. +By default the `@testset` macro will return a list of the testset objects used +in each iteration, though this behavior can be customized in other testset types. """ macro testloop(args...) length(args) > 0 || error("no arguments to @testloop") @@ -632,7 +647,7 @@ end pop_testset() Pops the last test set added to the task_local_storage. If there are no -active test sets, returns the default test set. +active test sets, returns the fallback default test set. """ function pop_testset() testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) From 89636aa609133c93f9561fabef184058303dc6aa Mon Sep 17 00:00:00 2001 From: Kiran Pamnany Date: Tue, 6 Oct 2015 16:15:15 -0700 Subject: [PATCH 0454/1938] Updated license. --- base/atomics.jl | 25 +-------------------- base/locks.jl | 25 +-------------------- base/threadingconstructs.jl | 26 +--------------------- base/threads.jl | 25 +-------------------- src/ia_misc.h | 2 ++ src/threadgroup.c | 27 +---------------------- src/threadgroup.h | 27 +---------------------- src/threading.c | 27 +---------------------- src/threading.h | 27 +---------------------- test/perf/threads/laplace3d/laplace3d.c | 27 +---------------------- test/perf/threads/laplace3d/laplace3d.jl | 2 ++ test/perf/threads/lbm3d/lbm3d.jl | 2 ++ test/perf/threads/stockcorr/pstockcorr.jl | 2 ++ 13 files changed, 17 insertions(+), 227 deletions(-) diff --git a/base/atomics.jl b/base/atomics.jl index 78b3095d89b9b..b211923a5dd6d 100644 --- a/base/atomics.jl +++ b/base/atomics.jl @@ -1,27 +1,4 @@ -# Copyright (c) 2015, Intel Corporation -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of Intel Corporation nor the names of its contributors -# may be used to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# This file is a part of Julia. License is MIT: http://julialang.org/license using Base.Intrinsics: llvmcall diff --git a/base/locks.jl b/base/locks.jl index 5f0d8dec168fa..232ccf1a105a7 100644 --- a/base/locks.jl +++ b/base/locks.jl @@ -1,27 +1,4 @@ -# Copyright (c) 2015, Intel Corporation -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of Intel Corporation nor the names of its contributors -# may be used to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# This file is a part of Julia. License is MIT: http://julialang.org/license export SpinLock, Mutex, init_lock!, destroy_lock!, lock!, trylock!, unlock! diff --git a/base/threadingconstructs.jl b/base/threadingconstructs.jl index 3be1eea112d2a..967e59ed0a14c 100644 --- a/base/threadingconstructs.jl +++ b/base/threadingconstructs.jl @@ -1,28 +1,4 @@ -# Copyright (c) 2015, Intel Corporation -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of Intel Corporation nor the names of its contributors -# may be used to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - +# This file is a part of Julia. License is MIT: http://julialang.org/license export threadid, maxthreads, nthreads, @threads diff --git a/base/threads.jl b/base/threads.jl index d0a9e37b1c083..4550e41dd0ba7 100644 --- a/base/threads.jl +++ b/base/threads.jl @@ -1,27 +1,4 @@ -# Copyright (c) 2015, Intel Corporation -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of Intel Corporation nor the names of its contributors -# may be used to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# This file is a part of Julia. License is MIT: http://julialang.org/license module Threads diff --git a/src/ia_misc.h b/src/ia_misc.h index 18cf8a24d0d9b..f94bea826a28d 100644 --- a/src/ia_misc.h +++ b/src/ia_misc.h @@ -1,3 +1,5 @@ +// This file is a part of Julia. License is MIT: http://julialang.org/license + #ifndef IA_MISC_H #define IA_MISC_H diff --git a/src/threadgroup.c b/src/threadgroup.c index e229fe4af5fe6..37b7a9d0a85af 100644 --- a/src/threadgroup.c +++ b/src/threadgroup.c @@ -1,29 +1,4 @@ -/* -Copyright (c) 2014, Intel Corporation - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +// This file is a part of Julia. License is MIT: http://julialang.org/license /* threading infrastructure diff --git a/src/threadgroup.h b/src/threadgroup.h index 61d492fd76114..0719a65e377fc 100644 --- a/src/threadgroup.h +++ b/src/threadgroup.h @@ -1,29 +1,4 @@ -/* -Copyright (c) 2014, Intel Corporation - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +// This file is a part of Julia. License is MIT: http://julialang.org/license #ifndef THREADGROUP_H #define THREADGROUP_H diff --git a/src/threading.c b/src/threading.c index 0578bc4463dd2..670670185fb07 100644 --- a/src/threading.c +++ b/src/threading.c @@ -1,29 +1,4 @@ -/* -Copyright (c) 2014, Intel Corporation - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +// This file is a part of Julia. License is MIT: http://julialang.org/license /* threading infrastructure diff --git a/src/threading.h b/src/threading.h index f6e226b0be5bf..a91ffcd2c0e2f 100644 --- a/src/threading.h +++ b/src/threading.h @@ -1,29 +1,4 @@ -/* -Copyright (c) 2014, Intel Corporation - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +// This file is a part of Julia. License is MIT: http://julialang.org/license #ifndef THREADING_H #define THREADING_H diff --git a/test/perf/threads/laplace3d/laplace3d.c b/test/perf/threads/laplace3d/laplace3d.c index 7e69b47935f8b..d4c20dc3f649c 100644 --- a/test/perf/threads/laplace3d/laplace3d.c +++ b/test/perf/threads/laplace3d/laplace3d.c @@ -1,29 +1,4 @@ -/* -Copyright (c) 2014, Intel Corporation - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +// This file is a part of Julia. License is MIT: http://julialang.org/license /* Laplace 3D diff --git a/test/perf/threads/laplace3d/laplace3d.jl b/test/perf/threads/laplace3d/laplace3d.jl index 616224e745d48..4cf7127ddd5eb 100644 --- a/test/perf/threads/laplace3d/laplace3d.jl +++ b/test/perf/threads/laplace3d/laplace3d.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + ## 3D Laplace equation using Base.Cartesian diff --git a/test/perf/threads/lbm3d/lbm3d.jl b/test/perf/threads/lbm3d/lbm3d.jl index 5bf9088dab549..36b6f282aac89 100644 --- a/test/perf/threads/lbm3d/lbm3d.jl +++ b/test/perf/threads/lbm3d/lbm3d.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + # 3D Lattice Boltzmann (BGK) model of a fluid. # (http://exolete.com/lbm/) # D3Q19 model. At each timestep, particle densities propagate diff --git a/test/perf/threads/stockcorr/pstockcorr.jl b/test/perf/threads/stockcorr/pstockcorr.jl index b4530c07f9556..c9f29492e33ff 100644 --- a/test/perf/threads/stockcorr/pstockcorr.jl +++ b/test/perf/threads/stockcorr/pstockcorr.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + ## Test case from Issue #445 ## ## Added threaded implementation (2015-04-06) From 1ed7b08edcd093bd23b8b33d9939fe58b46c63d7 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 6 Oct 2015 19:51:40 -0400 Subject: [PATCH 0455/1938] Fix backtrace test on linux with MCJIT We always need to look up the address just before the one we get in the backtrace because the address we obtain in the backtrace is the one of the instruction that will be executed next. --- test/backtrace.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/backtrace.jl b/test/backtrace.jl index 16bfec0f626d1..74aef91b1453e 100644 --- a/test/backtrace.jl +++ b/test/backtrace.jl @@ -76,7 +76,7 @@ loc = functionloc(f12977) @test loc[2] == linenum # issue #922: SimplifyCFG pass merges throws -code_loc(p, skipC=true) = ccall(:jl_lookup_code_address, Any, (Ptr{Void},Cint), p, skipC) +code_loc(p, skipC=true) = ccall(:jl_lookup_code_address, Any, (Ptr{Void},Cint), p-1, skipC) @noinline function test_throw_commoning(x) if x==1; throw(AssertionError()); end From 0f3ee7355066a5a4b561c95b0e2f968482b7d586 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 6 Oct 2015 20:18:44 -0400 Subject: [PATCH 0456/1938] Fix test that my last fix broke The original test passed a temporary array since `ind1` is a range - oops. --- test/backtrace.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/backtrace.jl b/test/backtrace.jl index 74aef91b1453e..d5b9f3ad97c5f 100644 --- a/test/backtrace.jl +++ b/test/backtrace.jl @@ -99,5 +99,5 @@ let ind2 = find(:test_throw_commoning .== map(b->code_loc(b)[1], b2)) @test !isempty(ind1) @test !isempty(ind2) - @test code_loc(b1[ind1])[3] != code_loc(b2[ind2])[3] + @test code_loc(b1[ind1[1]])[3] != code_loc(b2[ind2[1]])[3] end From f5da3f8aba86d14e32577ea9dc6e7f0993c2a748 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 6 Oct 2015 21:33:37 -0400 Subject: [PATCH 0457/1938] Fix atomics.jl on llvm-svn "svn" is not a `VersionNumber` --- base/atomics.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/atomics.jl b/base/atomics.jl index b211923a5dd6d..d69a5931fdded 100644 --- a/base/atomics.jl +++ b/base/atomics.jl @@ -30,7 +30,7 @@ unsafe_convert{T}(::Type{Ptr{T}}, x::Atomic{T}) = convert(Ptr{T}, pointer_from_o setindex!{T}(x::Atomic{T}, v) = setindex!(x, convert(T, v)) for (typ, lt) in atomicintsmap - rt = VersionNumber(Base.libllvm_version) >= v"3.6" ? "$lt, $lt*" : "$lt*" + rt = (Base.libllvm_version == "svn" || VersionNumber(Base.libllvm_version) >= v"3.6") ? "$lt, $lt*" : "$lt*" @eval getindex(x::Atomic{$typ}) = llvmcall($""" %rv = load atomic volatile $rt %0 monotonic, align $WORD_SIZE From 68bc1ca37c365677c64b862fe1e350f8690583be Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Tue, 6 Oct 2015 21:09:03 -0400 Subject: [PATCH 0458/1938] Fix gen_call_with_extracted unmatched case --- base/interactiveutil.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 8a9ec7f13e0be..bee7db3be6ebf 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -243,6 +243,7 @@ function gen_call_with_extracted_types(fcn, ex0) return Expr(:call, fcn, esc(args[1]), Expr(:call, typesof, map(esc, args[2:end])...)) end + exret = Expr(:none) ex = expand(ex0) if !isa(ex, Expr) exret = Expr(:call, :error, "expression is not a function call or symbol") @@ -266,7 +267,8 @@ function gen_call_with_extracted_types(fcn, ex0) Expr(:call, typesof, map(esc, a1.args[2:end])...)) end end - elseif ex.head == :thunk + end + if ex.head == :thunk || exret.head == :none exret = Expr(:call, :error, "expression is not a function call, " * "or is too complex for @$fcn to analyze; " * "break it down to simpler parts if possible") From c510114be12ce836befdceacd06f66a28a3db069 Mon Sep 17 00:00:00 2001 From: wildart Date: Wed, 7 Oct 2015 01:13:42 -0400 Subject: [PATCH 0459/1938] added: docs, `AbstractPayload` interface functions, type fixes --- base/libgit2.jl | 2 +- base/libgit2/callbacks.jl | 20 +++++++++----------- base/libgit2/consts.jl | 40 +++++++++++++++++++++++++++++---------- base/libgit2/types.jl | 10 +++++++++- 4 files changed, 49 insertions(+), 23 deletions(-) diff --git a/base/libgit2.jl b/base/libgit2.jl index b2b4dd141aa2c..c64daba7cb06a 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -301,7 +301,7 @@ function clone{P<:AbstractPayload}(repo_url::AbstractString, repo_path::Abstract # setup clone options fetch_opts=FetchOptions(callbacks = RemoteCallbacks(credentials_cb(), payload)) clone_opts = CloneOptions( - bare = Int32(isbare), + bare = Cint(isbare), checkout_branch = isempty(branch) ? Cstring_NULL : convert(Cstring, pointer(branch)), fetch_opts=fetch_opts, diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 78f6536736800..fbc57c5fe6d6b 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -37,30 +37,28 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, url = bytestring(url_ptr) if isset(allowed_types, Cuint(Consts.CREDTYPE_USERPASS_PLAINTEXT)) - username = password = "" + username = userpass = "" if payload_ptr != C_NULL payload = unsafe_pointer_to_objref(payload_ptr) if isa(payload, AbstractPayload) - username = isdefined(payload, :user) ? getfield(payload, :user) : "" - password = isdefined(payload, :pass) ? getfield(payload, :pass) : "" - if isdefined(payload, :used) - getfield(payload, :used) && return Cint(-1) - setfield!(payload, :used, true) - end + isused(payload) && return Cint(-1) + username = user(payload) + userpass = password(payload) + setused!(payload, true) end end if isempty(username) username = prompt("Username for '$url'") end - if isempty(password) - password = prompt("Password for '$username@$url'", password=true) + if isempty(userpass) + userpass = prompt("Password for '$username@$url'", password=true) end - length(username) == 0 && length(password) == 0 && return Cint(-1) + length(username) == 0 && length(userpass) == 0 && return Cint(-1) err = ccall((:git_cred_userpass_plaintext_new, :libgit2), Cint, (Ptr{Ptr{Void}}, Cstring, Cstring), - cred, username, password) + cred, username, userpass) err == 0 && return Cint(0) elseif isset(allowed_types, Cuint(Consts.CREDTYPE_SSH_KEY)) && err > 0 # use ssh-agent diff --git a/base/libgit2/consts.jl b/base/libgit2/consts.jl index ee24ec86c9367..3d9b54a89b207 100644 --- a/base/libgit2/consts.jl +++ b/base/libgit2/consts.jl @@ -242,10 +242,17 @@ module Consts SUBMODULE_IGNORE_DIRTY = 3, # only dirty if HEAD moved SUBMODULE_IGNORE_ALL = 4) # never dirty - @enum(GIT_REPOSITORY_OPEN, REPOSITORY_OPEN_DEFAULT = 0, # default value - REPOSITORY_OPEN_NO_SEARCH = 1<<0, # only open the repository if it can be immediately found - REPOSITORY_OPEN_CROSS_FS = 1<<1, # open will not continue searching across FS boundaries - REPOSITORY_OPEN_BARE = 1<<2) # open repository as a bare repo + """ +Option flags for `GitRepo`. + +* REPOSITORY_OPEN_NO_SEARCH - Only open the repository if it can be immediately found in the `path`. Do not walk up from the `path` looking at parent directories. +* REPOSITORY_OPEN_CROSS_FS - Unless this flag is set, open will not continue searching across filesystem boundaries. (E.g. Searching in a user's home directory "/home/user/source/" will not return "/.git/" as the found repo if "/" is a different filesystem than "/home".) +* REPOSITORY_OPEN_BARE - Open repository as a bare repo regardless of core.bare config, and defer loading config file for faster setup. + """ + @enum(GIT_REPOSITORY_OPEN, REPOSITORY_OPEN_DEFAULT = 0, + REPOSITORY_OPEN_NO_SEARCH = 1<<0, + REPOSITORY_OPEN_CROSS_FS = 1<<1, + REPOSITORY_OPEN_BARE = 1<<2) @enum(GIT_BRANCH, BRANCH_LOCAL = 1, BRANCH_REMOTE = 2) @@ -264,11 +271,24 @@ module Consts CREDTYPE_USERNAME = Cuint(1 << 5), CREDTYPE_SSH_MEMORY = Cuint(1 << 6)) + """ +Priority level of a config file. + +These priority levels correspond to the natural escalation logic (from higher to lower) when searching for config entries in git. + +* CONFIG_LEVEL_DEFAULT - Open the global, XDG and system configuration files if any available. +* CONFIG_LEVEL_SYSTEM - System-wide configuration file; /etc/gitconfig on Linux systems +* CONFIG_LEVEL_XDG - XDG compatible configuration file; typically ~/.config/git/config +* CONFIG_LEVEL_GLOBAL - User-specific configuration file (also called Global configuration file); typically ~/.gitconfig +* CONFIG_LEVEL_LOCAL - Repository specific configuration file; \$WORK_DIR/.git/config on non-bare repos +* CONFIG_LEVEL_APP - Application specific configuration file; freely defined by applications +* CONFIG_HIGHEST_LEVEL - Represents the highest level available config file (i.e. the most specific config file available that actually is loaded) + """ @enum(GIT_CONFIG, CONFIG_LEVEL_DEFAULT = 0, - CONFIG_LEVEL_SYSTEM = 1, # System-wide configuration file; /etc/gitconfig on Linux systems - CONFIG_LEVEL_XDG = 2, # XDG compatible configuration file; typically ~/.config/git/config - CONFIG_LEVEL_GLOBAL = 3, # User-specific configuration file (also called Global configuration file); typically ~/.gitconfig - CONFIG_LEVEL_LOCAL = 4, # Repository specific configuration file; $WORK_DIR/.git/config on non-bare repos - CONFIG_LEVEL_APP = 5, # Application specific configuration file; freely defined by applications - CONFIG_HIGHEST_LEVEL =-1) # Represents the highest level available config file (i.e. the most specific config file available that actually is loaded) + CONFIG_LEVEL_SYSTEM = 1, + CONFIG_LEVEL_XDG = 2, + CONFIG_LEVEL_GLOBAL = 3, + CONFIG_LEVEL_LOCAL = 4, + CONFIG_LEVEL_APP = 5, + CONFIG_HIGHEST_LEVEL =-1) end diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index 7fb092d5ce857..ad45090852ba1 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -52,6 +52,10 @@ end "Abstract payload type for callback functions" abstract AbstractPayload +user(p::AbstractPayload) = throw(AssertionError("Function 'user' is not implemented for type $(typeof(p))")) +password(p::AbstractPayload) = throw(AssertionError("Function 'password' is not implemented for type $(typeof(p))")) +isused(p::AbstractPayload) = throw(AssertionError("Function 'isused' is not implemented for type $(typeof(p))")) +setused!(p::AbstractPayload,v::Bool) = throw(AssertionError("Function 'setused!' is not implemented for type $(typeof(p))")) immutable CheckoutOptions version::Cuint @@ -529,4 +533,8 @@ type UserPasswordCredentials <: AbstractPayload pass::AbstractString used::Bool UserPasswordCredentials(u::AbstractString,p::AbstractString) = new(u,p,false) -end \ No newline at end of file +end +user(p::UserPasswordCredentials) = p.user +password(p::UserPasswordCredentials) = p.pass +isused(p::UserPasswordCredentials) = p.used +setused!(p::UserPasswordCredentials, val::Bool) = (p.used = val) From 442cf132cea528fc9f5853c100d2654eb08b18f7 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Tue, 6 Oct 2015 21:32:15 -0700 Subject: [PATCH 0460/1938] Add a test for #13464 --- test/reflection.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/reflection.jl b/test/reflection.jl index 2e2070e813aee..f3a53e633025e 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -193,3 +193,13 @@ end # issue #13264 @test isa((@which vcat(1...)), Method) + +# issue #13464 +let t13464 = "hey there sailor" + try + @which t13464[1,1] = (1.0,true) + error("unexpected") + catch err13464 + @test startswith(err13464.msg, "expression is not a function call, or is too complex") + end +end From 9feefa0c9061b3facf11310ee9f4f3e1c333149d Mon Sep 17 00:00:00 2001 From: wildart Date: Wed, 7 Oct 2015 05:07:12 -0400 Subject: [PATCH 0461/1938] use correct doc macro --- base/libgit2/consts.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/libgit2/consts.jl b/base/libgit2/consts.jl index 3d9b54a89b207..513a400ce7d3a 100644 --- a/base/libgit2/consts.jl +++ b/base/libgit2/consts.jl @@ -242,11 +242,11 @@ module Consts SUBMODULE_IGNORE_DIRTY = 3, # only dirty if HEAD moved SUBMODULE_IGNORE_ALL = 4) # never dirty - """ + doc""" Option flags for `GitRepo`. * REPOSITORY_OPEN_NO_SEARCH - Only open the repository if it can be immediately found in the `path`. Do not walk up from the `path` looking at parent directories. -* REPOSITORY_OPEN_CROSS_FS - Unless this flag is set, open will not continue searching across filesystem boundaries. (E.g. Searching in a user's home directory "/home/user/source/" will not return "/.git/" as the found repo if "/" is a different filesystem than "/home".) +* REPOSITORY_OPEN_CROSS_FS - Unless this flag is set, open will not continue searching across filesystem boundaries. (E.g. Searching in a user's home directory `/home/user/source/` will not return `/.git/` as the found repo if `/` is a different filesystem than `/home`.) * REPOSITORY_OPEN_BARE - Open repository as a bare repo regardless of core.bare config, and defer loading config file for faster setup. """ @enum(GIT_REPOSITORY_OPEN, REPOSITORY_OPEN_DEFAULT = 0, @@ -271,7 +271,7 @@ Option flags for `GitRepo`. CREDTYPE_USERNAME = Cuint(1 << 5), CREDTYPE_SSH_MEMORY = Cuint(1 << 6)) - """ + doc""" Priority level of a config file. These priority levels correspond to the natural escalation logic (from higher to lower) when searching for config entries in git. From d14c1c215b227ffc623b6edf7bb41a60cffaf323 Mon Sep 17 00:00:00 2001 From: wildart Date: Wed, 7 Oct 2015 07:46:06 -0400 Subject: [PATCH 0462/1938] fix #13375 --- base/pkg/read.jl | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/base/pkg/read.jl b/base/pkg/read.jl index df00d661ec88d..2cf17750d20a2 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -130,8 +130,8 @@ function installed_version(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::D Base.warn_once("unknown $pkg commit $(sha1[1:8]), metadata may be ahead of package cache") continue end - base == sha1 && push!(ancestors,ver) - base == head && push!(descendants,ver) + string(base) == sha1 && push!(ancestors,ver) + string(base) == head && push!(descendants,ver) end finally cache_has_head && LibGit2.finalize(crepo) @@ -180,17 +180,13 @@ function installed(avail::Dict=available()) ap = get(avail,pkg,Dict{VersionNumber,Available}()) prepo = LibGit2.GitRepo(pkg) try - try - ver = installed_version(pkg, prepo, ap) - fixed = isfixed(pkg, prepo, ap) - pkgs[pkg] = (ver, fixed) - catch e - pkgs[pkg] = (typemin(VersionNumber), true) - finally - LibGit2.finalize(prepo) - end + ver = installed_version(pkg, prepo, ap) + fixed = isfixed(pkg, prepo, ap) + pkgs[pkg] = (ver, fixed) catch pkgs[pkg] = (typemin(VersionNumber), true) + finally + finalize(prepo) end end return pkgs From 91d1eba3f86222d0b5516152dbce1ccfb175fad8 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 6 Oct 2015 16:15:31 -0400 Subject: [PATCH 0463/1938] fix Makefile conflict between USE_SYSTEM_BLAS and openblas-renaming --- deps/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deps/Makefile b/deps/Makefile index 7f935ac5f2d1f..6c5ac566c9e7b 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -1135,6 +1135,7 @@ $(OPENBLAS_OBJ_SOURCE): $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/config.status echo $(MAKE) -C $(dir $<) $(OPENBLAS_BUILD_OPTS) # echo first, so we only print the error message below in a failure case @$(MAKE) -C $(dir $<) $(OPENBLAS_BUILD_OPTS) || (echo $(WARNCOLOR)"*** Clean the OpenBLAS build with 'make -C deps clean-openblas'. Rebuild with 'make OPENBLAS_USE_THREAD=0 if OpenBLAS had trouble linking libpthread.so, and with 'make OPENBLAS_TARGET_ARCH=NEHALEM' if there were errors building SandyBridge support. Both these options can also be used simultaneously. ***"$(ENDCOLOR) && false) touch -c $@ +ifneq ($(USE_SYSTEM_BLAS),1) $(OPENBLAS_OBJ_TARGET): $(OPENBLAS_OBJ_SOURCE) | $(build_shlibdir) cp -f $< $@ ifeq ($(OS), Linux) @@ -1142,6 +1143,7 @@ ifeq ($(OS), Linux) ln -sf $(LIBBLASNAME).$(SHLIB_EXT) $(LIBBLASNAME).$(SHLIB_EXT).0 endif $(INSTALL_NAME_CMD)$(LIBBLASNAME).$(SHLIB_EXT) $@ +endif clean-openblas: -$(MAKE) -C $(BUILDDIR)/$(OPENBLAS_SRC_DIR) clean From 85f8a67d019627e7bbd8ea5fb5354577e4c60a00 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 6 Oct 2015 17:59:57 -0400 Subject: [PATCH 0464/1938] precompile_unspecialized was on the wrong branch, resulting in some defs not getting precompiled --- src/gf.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/gf.c b/src/gf.c index 6b3421ed4a9ac..b94198cb2c4e9 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1557,14 +1557,15 @@ void jl_compile_all_defs(jl_function_t *gf) continue; } } - if (m->func->linfo->unspecialized == NULL) { + func = m->func->linfo->unspecialized; + if (func == NULL) { func = jl_instantiate_method(m->func, jl_emptysvec); if (func->env != (jl_value_t*)jl_emptysvec) func->env = NULL; m->func->linfo->unspecialized = func; jl_gc_wb(m->func->linfo, func); - precompile_unspecialized(func, m->sig, m->tvars); } + precompile_unspecialized(func, m->sig, m->tvars); m = m->next; } JL_GC_POP(); From 770d96d7756e46dc0c61a501ad993205c56339fa Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 5 Oct 2015 15:17:06 -0400 Subject: [PATCH 0465/1938] fix minor makefile grievances --- Makefile | 2 +- deps/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index fa98ad496afda..8890e06a23f08 100644 --- a/Makefile +++ b/Makefile @@ -103,7 +103,7 @@ julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink ju debug release : % : julia-% julia-genstdlib: julia-sysimg-$(JULIA_BUILD_MODE) - @$(call PRINT_JULIA, $(JULIA_EXECUTABLE) doc/genstdlib.jl) + @$(call PRINT_JULIA, $(JULIA_EXECUTABLE) $(JULIAHOME)/doc/genstdlib.jl) docs: julia-genstdlib @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/doc diff --git a/deps/Makefile b/deps/Makefile index 6c5ac566c9e7b..5f7bddb04da82 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -1908,7 +1908,7 @@ $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER)/configure: $(SRCDIR)/srccache/patche cd $(dir $<) && $(TAR) zxf $< touch -c $@ $(BUILDDIR)/patchelf-$(PATCHELF_VER)/config.status: $(SRCDIR)/srccache/patchelf-$(PATCHELF_VER)/configure - mkdir $(dir $@) + mkdir -p $(dir $@) cd $(dir $@) && \ $< $(CONFIGURE_COMMON) touch -c $@ From 99e900ff16b521f94c849c05403c1bcaaf03fb8a Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 1 Oct 2015 13:13:59 -0400 Subject: [PATCH 0466/1938] fix whitespace script to not get confused by glob expansion, and whitespace errors --- contrib/check-whitespace.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/check-whitespace.sh b/contrib/check-whitespace.sh index ddbd3dda16bc8..cdb843c6fa876 100755 --- a/contrib/check-whitespace.sh +++ b/contrib/check-whitespace.sh @@ -5,6 +5,7 @@ # report an error if so # Files to check: +set -f # disable glob expansion in this script file_patterns=' *.1 *.c From 2d5d9a36d51303539a5a2fc2c3535ccd383f2782 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 2 Sep 2015 00:20:57 -0400 Subject: [PATCH 0467/1938] reduce cruft from codegen init --- src/codegen.cpp | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index c1ca8adb2f401..e3a99d67f2a80 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6016,6 +6016,11 @@ extern "C" void jl_init_codegen(void) .setTargetOptions(options) .setRelocationModel(Reloc::PIC_) .setCodeModel(CodeModel::Small) +#ifdef DISABLE_OPT + .setOptLevel(CodeGenOpt::None) +#else + .setOptLevel(CodeGenOpt::Aggressive) +#endif #if defined(USE_MCJIT) && !defined(LLVM36) .setUseMCJIT(true) #endif @@ -6038,33 +6043,13 @@ extern "C" void jl_init_codegen(void) MAttrs.append(1, "+vfp2"); // the processors that don't have VFP are old and (hopefully) rare. this affects the platform calling convention. #endif } - TargetMachine *targetMachine = eb.selectTarget( + jl_TargetMachine = eb.selectTarget( TheTriple, "", TheCPU, MAttrs); - assert(targetMachine && "Failed to select target machine -" + assert(jl_TargetMachine && "Failed to select target machine -" " Is the LLVM backend for this CPU enabled?"); - jl_TargetMachine = targetMachine->getTarget().createTargetMachine( - TheTriple.getTriple(), - targetMachine->getTargetCPU(), - targetMachine->getTargetFeatureString(), - targetMachine->Options, -#ifdef CODEGEN_TLS - Reloc::PIC_, - CodeModel::Small, -#else - Reloc::Default, - CodeModel::JITDefault, -#endif -#ifdef DISABLE_OPT - CodeGenOpt::None -#else - CodeGenOpt::Aggressive // -O3 -#endif - ); - delete targetMachine; - assert(jl_TargetMachine); #if defined(USE_MCJIT) && !defined(_CPU_ARM_) // FastISel seems to be buggy for ARM. Ref #13321 jl_TargetMachine->setFastISel(true); From 1b41d643d6eb5c81a6423cacf42a8066a42ebcc3 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 5 Oct 2015 14:13:19 -0400 Subject: [PATCH 0468/1938] work around a gcc register allocation error see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67814 --- src/alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index 9e975dcaa9019..ab453d5a8e264 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -581,7 +581,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) } if (al != 0) { size_t alsz = LLT_ALIGN(sz, al); - if (alsz > sz) + if (sz & (al - 1)) st->haspadding = 1; sz = alsz; if (al > alignm) From 930e6b55327c6caf60d0eae4d71ffdf43e55fbae Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 1 Oct 2015 14:54:33 -0400 Subject: [PATCH 0469/1938] set a min target architecture of pentium4 to require support for sse2 and _mm_pause --- .travis.yml | 1 + Make.inc | 22 ++++++++++++++++------ contrib/windows/msys_build.sh | 3 +-- src/codegen.cpp | 4 ++-- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 043df379cc610..9f073f494552b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,7 @@ before_install: sudo add-apt-repository ppa:staticfloat/julia-deps -y; sudo apt-get update -qq -y; if [ "$ARCH" = "i686" ]; then + export BUILDOPTS="$BUILDOPTS MARCH=pentium4"; sudo apt-get remove libblas3gf liblapack3gf libarmadillo2 -y; sudo apt-get install binutils:i386 -y; sudo apt-get install gcc:i386 g++:i386 make:i386 cpp:i386 g++-4.6:i386 gcc-4.6:i386 libssl-dev:i386 patchelf:i386 gfortran:i386 llvm-3.3-dev:i386 libsuitesparse-dev:i386 libopenblas-dev:i386 libopenblas-base:i386 libblas-dev:i386 liblapack-dev:i386 liblapack3:i386 libarpack2-dev:i386 libarpack2:i386 libfftw3-dev:i386 libgmp-dev:i386 libpcre3-dev:i386 libunwind7-dev:i386 libopenlibm-dev:i386 libmpfr-dev:i386 -y; diff --git a/Make.inc b/Make.inc index 610ec5d91df3e..9e234c2ac1be1 100644 --- a/Make.inc +++ b/Make.inc @@ -379,6 +379,8 @@ DEBUGFLAGS := -O0 -g -DJL_DEBUG_BUILD -fstack-protector-all SHIPFLAGS := -O3 -g -falign-functions endif +JCPPFLAGS += -ftls-model=global-dynamic + ifeq ($(USECCACHE), 1) # expand CC and CXX at declaration time because we will redefine them CC_ARG := $(CC) # Expand CC and CXX here already because we want @@ -497,6 +499,8 @@ else ifneq ($(XC_HOST),) XC_HOST := $(ARCH)$(shell echo $(XC_HOST) | sed "s/[^-]*\(.*\)$$/\1/") MARCH := $(subst _,-,$(ARCH)) +else # insert ARCH into HOST +XC_HOST := $(ARCH)$(shell echo $(BUILD_MACHINE) | sed "s/[^-]*\(.*\)$$/\1/") endif endif @@ -528,6 +532,18 @@ ARCH := $(BUILD_OS) endif endif +# Detect common pre-SSE2 JULIA_CPU_TARGET values known not to work (#7185) +ifeq ($(MARCH),) +ifneq ($(findstring $(ARCH),i386 i486 i586 i686),) +MARCH := pentium4 +endif +endif + +ifneq ($(findstring $(MARCH),i386 i486 i586 i686 pentium pentium2 pentium3),) +$(error Pre-SSE2 CPU targets not supported. To create a generic 32-bit x86 binary, \ +pass 'MARCH=pentium4'.) +endif + ifneq ($(MARCH),) CC += -march=$(MARCH) CXX += -march=$(MARCH) @@ -551,12 +567,6 @@ endif JULIA_CPU_TARGET ?= native -# Detect common pre-SSE2 JULIA_CPU_TARGET values known not to work (#7185) -ifneq ($(findstring $(JULIA_CPU_TARGET),i386 i486 i586 i686 pentium pentium2 pentium3),) -$(error Pre-SSE2 CPU targets not supported. To create a generic 32-bit x86 binary, \ -pass 'MARCH=i686 JULIA_CPU_TARGET=pentium4', or 'MARCH=pentium4' if not building any dependencies.) -endif - # We map amd64 to x86_64 for compatibility with systems that identify 64-bit systems as such ifeq ($(ARCH),amd64) override ARCH := x86_64 diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index 64db00243f3df..87cc53c4c22e3 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -62,8 +62,7 @@ else bits=32 archsuffix=86 exc=sjlj - echo "override MARCH = i686" >> Make.user - echo "override JULIA_CPU_TARGET = pentium4" >> Make.user + echo "override MARCH = pentium4" >> Make.user echo 'LIBBLAS = -L$(JULIAHOME)/usr/bin -lopenblas' >> Make.user echo 'LIBBLASNAME = libopenblas' >> Make.user fi diff --git a/src/codegen.cpp b/src/codegen.cpp index e3a99d67f2a80..3e88bdfc56bf0 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6014,8 +6014,8 @@ extern "C" void jl_init_codegen(void) .setMCJITMemoryManager(std::move(std::unique_ptr{new SectionMemoryManager()})) #endif .setTargetOptions(options) - .setRelocationModel(Reloc::PIC_) - .setCodeModel(CodeModel::Small) + .setRelocationModel(Reloc::Default) + .setCodeModel(CodeModel::JITDefault) #ifdef DISABLE_OPT .setOptLevel(CodeGenOpt::None) #else From eaf553297a662b5b62090821300cdde7f8e9b6fe Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Wed, 7 Oct 2015 12:01:27 -0400 Subject: [PATCH 0470/1938] Fix BigInt parser. Add test for readdlm(BigInt) --- base/gmp.jl | 2 +- test/readdlm.jl | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/base/gmp.jl b/base/gmp.jl index 7e965fc986061..453110bfd5c4d 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -85,7 +85,7 @@ function tryparse_internal(::Type{BigInt}, s::AbstractString, startpos::Int, end _n = Nullable{BigInt}() # don't make a copy in the common case where we are parsing a whole bytestring - bstr = startpos == start(s) && endpos == endof(s) ? bytestring(s) : bytestring(SubString(s,i,endpos)) + bstr = startpos == start(s) && endpos == endof(s) ? bytestring(s) : bytestring(SubString(s,startpos,endpos)) sgn, base, i = Base.parseint_preamble(true,base,bstr,start(bstr),endof(bstr)) if i == 0 diff --git a/test/readdlm.jl b/test/readdlm.jl index 2957910792b42..3e0674f664b2e 100644 --- a/test/readdlm.jl +++ b/test/readdlm.jl @@ -228,3 +228,8 @@ end # fix #13179 parsing unicode lines with default delmiters @test isequaldlm(readdlm(IOBuffer("# Should ignore this π\n1\tα\n2\tβ\n")), Any[1 "α"; 2 "β"], Any) + +# BigInt parser +let data = "1 2 3" + readdlm(IOBuffer(data), ' ', BigInt) == BigInt[1 2 3] +end From 407b00a98171fc77d6cb43d10cb1ea4d8be6030e Mon Sep 17 00:00:00 2001 From: Kiran Pamnany Date: Wed, 7 Oct 2015 09:13:16 -0700 Subject: [PATCH 0471/1938] Added credits. --- test/perf/kernel/stockcorr.m | 1 + test/perf/threads/stockcorr/stockcorr_devec.m | 1 + test/perf/threads/stockcorr/stockcorr_vec.m | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/perf/kernel/stockcorr.m b/test/perf/kernel/stockcorr.m index e5df602c570c2..687402b401f3e 100644 --- a/test/perf/kernel/stockcorr.m +++ b/test/perf/kernel/stockcorr.m @@ -1,4 +1,5 @@ %ORIGINAL_CORR - The original, unoptimised code that simulates two correlated assets +%Mike Croucher (http://www.walkingrandomly.com/?p=3604) %% Correlated asset information CurrentPrice = [78 102]; %Initial Prices of the two stocks diff --git a/test/perf/threads/stockcorr/stockcorr_devec.m b/test/perf/threads/stockcorr/stockcorr_devec.m index 39987ea28c356..fd8aa550f0654 100644 --- a/test/perf/threads/stockcorr/stockcorr_devec.m +++ b/test/perf/threads/stockcorr/stockcorr_devec.m @@ -1,4 +1,5 @@ %% Simulate two correlated assets +%% Original code from Mike Croucher (http://www.walkingrandomly.com/?p=3604) %% Manually hoisted some computations outside the loop; see ../../kernel/ %% for the original version diff --git a/test/perf/threads/stockcorr/stockcorr_vec.m b/test/perf/threads/stockcorr/stockcorr_vec.m index dce3c8159aa7b..7264221880d74 100644 --- a/test/perf/threads/stockcorr/stockcorr_vec.m +++ b/test/perf/threads/stockcorr/stockcorr_vec.m @@ -1,5 +1,5 @@ %% Simulate two correlated assets -%% Vectorized version (from http://www.walkingrandomly.com/?p=3604) +%% Original code from Mike Croucher (http://www.walkingrandomly.com/?p=3604) %% Correlated asset information CurrentPrice = [78 102]; %Initial Prices of the two stocks From 7a6a9e0f275225d4e99d7007fd876e37c45a3577 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 23 Jul 2015 01:44:52 -0400 Subject: [PATCH 0472/1938] WIP: start implementation of generic versions of the julia intrinsic functions --- base/pointer.jl | 4 +- src/APInt-C.cpp | 486 ++++++++++++++++++++ src/APInt-C.h | 78 ++++ src/Makefile | 2 +- src/alloc.c | 40 -- src/codegen.cpp | 1 + src/intrinsics.cpp | 231 +++++++--- src/julia_internal.h | 92 ++++ src/runtime_intrinsics.c | 930 +++++++++++++++++++++++++++++++++++++++ 9 files changed, 1764 insertions(+), 100 deletions(-) create mode 100644 src/APInt-C.cpp create mode 100644 src/APInt-C.h create mode 100644 src/runtime_intrinsics.c diff --git a/base/pointer.jl b/base/pointer.jl index 1956ef2263b66..c532b3e3053e3 100644 --- a/base/pointer.jl +++ b/base/pointer.jl @@ -5,7 +5,7 @@ const C_NULL = box(Ptr{Void}, 0) # pointer to integer -convert{T<:Union{Int,UInt}}(::Type{T}, x::Ptr) = box(T, unbox(Ptr,x)) +convert{T<:Union{Int,UInt}}(::Type{T}, x::Ptr) = box(T, unbox(Ptr{Void},x)) convert{T<:Integer}(::Type{T}, x::Ptr) = convert(T,convert(UInt, x)) # integer to pointer @@ -14,7 +14,7 @@ convert{T}(::Type{Ptr{T}}, x::Int) = box(Ptr{T},unbox(Int,Int(x))) # pointer to pointer convert{T}(::Type{Ptr{T}}, p::Ptr{T}) = p -convert{T}(::Type{Ptr{T}}, p::Ptr) = box(Ptr{T}, unbox(Ptr,p)) +convert{T}(::Type{Ptr{T}}, p::Ptr) = box(Ptr{T}, unbox(Ptr{Void},p)) # object to pointer (when used with ccall) unsafe_convert(::Type{Ptr{UInt8}}, x::Symbol) = ccall(:jl_symbol_name, Ptr{UInt8}, (Any,), x) diff --git a/src/APInt-C.cpp b/src/APInt-C.cpp new file mode 100644 index 0000000000000..16a4c3059493e --- /dev/null +++ b/src/APInt-C.cpp @@ -0,0 +1,486 @@ +#include "llvm-version.h" +#include +#include +#include + +#define DLLEXPORT +extern "C" DLLEXPORT void jl_error(const char *str); + +using namespace llvm; + +/* create "APInt s" from "integerPart *ps" */ +#define CREATE(s) \ + APInt s; \ + if ((numbits % integerPartWidth) != 0) { \ + /* use LLT_ALIGN to round the memory area up to the nearest integerPart-sized chunk */ \ + unsigned nbytes = RoundUpToAlignment(numbits, integerPartWidth) / host_char_bit; \ + integerPart *data_a64 = (integerPart*)alloca(nbytes); \ + /* TODO: this memcpy assumes little-endian, + * for big-endian, need to align the copy to the other end */ \ + memcpy(data_a64, p##s, RoundUpToAlignment(numbits, host_char_bit) / host_char_bit); \ + s = APInt(numbits, makeArrayRef(data_a64, nbytes / sizeof(integerPart))); \ + } \ + else { \ + s = APInt(numbits, makeArrayRef(p##s, numbits / integerPartWidth)); \ + } + +/* assign to "integerPart *pr" from "APInt a" */ +#define ASSIGN(r, a) \ + if (numbits <= 8) \ + *(uint8_t*)p##r = a.getZExtValue(); \ + else if (numbits <= 16) \ + *(uint16_t*)p##r = a.getZExtValue(); \ + else if (numbits <= 32) \ + *(uint32_t*)p##r = a.getZExtValue(); \ + else if (numbits <= 64) \ + *(uint64_t*)p##r = a.getZExtValue(); \ + else \ + memcpy(p##r, a.getRawData(), RoundUpToAlignment(numbits, host_char_bit) / host_char_bit); \ + +extern "C" DLLEXPORT +void LLVMNeg(unsigned numbits, integerPart *pa, integerPart *pr) { + APInt z(numbits, 0); + CREATE(a) + z -= a; + ASSIGN(r, z) +} + +extern "C" DLLEXPORT +void LLVMAdd(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + a += b; + ASSIGN(r, a) +} + +extern "C" DLLEXPORT +void LLVMSub(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + a -= b; + ASSIGN(r, a) +} + +extern "C" DLLEXPORT +void LLVMMul(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + a *= b; + ASSIGN(r, a) +} + +extern "C" DLLEXPORT +void LLVMSDiv(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + a = a.sdiv(b); + ASSIGN(r, a) +} + +extern "C" DLLEXPORT +void LLVMUDiv(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + a = a.udiv(b); + ASSIGN(r, a) +} + +extern "C" DLLEXPORT +void LLVMSRem(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + a = a.srem(b); + ASSIGN(r, a) +} + +extern "C" DLLEXPORT +void LLVMURem(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + a = a.urem(b); + ASSIGN(r, a) +} + +extern "C" DLLEXPORT +int LLVMICmpEQ(unsigned numbits, integerPart *pa, integerPart *pb) { + CREATE(a) + CREATE(b) + return a.eq(b); +} + +extern "C" DLLEXPORT +int LLVMICmpNE(unsigned numbits, integerPart *pa, integerPart *pb) { + CREATE(a) + CREATE(b) + return a.ne(b); +} + +extern "C" DLLEXPORT +int LLVMICmpSLT(unsigned numbits, integerPart *pa, integerPart *pb) { + CREATE(a) + CREATE(b) + return a.slt(b); +} + +extern "C" DLLEXPORT +int LLVMICmpULT(unsigned numbits, integerPart *pa, integerPart *pb) { + CREATE(a) + CREATE(b) + return a.ult(b); +} + +extern "C" DLLEXPORT +int LLVMICmpSLE(unsigned numbits, integerPart *pa, integerPart *pb) { + CREATE(a) + CREATE(b) + return a.sle(b); +} + +extern "C" DLLEXPORT +int LLVMICmpULE(unsigned numbits, integerPart *pa, integerPart *pb) { + CREATE(a) + CREATE(b) + return a.ule(b); +} + +extern "C" DLLEXPORT +void LLVMAnd(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + a &= b; + ASSIGN(r, a) +} + +extern "C" DLLEXPORT +void LLVMOr(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + a |= b; + ASSIGN(r, a) +} + +extern "C" DLLEXPORT +void LLVMXor(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + a ^= b; + ASSIGN(r, a) +} + +extern "C" DLLEXPORT +void LLVMShl(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + a = a.shl(b); + ASSIGN(r, a) +} + +extern "C" DLLEXPORT +void LLVMLShr(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + a = a.lshr(b); + ASSIGN(r, a) +} +extern "C" DLLEXPORT +void LLVMAShr(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + a = a.ashr(b); + ASSIGN(r, a) +} + +extern "C" DLLEXPORT +void LLVMFlipAllBits(unsigned numbits, integerPart *pa, integerPart *pr) { + CREATE(a) + a.flipAllBits(); + ASSIGN(r, a) +} + +extern "C" DLLEXPORT +int LLVMAdd_uov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + bool Overflow; + a = a.uadd_ov(b, Overflow); + ASSIGN(r, a) + return Overflow; +} + +extern "C" DLLEXPORT +int LLVMAdd_sov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + bool Overflow; + a = a.sadd_ov(b, Overflow); + ASSIGN(r, a) + return Overflow; +} + +extern "C" DLLEXPORT +int LLVMSub_uov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + bool Overflow; + a = a.usub_ov(b, Overflow); + ASSIGN(r, a) + return Overflow; +} + +extern "C" DLLEXPORT +int LLVMSub_sov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + bool Overflow; + a = a.ssub_ov(b, Overflow); + ASSIGN(r, a) + return Overflow; +} + +extern "C" DLLEXPORT +int LLVMMul_sov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + bool Overflow; + a = a.smul_ov(b, Overflow); + ASSIGN(r, a) + return Overflow; +} + +extern "C" DLLEXPORT +int LLVMMul_uov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + bool Overflow; + a = a.umul_ov(b, Overflow); + ASSIGN(r, a) + return Overflow; +} + +extern "C" DLLEXPORT +void LLVMByteSwap(unsigned numbits, integerPart *pa, integerPart *pr) { + CREATE(a) + a = a.byteSwap(); + ASSIGN(r, a) +} + +void LLVMFPtoInt(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr, bool isSigned, bool *isExact) { + double Val; + if (numbits == 32) + Val = *(float*)pa; + else if (numbits == 64) + Val = *(double*)pa; + else + jl_error("FPtoSI: runtime floating point intrinsics are not implemented for bit sizes other than 32 and 64"); + unsigned onumbytes = RoundUpToAlignment(onumbits, host_char_bit) / host_char_bit; + if (onumbits <= 64) { // fast-path, if possible + if (isSigned) { + int64_t ia = Val; + memcpy(pr, &ia, onumbytes); // TODO: assumes little-endian + if (isExact) { + // check whether the conversion was lossless + int64_t ia2 = ia < 0 ? -1 : 0; + memcpy(&ia2, pr, onumbytes); + *isExact = (Val == (double)ia2 && ia == ia2); + } + } + else { + uint64_t ia = Val; + memcpy(pr, &ia, onumbytes); // TODO: assumes little-endian + if (isExact) { + // check whether the conversion was lossless + uint64_t ia2 = 0; + memcpy(&ia2, pr, onumbytes); + *isExact = (Val == (double)ia2 && ia == ia2); + } + } + } + else { + APFloat a(Val); + bool isVeryExact; + APFloat::roundingMode rounding_mode = APFloat::rmNearestTiesToEven; + unsigned nbytes = RoundUpToAlignment(onumbits, integerPartWidth) / host_char_bit; + integerPart *parts = (integerPart*)alloca(nbytes); + APFloat::opStatus status = a.convertToInteger(parts, onumbits, isSigned, rounding_mode, &isVeryExact); + memcpy(pr, parts, onumbytes); + if (isExact) + *isExact = (status == APFloat::opOK); + } +} + +extern "C" DLLEXPORT +void LLVMFPtoSI(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr) { + LLVMFPtoInt(numbits, pa, onumbits, pr, true, NULL); +} + +extern "C" DLLEXPORT +void LLVMFPtoUI(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr) { + LLVMFPtoInt(numbits, pa, onumbits, pr, false, NULL); +} + +extern "C" DLLEXPORT +int LLVMFPtoSI_exact(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr) { + bool isExact; + LLVMFPtoInt(numbits, pa, onumbits, pr, true, &isExact); + return isExact; +} + +extern "C" DLLEXPORT +int LLVMFPtoUI_exact(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr) { + bool isExact; + LLVMFPtoInt(numbits, pa, onumbits, pr, false, &isExact); + return isExact; +} + +extern "C" DLLEXPORT +void LLVMSItoFP(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr) { + CREATE(a) + double val = a.roundToDouble(true); + if (onumbits == 32) + *(float*)pr = val; + else if (onumbits == 64) + *(double*)pr = val; + else + jl_error("SItoFP: runtime floating point intrinsics are not implemented for bit sizes other than 32 and 64"); +} + +extern "C" DLLEXPORT +void LLVMUItoFP(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr) { + CREATE(a) + double val = a.roundToDouble(false); + if (onumbits == 32) + *(float*)pr = val; + else if (onumbits == 64) + *(double*)pr = val; + else + jl_error("UItoFP: runtime floating point intrinsics are not implemented for bit sizes other than 32 and 64"); +} + +extern "C" DLLEXPORT +void LLVMSExt(unsigned inumbits, integerPart *pa, unsigned onumbits, integerPart *pr) { + assert(inumbits < onumbits); + unsigned inumbytes = RoundUpToAlignment(inumbits, host_char_bit) / host_char_bit; + unsigned onumbytes = RoundUpToAlignment(onumbits, host_char_bit) / host_char_bit; + int bits = (0 - inumbits) % host_char_bit; + int signbit = (inumbits - 1) % host_char_bit; + int sign = ((unsigned char*)pa)[inumbytes - 1] & (1 << signbit) ? -1 : 0; + // copy over the input bytes + memcpy(pr, pa, inumbytes); + if (bits) { + // sign-extend the partial byte + ((signed char*)pr)[inumbytes - 1] = ((signed char*)pa)[inumbytes - 1] << bits >> bits; + } + // sign-extend the rest of the bytes + memset((char*)pr + inumbytes, sign, onumbytes - inumbytes); +} + +extern "C" DLLEXPORT +void LLVMZExt(unsigned inumbits, integerPart *pa, unsigned onumbits, integerPart *pr) { + assert(inumbits < onumbits); + unsigned inumbytes = RoundUpToAlignment(inumbits, host_char_bit) / host_char_bit; + unsigned onumbytes = RoundUpToAlignment(onumbits, host_char_bit) / host_char_bit; + int bits = (0 - inumbits) % host_char_bit; + // copy over the input bytes + memcpy(pr, pa, inumbytes); + if (bits) { + // zero the remaining bits of the partial byte + ((unsigned char*)pr)[inumbytes - 1] = ((unsigned char*)pa)[inumbytes - 1] << bits >> bits; + } + // zero-extend the rest of the bytes + memset((char*)pr + inumbytes, 0, onumbytes - inumbytes); +} + +extern "C" DLLEXPORT +void LLVMTrunc(unsigned inumbits, integerPart *pa, unsigned onumbits, integerPart *pr) { + assert(inumbits > onumbits); + unsigned onumbytes = RoundUpToAlignment(onumbits, host_char_bit) / host_char_bit; + memcpy(pr, pa, onumbytes); +} + +extern "C" DLLEXPORT +unsigned countTrailingZeros_8(uint8_t Val) { +#ifdef LLVM35 + return countTrailingZeros(Val); +#else + return CountTrailingZeros_32(Val); +#endif +} + +extern "C" DLLEXPORT +unsigned countTrailingZeros_16(uint16_t Val) { +#ifdef LLVM35 + return countTrailingZeros(Val); +#else + return CountTrailingZeros_32(Val); +#endif +} + +extern "C" DLLEXPORT +unsigned countTrailingZeros_32(uint32_t Val) { +#ifdef LLVM35 + return countTrailingZeros(Val); +#else + return CountTrailingZeros_32(Val); +#endif +} + +extern "C" DLLEXPORT +unsigned countTrailingZeros_64(uint64_t Val) { +#ifdef LLVM35 + return countTrailingZeros(Val); +#else + return CountTrailingZeros_64(Val); +#endif +} + +extern "C" DLLEXPORT +void jl_LLVMSMod(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + CREATE(a) + CREATE(b) + APInt r = a.srem(b); + if (a.isNegative() != b.isNegative()) { + r = (b + r).srem(b); + } + ASSIGN(r, r) +} + +extern "C" DLLEXPORT +void jl_LLVMFlipSign(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr) { + unsigned numbytes = RoundUpToAlignment(numbits, host_char_bit) / host_char_bit; + int signbit = (numbits - 1) % host_char_bit; + int sign = ((unsigned char*)pa)[numbytes - 1] & (1 << signbit) ? -1 : 0; + if (sign) + LLVMNeg(numbits, pa, pr); + else + memcpy(pr, pa, numbytes); +} + +extern "C" DLLEXPORT +unsigned LLVMCountPopulation(unsigned numbits, integerPart *pa) { + CREATE(a) + return a.countPopulation(); +} + +extern "C" DLLEXPORT +unsigned LLVMCountTrailingOnes(unsigned numbits, integerPart *pa) { + CREATE(a) + return a.countTrailingOnes(); +} + +extern "C" DLLEXPORT +unsigned LLVMCountTrailingZeros(unsigned numbits, integerPart *pa) { + CREATE(a) + return a.countTrailingZeros(); +} + +extern "C" DLLEXPORT +unsigned LLVMCountLeadingOnes(unsigned numbits, integerPart *pa) { + CREATE(a) + return a.countLeadingOnes(); +} + +extern "C" DLLEXPORT +unsigned LLVMCountLeadingZeros(unsigned numbits, integerPart *pa) { + CREATE(a) + return a.countLeadingZeros(); +} diff --git a/src/APInt-C.h b/src/APInt-C.h new file mode 100644 index 0000000000000..da7c08652d2a8 --- /dev/null +++ b/src/APInt-C.h @@ -0,0 +1,78 @@ + +#ifndef APINT_C_H +#define APINT_C_H + +#ifdef __cplusplus +extern "C" { +#endif +typedef void integerPart; + +void LLVMNeg(unsigned numbits, integerPart *pa, integerPart *pr); +void LLVMByteSwap(unsigned numbits, integerPart *pa, integerPart *pr); + +void LLVMAdd(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +void LLVMSub(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +void LLVMMul(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +void LLVMSDiv(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +void LLVMUDiv(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +void LLVMSRem(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +void LLVMURem(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); + +void LLVMAnd(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +void LLVMOr(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +void LLVMXor(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +void LLVMShl(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +void LLVMLShr(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +void LLVMAShr(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +void LLVMFlipAllBits(unsigned numbits, integerPart *pa, integerPart *pr); + +int LLVMICmpEQ(unsigned numbits, integerPart *pa, integerPart *pr); +int LLVMICmpNE(unsigned numbits, integerPart *pa, integerPart *pb); +int LLVMICmpSLT(unsigned numbits, integerPart *pa, integerPart *pb); +int LLVMICmpULT(unsigned numbits, integerPart *pa, integerPart *pb); +int LLVMICmpSLE(unsigned numbits, integerPart *pa, integerPart *pb); +int LLVMICmpULE(unsigned numbits, integerPart *pa, integerPart *pb); + +int LLVMAdd_uov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +int LLVMAdd_sov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +int LLVMSub_uov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +int LLVMSub_sov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +int LLVMMul_sov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +int LLVMMul_uov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); + +unsigned LLVMCountPopulation(unsigned numbits, integerPart *pa); +unsigned LLVMCountTrailingOnes(unsigned numbits, integerPart *pa); +unsigned LLVMCountTrailingZeros(unsigned numbits, integerPart *pa); +unsigned LLVMCountLeadingOnes(unsigned numbits, integerPart *pa); +unsigned LLVMCountLeadingZeros(unsigned numbits, integerPart *pa); + +void LLVMFPtoSI(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); +void LLVMFPtoUI(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); +void LLVMSItoFP(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); +void LLVMUItoFP(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); +void LLVMSExt(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); +void LLVMZExt(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); +void LLVMTrunc(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); + +int LLVMFPtoSI_exact(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); +int LLVMFPtoUI_exact(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); + +void jl_LLVMSMod(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +void jl_LLVMFlipSign(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); + +unsigned countTrailingZeros_8(uint8_t Val); +unsigned countTrailingZeros_16(uint16_t Val); +unsigned countTrailingZeros_32(uint32_t Val); +unsigned countTrailingZeros_64(uint64_t Val); + +uint8_t getSwappedBytes_8(uint8_t Value); // no-op +uint16_t getSwappedBytes_16(uint16_t Value); +uint32_t getSwappedBytes_32(uint32_t Value); +uint64_t getSwappedBytes_64(uint64_t Value); + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/Makefile b/src/Makefile index f403327b5fc46..2731b897445e2 100644 --- a/src/Makefile +++ b/src/Makefile @@ -14,7 +14,7 @@ override CPPFLAGS += $(JCPPFLAGS) SRCS := \ jltypes gf ast builtins module codegen disasm debuginfo interpreter \ alloc dlload sys init task array dump toplevel jl_uv jlapi signal-handling \ - llvm-simdloop simplevector + llvm-simdloop simplevector APInt-C runtime_intrinsics ifeq ($(JULIAGC),MARKSWEEP) SRCS += gc endif diff --git a/src/alloc.c b/src/alloc.c index ab453d5a8e264..52be1ed3b2a2b 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -159,25 +159,6 @@ jl_value_t *jl_new_bits(jl_value_t *bt, void *data) return jl_new_bits_internal(bt, data, &len); } -// run time version of pointerref intrinsic (warning: i is not rooted) -DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i) -{ - JL_TYPECHK(pointerref, pointer, p); - JL_TYPECHK(pointerref, long, i); - jl_value_t *ety = jl_tparam0(jl_typeof(p)); - if (ety == (jl_value_t*)jl_any_type) { - jl_value_t **pp = (jl_value_t**)(jl_unbox_long(p) + (jl_unbox_long(i)-1)*sizeof(void*)); - return *pp; - } - else { - if (!jl_is_datatype(ety)) - jl_error("pointerref: invalid pointer"); - size_t nb = LLT_ALIGN(jl_datatype_size(ety), ((jl_datatype_t*)ety)->alignment); - char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb; - return jl_new_bits(ety, pp); - } -} - void jl_assign_bits(void *dest, jl_value_t *bits) { size_t nb = jl_datatype_size(jl_typeof(bits)); @@ -192,27 +173,6 @@ void jl_assign_bits(void *dest, jl_value_t *bits) } } -// run time version of pointerset intrinsic (warning: x is not gc-rooted) -DLLEXPORT void jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t *i) -{ - JL_TYPECHK(pointerset, pointer, p); - JL_TYPECHK(pointerset, long, i); - jl_value_t *ety = jl_tparam0(jl_typeof(p)); - if (ety == (jl_value_t*)jl_any_type) { - jl_value_t **pp = (jl_value_t**)(jl_unbox_long(p) + (jl_unbox_long(i)-1)*sizeof(void*)); - *pp = x; - } - else { - if (!jl_is_datatype(ety)) - jl_error("pointerset: invalid pointer"); - size_t nb = LLT_ALIGN(jl_datatype_size(ety), ((jl_datatype_t*)ety)->alignment); - char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb; - if (jl_typeof(x) != ety) - jl_error("pointerset: type mismatch in assign"); - jl_assign_bits(pp, x); - } -} - int jl_field_index(jl_datatype_t *t, jl_sym_t *fld, int err) { jl_svec_t *fn = t->name->names; diff --git a/src/codegen.cpp b/src/codegen.cpp index 3e88bdfc56bf0..d88be7d4be5ed 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6121,6 +6121,7 @@ extern "C" void jl_init_codegen(void) "jl_box32", (void*)&jl_box32, m); box64_func = boxfunc_llvm(ft2arg(T_pjlvalue, T_pjlvalue, T_int64), "jl_box64", (void*)&jl_box64, m); + jl_init_intrinsic_functions_codegen(m); } // for debugging from gdb diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index a1bcb335872f0..d8db10bfe7288 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -43,16 +43,21 @@ namespace JL_I { // pointer access pointerref, pointerset, // c interface - ccall, cglobal, llvmcall + ccall, cglobal, llvmcall, + // terminator + fptoui_auto, fptosi_auto, + num_intrinsics }; } using namespace JL_I; +Function *runtime_func[num_intrinsics]; +unsigned intrinsic_nargs[num_intrinsics]; #include "ccall.cpp" /* - low-level intrinsics design: + low-level intrinsics design: TODO: fix description below functions like add_int expect unboxed values of matching bit-length. every operation that can return an unboxed value does so. this maximizes opportunities for composing functions without @@ -107,12 +112,22 @@ static Type *JL_INTT(Type *t) assert(t == T_void); return T_void; } -static jl_value_t *JL_JLINTT(Type *t) +// convert float type to same-size int type (as a Julia type) +static jl_value_t *JL_JLUINTT(Type *t) { assert(!t->isIntegerTy()); - if (t == T_float32) return (jl_value_t*)jl_float32_type; - if (t == T_float64) return (jl_value_t*)jl_float64_type; - if (t == Type::getHalfTy(jl_LLVMContext)) return jl_get_global(jl_base_module, jl_symbol("Float16")); + if (t == T_float32) return (jl_value_t*)jl_uint32_type; + if (t == T_float64) return (jl_value_t*)jl_uint64_type; + if (t == Type::getHalfTy(jl_LLVMContext)) return (jl_value_t*)jl_uint16_type; + assert(t == T_void); + return jl_bottom_type; +} +static jl_value_t *JL_JLSINTT(Type *t) +{ + assert(!t->isIntegerTy()); + if (t == T_float32) return (jl_value_t*)jl_int32_type; + if (t == T_float64) return (jl_value_t*)jl_int64_type; + if (t == Type::getHalfTy(jl_LLVMContext)) return (jl_value_t*)jl_int16_type; assert(t == T_void); return jl_bottom_type; } @@ -413,10 +428,7 @@ static jl_cgval_t generic_box(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ctx } else { if (!jl_is_leaf_type(v.typ) && !jl_is_bitstype(v.typ)) { - // TODO: currently doesn't handle the case where the type of neither argument is understood at compile time - // since codegen has no idea what size it might have - jl_error("codegen: failed during evaluation of a call to reinterpret"); - return jl_cgval_t(); + return jl_cgval_t(); // TODO: XXX } nb = jl_datatype_size(v.typ); llvmt = staticeval_bitstype(v.typ); @@ -820,25 +832,59 @@ static Value *emit_smod(Value *x, Value *den, jl_codectx_t *ctx) } #define HANDLE(intr,n) \ - case intr: if (nargs!=n) jl_error(#intr": wrong number of arguments"); + case intr: static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, size_t nargs, jl_codectx_t *ctx, jl_datatype_t* *newtyp); static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, jl_codectx_t *ctx) { + assert(f < num_intrinsics); + if (f == fptoui && nargs == 1) + f = fptoui_auto; + if (f == fptosi && nargs == 1) + f = fptosi_auto; + unsigned expected_nargs = intrinsic_nargs[f]; + if (expected_nargs && expected_nargs != nargs) { + jl_errorf("intrinsic #%d: wrong number of arguments", f); + } + switch (f) { case ccall: return emit_ccall(args, nargs, ctx); case cglobal: return emit_cglobal(args, nargs, ctx); case llvmcall: return emit_llvmcall(args, nargs, ctx); - - HANDLE(pointerref,2) +#if 0 + default: + int ldepth = ctx->gc.argDepth; + Value *r; + if (nargs == 1) { + Value *x = emit_boxed_rooted(args[1], ctx).V; + r = builder.CreateCall(runtime_func[f], x); + } + else if (nargs == 2) { + Value *x = emit_boxed_rooted(args[1], ctx).V; + Value *y = emit_boxed_rooted(args[2], ctx).V; + r = builder.CreateCall2(runtime_func[f], x, y); + } + else if (nargs == 3) { + Value *x = emit_boxed_rooted(args[1], ctx).V; + Value *y = emit_boxed_rooted(args[2], ctx).V; + Value *z = emit_boxed_rooted(args[3], ctx).V; + r = builder.CreateCall3(runtime_func[f], x, y, z); + } + else { + assert(0); + } + ctx->gc.argDepth = ldepth; + return mark_julia_type(r, true, (jl_value_t*)jl_any_type); +#else + case pointerref: return emit_pointerref(args[1], args[2], ctx); - HANDLE(pointerset,3) + case pointerset: return emit_pointerset(args[1], args[2], args[3], ctx); - HANDLE(box,2) + case box: return generic_box(args[1], args[2], ctx); - HANDLE(unbox,2) // TODO: deprecate this + case unbox: return generic_box(args[1], args[2], ctx); HANDLE(trunc_int,2) return generic_trunc(args[1], args[2], ctx, false, false); @@ -873,46 +919,38 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, return mark_julia_type(builder.CreateSIToFP(xi, FTnbits(nb)), false, bt); } - case fptoui: - if (nargs == 1) { - Value *x = FP(auto_unbox(args[1], ctx)); - if (x->getType() == T_void) return jl_cgval_t(); // auto_unbox threw an error - return mark_julia_type( - builder.CreateFPToUI(FP(x), JL_INTT(x->getType())), - false, - JL_JLINTT(x->getType())); - } - else if (nargs == 2) { - jl_value_t *bt = staticeval_bitstype(args[1], "sitofp", ctx); - if (!bt) return jl_cgval_t(); - int nb = get_bitstype_nbits(bt); - Value *xf = FP(auto_unbox(args[2],ctx)); - if (xf->getType() == T_void) return jl_cgval_t(); // auto_unbox threw an error - return mark_julia_type(builder.CreateFPToUI(xf, Type::getIntNTy(jl_LLVMContext, nb)), false, bt); - } - else { - jl_error("fptoui: wrong number of arguments"); - } + case fptoui_auto: { + Value *x = FP(auto_unbox(args[1], ctx)); + if (x->getType() == T_void) return jl_cgval_t(); // auto_unbox threw an error + return mark_julia_type( + builder.CreateFPToUI(FP(x), JL_INTT(x->getType())), + false, + JL_JLUINTT(x->getType())); + } + case fptoui: { + jl_value_t *bt = staticeval_bitstype(args[1], "sitofp", ctx); + if (!bt) return jl_cgval_t(); + int nb = get_bitstype_nbits(bt); + Value *xf = FP(auto_unbox(args[2],ctx)); + if (xf->getType() == T_void) return jl_cgval_t(); // auto_unbox threw an error + return mark_julia_type(builder.CreateFPToUI(xf, Type::getIntNTy(jl_LLVMContext, nb)), false, bt); + } - case fptosi: - if (nargs == 1) { - Value *x = FP(auto_unbox(args[1], ctx)); - return mark_julia_type( - builder.CreateFPToSI(FP(x), JL_INTT(x->getType())), - false, - JL_JLINTT(x->getType())); - } - else if (nargs == 2) { - jl_value_t *bt = staticeval_bitstype(args[1], "sitofp", ctx); - if (!bt) return jl_cgval_t(); - int nb = get_bitstype_nbits(bt); - Value *xf = FP(auto_unbox(args[2],ctx)); - if (xf->getType() == T_void) return jl_cgval_t(); // auto_unbox threw an error - return mark_julia_type(builder.CreateFPToSI(xf, Type::getIntNTy(jl_LLVMContext, nb)), false, bt); - } - else { - jl_error("fptosi: wrong number of arguments"); - } + case fptosi_auto: { + Value *x = FP(auto_unbox(args[1], ctx)); + return mark_julia_type( + builder.CreateFPToSI(FP(x), JL_INTT(x->getType())), + false, + JL_JLSINTT(x->getType())); + } + case fptosi: { + jl_value_t *bt = staticeval_bitstype(args[1], "sitofp", ctx); + if (!bt) return jl_cgval_t(); + int nb = get_bitstype_nbits(bt); + Value *xf = FP(auto_unbox(args[2],ctx)); + if (xf->getType() == T_void) return jl_cgval_t(); // auto_unbox threw an error + return mark_julia_type(builder.CreateFPToSI(xf, Type::getIntNTy(jl_LLVMContext, nb)), false, bt); + } HANDLE(fptrunc,2) { jl_value_t *bt = staticeval_bitstype(args[1], "sitofp", ctx); @@ -1016,7 +1054,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, r = builder.CreateBitCast(r, x->getType()); return mark_julia_type(r, false, newtyp ? newtyp : xinfo.typ); } - +#endif } assert(0); } @@ -1385,7 +1423,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, HANDLE(powi_llvm,2) { x = FP(x); y = JL_INT(y); - Type *tx = x->getType(); + Type *tx = x->getType(); // TODO: LLVM expects this to be i32 #ifdef LLVM36 Type *ts[1] = { tx }; Value *powi = Intrinsic::getDeclaration(jl_Module, Intrinsic::powi, @@ -1409,6 +1447,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, ArrayRef(x->getType())), x); } + default: assert(false); } @@ -1464,6 +1503,7 @@ extern "C" void jl_init_intrinsic_functions(void) jl_module_t *inm = jl_new_module(jl_symbol("Intrinsics")); inm->parent = jl_core_module; jl_set_const(jl_core_module, jl_symbol("Intrinsics"), (jl_value_t*)inm); + ADD_I(box); ADD_I(unbox); ADD_I(neg_int); ADD_I(add_int); ADD_I(sub_int); ADD_I(mul_int); ADD_I(sdiv_int); ADD_I(udiv_int); ADD_I(srem_int); ADD_I(urem_int); @@ -1501,6 +1541,83 @@ extern "C" void jl_init_intrinsic_functions(void) ADD_I(checked_trunc_uint); ADD_I(check_top_bit); ADD_I(nan_dom_err); + //ADD_I(fptosi_auto); ADD_I(fptoui_auto); // these intrinsics are "hidden" in fpto*i ADD_I(ccall); ADD_I(cglobal); ADD_I(llvmcall); } +#undef ADD_I + +static void add_intrinsic_to_codegen(Module *m, const std::string &name, intrinsic f, + unsigned nargs, std::vector args, void *pfunc) { + Function *func = Function::Create(FunctionType::get(T_pjlvalue, args, false), + Function::ExternalLinkage, name, m); + runtime_func[f] = func; + add_named_global(func, pfunc); + intrinsic_nargs[f] = nargs; +} + +static void add_intrinsic_to_codegen(intrinsic alias, intrinsic base) +{ + runtime_func[alias] = runtime_func[base]; + intrinsic_nargs[alias] = intrinsic_nargs[base]; +} + +#define ADD_I(name, nargs) add_intrinsic_to_codegen(m, "jl_" #name, name, nargs, args##nargs, (void*)&jl_##name) +#define ALIAS(alias, base) add_intrinsic_to_codegen(alias, base) + +static void jl_init_intrinsic_functions_codegen(Module *m) +{ + std::vector args1(0); + args1.push_back(T_pjlvalue); + std::vector args2(0); + args2.push_back(T_pjlvalue); + args2.push_back(T_pjlvalue); + std::vector args3(0); + args3.push_back(T_pjlvalue); + args3.push_back(T_pjlvalue); + args3.push_back(T_pjlvalue); + + add_intrinsic_to_codegen(m, "jl_reinterpret", box, + 2, args2, (void*)&jl_reinterpret); + ALIAS(unbox, box); + ADD_I(neg_int, 1); ADD_I(add_int, 2); ADD_I(sub_int, 2); ADD_I(mul_int, 2); + ADD_I(sdiv_int, 2); ADD_I(udiv_int, 2); ADD_I(srem_int, 2); ADD_I(urem_int, 2); + ADD_I(smod_int, 2); + ADD_I(neg_float, 1); ADD_I(add_float, 2); ADD_I(sub_float, 2); ADD_I(mul_float, 2); + ADD_I(div_float, 2); ADD_I(rem_float, 2); ADD_I(fma_float, 3); ADD_I(muladd_float, 3); + ALIAS(neg_float_fast, neg_float); ALIAS(add_float_fast, add_float); ALIAS(sub_float_fast, sub_float); + ALIAS(mul_float_fast, mul_float); ALIAS(div_float_fast, div_float); ALIAS(rem_float_fast, rem_float); + ADD_I(eq_int, 2); ADD_I(ne_int, 2); + ADD_I(slt_int, 2); ADD_I(ult_int, 2); + ADD_I(sle_int, 2); ADD_I(ule_int, 2); + ADD_I(eq_float, 2); ADD_I(ne_float, 2); + ADD_I(lt_float, 2); ADD_I(le_float, 2); + ALIAS(eq_float_fast, eq_float); ALIAS(ne_float_fast, ne_float); + ALIAS(lt_float_fast, lt_float); ALIAS(le_float_fast, le_float); + ADD_I(fpiseq, 2); ADD_I(fpislt, 2); + ADD_I(and_int, 2); ADD_I(or_int, 2); ADD_I(xor_int, 2); ADD_I(not_int, 1); + ADD_I(shl_int, 2); ADD_I(lshr_int, 2); ADD_I(ashr_int, 2); ADD_I(bswap_int, 1); + ADD_I(ctpop_int, 1); ADD_I(ctlz_int, 1); ADD_I(cttz_int, 1); + ADD_I(sext_int, 2); ADD_I(zext_int, 2); ADD_I(trunc_int, 2); + ADD_I(fptoui, 2); ADD_I(fptosi, 2); + ADD_I(uitofp, 2); ADD_I(sitofp, 2); + ADD_I(fptrunc, 2); ADD_I(fpext, 2); + ADD_I(abs_float, 1); ADD_I(copysign_float, 2); + ADD_I(flipsign_int, 2); ADD_I(select_value, 3); + ADD_I(ceil_llvm, 1); ADD_I(floor_llvm, 1); ADD_I(trunc_llvm, 1); ADD_I(rint_llvm, 1); + ADD_I(sqrt_llvm, 1); ADD_I(powi_llvm, 2); + ALIAS(sqrt_llvm_fast, sqrt_llvm); + ADD_I(pointerref, 2); ADD_I(pointerset, 3); + ADD_I(checked_sadd, 2); ADD_I(checked_uadd, 2); + ADD_I(checked_ssub, 2); ADD_I(checked_usub, 2); + ADD_I(checked_smul, 2); ADD_I(checked_umul, 2); + ADD_I(checked_fptosi, 2); ADD_I(checked_fptoui, 2); + ADD_I(checked_trunc_sint, 2); + ADD_I(checked_trunc_uint, 2); + ADD_I(check_top_bit, 1); + ADD_I(nan_dom_err, 2); + ADD_I(fptosi_auto, 1); ADD_I(fptoui_auto, 1); +} + +#undef ADD_I +#undef ALIAS diff --git a/src/julia_internal.h b/src/julia_internal.h index ea14b50079e83..d3798c1964f96 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -224,6 +224,98 @@ DLLEXPORT int jl_fs_rename(const char *src_path, const char *dst_path); extern DLLEXPORT jl_value_t *jl_segv_exception; #endif +// Runtime intrinsics // + +DLLEXPORT jl_value_t *jl_reinterpret(jl_value_t *ty, jl_value_t *v); +DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i); +DLLEXPORT jl_value_t *jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t *i); + +DLLEXPORT jl_value_t *jl_neg_int(jl_value_t *a); +DLLEXPORT jl_value_t *jl_add_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_sub_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_mul_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_sdiv_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_udiv_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_srem_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_urem_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_smod_int(jl_value_t *a, jl_value_t *b); + +DLLEXPORT jl_value_t *jl_neg_float(jl_value_t *a); +DLLEXPORT jl_value_t *jl_add_float(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_sub_float(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_mul_float(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_div_float(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_rem_float(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_fma_float(jl_value_t *a, jl_value_t *b, jl_value_t *c); +DLLEXPORT jl_value_t *jl_muladd_float(jl_value_t *a, jl_value_t *b, jl_value_t *c); + +DLLEXPORT jl_value_t *jl_eq_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_ne_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_slt_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_ult_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_sle_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_ule_int(jl_value_t *a, jl_value_t *b); + +DLLEXPORT jl_value_t *jl_eq_float(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_ne_float(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_lt_float(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_le_float(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_fpiseq(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_fpislt(jl_value_t *a, jl_value_t *b); + +DLLEXPORT jl_value_t *jl_not_int(jl_value_t *a); +DLLEXPORT jl_value_t *jl_and_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_or_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_xor_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_shl_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_lshr_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_ashr_int(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_bswap_int(jl_value_t *a); +DLLEXPORT jl_value_t *jl_ctpop_int(jl_value_t *a); +DLLEXPORT jl_value_t *jl_ctlz_int(jl_value_t *a); +DLLEXPORT jl_value_t *jl_cttz_int(jl_value_t *a); + +DLLEXPORT jl_value_t *jl_sext_int(jl_value_t *ty, jl_value_t *a); +DLLEXPORT jl_value_t *jl_zext_int(jl_value_t *ty, jl_value_t *a); +DLLEXPORT jl_value_t *jl_trunc_int(jl_value_t *ty, jl_value_t *a); +DLLEXPORT jl_value_t *jl_sitofp(jl_value_t *ty, jl_value_t *a); +DLLEXPORT jl_value_t *jl_uitofp(jl_value_t *ty, jl_value_t *a); + +DLLEXPORT jl_value_t *jl_fptoui(jl_value_t *ty, jl_value_t *a); +DLLEXPORT jl_value_t *jl_fptosi(jl_value_t *ty, jl_value_t *a); +DLLEXPORT jl_value_t *jl_fptrunc(jl_value_t *ty, jl_value_t *a); +DLLEXPORT jl_value_t *jl_fpext(jl_value_t *ty, jl_value_t *a); +DLLEXPORT jl_value_t *jl_fptoui_auto(jl_value_t *a); +DLLEXPORT jl_value_t *jl_fptosi_auto(jl_value_t *a); + +DLLEXPORT jl_value_t *jl_checked_fptoui(jl_value_t *ty, jl_value_t *a); +DLLEXPORT jl_value_t *jl_checked_fptosi(jl_value_t *ty, jl_value_t *a); +DLLEXPORT jl_value_t *jl_checked_trunc_sint(jl_value_t *ty, jl_value_t *a); +DLLEXPORT jl_value_t *jl_checked_trunc_uint(jl_value_t *ty, jl_value_t *a); + +DLLEXPORT jl_value_t *jl_check_top_bit(jl_value_t *a); +DLLEXPORT jl_value_t *jl_checked_sadd(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_checked_uadd(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_checked_ssub(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_checked_usub(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_checked_smul(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_checked_umul(jl_value_t *a, jl_value_t *b); + +DLLEXPORT jl_value_t *jl_nan_dom_err(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_ceil_llvm(jl_value_t *a); +DLLEXPORT jl_value_t *jl_floor_llvm(jl_value_t *a); +DLLEXPORT jl_value_t *jl_trunc_llvm(jl_value_t *a); +DLLEXPORT jl_value_t *jl_rint_llvm(jl_value_t *a); +DLLEXPORT jl_value_t *jl_sqrt_llvm(jl_value_t *a); +DLLEXPORT jl_value_t *jl_powi_llvm(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_abs_float(jl_value_t *a); +DLLEXPORT jl_value_t *jl_copysign_float(jl_value_t *a, jl_value_t *b); +DLLEXPORT jl_value_t *jl_flipsign_int(jl_value_t *a, jl_value_t *b); + +DLLEXPORT jl_value_t *jl_select_value(jl_value_t *isfalse, jl_value_t *a, jl_value_t *b); + + + #ifdef __cplusplus } #endif diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c new file mode 100644 index 0000000000000..bd2c296b99f87 --- /dev/null +++ b/src/runtime_intrinsics.c @@ -0,0 +1,930 @@ +// This file is a part of Julia. License is MIT: http://julialang.org/license +// +// This is in implementation of the Julia intrinsic functions against boxed types +// excluding the c interface (ccall, cglobal, llvmcall) +// +// this file assumes a little-endian processor, although that isn't too hard to fix +// it also assumes two's complement negative numbers, which might be a bit harder to fix +// +// TODO: add half-float support + +#include "julia.h" +#include "julia_internal.h" +#include "APInt-C.h" +const unsigned int host_char_bit = 8; + +// run time version of box/unbox intrinsic +DLLEXPORT jl_value_t *jl_reinterpret(jl_value_t *ty, jl_value_t *v) +{ + JL_TYPECHK(reinterpret, datatype, ty); + if (!jl_is_leaf_type(ty) || !jl_is_bitstype(ty)) + jl_error("reinterpret: target type not a leaf bitstype"); + if (!jl_is_bitstype(jl_typeof(v))) + jl_error("reinterpret: value not a bitstype"); + if (jl_datatype_size(jl_typeof(v)) != jl_datatype_size(ty)) + jl_error("reinterpret: argument size does not match size of target type"); + if (ty == jl_typeof(v)) + return v; + if (ty == (jl_value_t*)jl_bool_type) + return *(uint8_t*)jl_data_ptr(v) & 1 ? jl_true : jl_false; + return jl_new_bits(ty, jl_data_ptr(v)); +} + +// run time version of pointerref intrinsic (warning: i is not rooted) +DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i) +{ + JL_TYPECHK(pointerref, pointer, p); + JL_TYPECHK(pointerref, long, i); + jl_value_t *ety = jl_tparam0(jl_typeof(p)); + if (ety == (jl_value_t*)jl_any_type) { + jl_value_t **pp = (jl_value_t**)(jl_unbox_long(p) + (jl_unbox_long(i)-1)*sizeof(void*)); + return *pp; + } + else { + if (!jl_is_datatype(ety)) + jl_error("pointerref: invalid pointer"); + size_t nb = LLT_ALIGN(jl_datatype_size(ety), ((jl_datatype_t*)ety)->alignment); + char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb; + return jl_new_bits(ety, pp); + } +} + +// run time version of pointerset intrinsic (warning: x is not gc-rooted) +DLLEXPORT jl_value_t *jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t *i) +{ + JL_TYPECHK(pointerset, pointer, p); + JL_TYPECHK(pointerset, long, i); + jl_value_t *ety = jl_tparam0(jl_typeof(p)); + if (ety == (jl_value_t*)jl_any_type) { + jl_value_t **pp = (jl_value_t**)(jl_unbox_long(p) + (jl_unbox_long(i)-1)*sizeof(void*)); + *pp = x; + } + else { + if (!jl_is_datatype(ety)) + jl_error("pointerset: invalid pointer"); + size_t nb = LLT_ALIGN(jl_datatype_size(ety), ((jl_datatype_t*)ety)->alignment); + char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb; + if (jl_typeof(x) != ety) + jl_error("pointerset: type mismatch in assign"); + jl_assign_bits(pp, x); + } + return p; +} + + +static inline unsigned int next_power_of_two(unsigned int val) { + /* this function taken from libuv src/unix/core.c */ + val -= 1; + val |= val >> 1; + val |= val >> 2; + val |= val >> 4; + val |= val >> 8; + val |= val >> 16; + val += 1; + return val; +} + +static inline char signbitbyte(void *a, unsigned bytes) { + // sign bit of an signed number of n bytes, as a byte + return signbit(((signed char*)a)[bytes-1]) ? ~0 : 0; +} + +static inline char usignbitbyte(void *a, unsigned bytes) { + // sign bit of an unsigned number + return 0; +} + +static inline unsigned select_by_size(unsigned sz) +{ + /* choose the right sized function specialization */ + switch (sz) { + default: return 0; + case 1: return 1; + case 2: return 2; + case 4: return 3; + case 8: return 4; + case 16: return 5; + } +} + +#define SELECTOR_FUNC(intrinsic) \ + typedef intrinsic##_t select_##intrinsic##_t[6]; \ + static inline intrinsic##_t select_##intrinsic(unsigned sz, select_##intrinsic##_t list) \ + { \ + return list[select_by_size(sz)] ?: list[0]; \ + } + +#define fp_select(a, func) \ + sizeof(a) == sizeof(float) ? func##f((float)a) : func(a) +#define fp_select2(a, b, func) \ + sizeof(a) == sizeof(float) ? func##f(a, b) : func(a, b) + +// fast-function generators // + +// integer input +// OP::Function macro(input) +// name::unique string +// nbits::number of bits +// c_type::c_type corresponding to nbits +#define un_iintrinsic_ctype(OP, name, nbits, c_type) \ +static inline void jl_##name##nbits(unsigned runtime_nbits, void *pa, void *pr) \ +{ \ + c_type a = *(c_type*)pa; \ + *(c_type*)pr = OP(a); \ +} + +// integer input, unsigned output +// OP::Function macro(input) +// name::unique string +// nbits::number of bits +// c_type::c_type corresponding to nbits +#define uu_iintrinsic_ctype(OP, name, nbits, c_type) \ +static inline unsigned jl_##name##nbits(unsigned runtime_nbits, void *pa) \ +{ \ + c_type a = *(c_type*)pa; \ + return OP(a); \ +} + +// floating point +// OP::Function macro(output pointer, input) +// name::unique string +// nbits::number of bits in the *input* +// c_type::c_type corresponding to nbits +#define un_fintrinsic_ctype(OP, name, c_type) \ +static inline void name(unsigned osize, void *pa, void *pr) \ +{ \ + c_type a = *(c_type*)pa; \ + OP((c_type*)pr, a); \ +} + +// float or integer inputs +// OP::Function macro(inputa, inputb) +// name::unique string +// nbits::number of bits +// c_type::c_type corresponding to nbits +#define bi_intrinsic_ctype(OP, name, nbits, c_type) \ +static void jl_##name##nbits(unsigned runtime_nbits, void *pa, void *pb, void *pr) \ +{ \ + c_type a = *(c_type*)pa; \ + c_type b = *(c_type*)pb; \ + *(c_type*)pr = (c_type)OP(a, b); \ +} + +// float or integer inputs, bool output +// OP::Function macro(inputa, inputb) +// name::unique string +// nbits::number of bits +// c_type::c_type corresponding to nbits +#define bool_intrinsic_ctype(OP, name, nbits, c_type) \ +static int jl_##name##nbits(unsigned runtime_nbits, void *pa, void *pb) \ +{ \ + c_type a = *(c_type*)pa; \ + c_type b = *(c_type*)pb; \ + return OP(a, b); \ +} + +// integer inputs, with precondition test +// OP::Function macro(inputa, inputb) +// name::unique string +// nbits::number of bits +// c_type::c_type corresponding to nbits +#define checked_intrinsic_ctype(CHECK_OP, OP, name, nbits, c_type) \ +static int jl_##name##nbits(unsigned runtime_nbits, void *pa, void *pb, void *pr) \ +{ \ + c_type a = *(c_type*)pa; \ + c_type b = *(c_type*)pb; \ + if (CHECK_OP(a, b)) \ + return 1; \ + *(c_type*)pr = (c_type)OP(a, b); \ + return 0; \ +} + +// float inputs +// OP::Function macro(inputa, inputb, inputc) +// name::unique string +// nbits::number of bits +// c_type::c_type corresponding to nbits +#define ter_intrinsic_ctype(OP, name, nbits, c_type) \ +static void jl_##name##nbits(unsigned runtime_nbits, void *pa, void *pb, void *pc, void *pr) \ +{ \ + c_type a = *(c_type*)pa; \ + c_type b = *(c_type*)pb; \ + c_type c = *(c_type*)pc; \ + *(c_type*)pr = (c_type)OP(a, b, c); \ +} + + +// unary operator generator // + +typedef void (*intrinsic_1_t)(unsigned, void*, void*); +SELECTOR_FUNC(intrinsic_1) +#define un_iintrinsic(name, u) \ +DLLEXPORT jl_value_t *jl_##name(jl_value_t *a) \ +{ \ + return jl_iintrinsic_1(jl_typeof(a), a, #name, u##signbitbyte, jl_intrinsiclambda_ty1, name##_list); \ +} +#define un_iintrinsic_fast(LLVMOP, OP, name, u) \ +un_iintrinsic_ctype(OP, name, 8, u##int##8_t) \ +un_iintrinsic_ctype(OP, name, 16, u##int##16_t) \ +un_iintrinsic_ctype(OP, name, 32, u##int##32_t) \ +un_iintrinsic_ctype(OP, name, 64, u##int##64_t) \ +static select_intrinsic_1_t name##_list = { \ + LLVMOP, \ + jl_##name##8, \ + jl_##name##16, \ + jl_##name##32, \ + jl_##name##64, \ +}; \ +un_iintrinsic(name, u) +#define un_iintrinsic_slow(LLVMOP, name, u) \ +static select_intrinsic_1_t name##_list = { \ + LLVMOP \ +}; \ +un_iintrinsic(name, u) + +typedef unsigned (*intrinsic_u1_t)(unsigned, void*); +SELECTOR_FUNC(intrinsic_u1) +#define uu_iintrinsic(name, u) \ +DLLEXPORT jl_value_t *jl_##name(jl_value_t *a) \ +{ \ + return jl_iintrinsic_1(jl_typeof(a), a, #name, u##signbitbyte, jl_intrinsiclambda_u1, name##_list); \ +} +#define uu_iintrinsic_fast(LLVMOP, OP, name, u) \ +uu_iintrinsic_ctype(OP, name, 8, u##int##8_t) \ +uu_iintrinsic_ctype(OP, name, 16, u##int##16_t) \ +uu_iintrinsic_ctype(OP, name, 32, u##int##32_t) \ +uu_iintrinsic_ctype(OP, name, 64, u##int##64_t) \ +static select_intrinsic_u1_t name##_list = { \ + LLVMOP, \ + jl_##name##8, \ + jl_##name##16, \ + jl_##name##32, \ + jl_##name##64, \ +}; \ +uu_iintrinsic(name, u) +#define uu_iintrinsic_slow(LLVMOP, name, u) \ +static select_intrinsic_u1_t name##_list = { \ + LLVMOP \ +}; \ +uu_iintrinsic(name, u) + +static inline jl_value_t *jl_iintrinsic_1(jl_value_t *ty, jl_value_t *a, const char *name, char (*getsign)(void*, unsigned), + jl_value_t* (*lambda1)(jl_value_t*, void*, unsigned, unsigned, void*), void *list) +{ + if (!jl_is_bitstype(jl_typeof(a))) + jl_errorf("%s: value is not a bitstype", name); + if (!jl_is_bitstype(ty)) + jl_errorf("%s: type is not a bitstype", name); + void *pa = jl_data_ptr(a); + unsigned isize = jl_datatype_size(jl_typeof(a)); + unsigned isize2 = next_power_of_two(isize); + unsigned osize = jl_datatype_size(ty); + unsigned osize2 = next_power_of_two(osize); + if (isize2 > osize2) + osize2 = isize2; + if (osize2 > isize || isize2 > isize) { + /* if needed, round type up to a real c-type and set/clear the unused bits */ + void *pa2; + pa2 = alloca(osize2); + /* TODO: this memcpy assumes little-endian, + * for big-endian, need to align the copy to the other end */ \ + memcpy(pa2, pa, isize); + memset(pa2 + isize, getsign(pa, isize), osize2 - isize); + pa = pa2; + } + jl_value_t *newv = lambda1(ty, pa, osize, osize2, list); + if (ty == (jl_value_t*)jl_bool_type) + return *(uint8_t*)jl_data_ptr(newv) & 1 ? jl_true : jl_false; + return newv; +} + +static inline jl_value_t *jl_intrinsiclambda_ty1(jl_value_t *ty, void *pa, unsigned osize, unsigned osize2, void *voidlist) +{ + jl_value_t *newv = newstruct((jl_datatype_t*)ty); + intrinsic_1_t op = select_intrinsic_1(osize2, (intrinsic_1_t*)voidlist); + op(osize * host_char_bit, pa, jl_data_ptr(newv)); + return newv; +} + +static inline jl_value_t *jl_intrinsiclambda_u1(jl_value_t *ty, void *pa, unsigned osize, unsigned osize2, void *voidlist) +{ + jl_value_t *newv = newstruct((jl_datatype_t*)ty); + intrinsic_u1_t op = select_intrinsic_u1(osize2, (intrinsic_u1_t*)voidlist); + unsigned cnt = op(osize * host_char_bit, pa); + // TODO: the following memset/memcpy assumes little-endian + // for big-endian, need to copy from the other end of cnt + if (osize > sizeof(unsigned)) { + // perform zext, if needed + memset((char*)jl_data_ptr(newv) + sizeof(unsigned), 0, osize - sizeof(unsigned)); + osize = sizeof(unsigned); + } + memcpy(jl_data_ptr(newv), &cnt, osize); + return newv; +} + +// conversion operator + +typedef void (*intrinsic_cvt_t)(unsigned, void*, unsigned, void*); +typedef unsigned (*intrinsic_cvt_check_t)(unsigned, unsigned, void*); +#define cvt_iintrinsic_checked(LLVMOP, check_op, name) \ +DLLEXPORT jl_value_t *jl_##name(jl_value_t *ty, jl_value_t *a) \ +{ \ + return jl_intrinsic_cvt(ty, a, #name, LLVMOP, check_op); \ +} +#define cvt_iintrinsic(LLVMOP, name) \ + cvt_iintrinsic_checked(LLVMOP, NULL, name) \ + +static inline jl_value_t *jl_intrinsic_cvt(jl_value_t *ty, jl_value_t *a, const char *name, intrinsic_cvt_t op, intrinsic_cvt_check_t check_op) +{ + jl_value_t *aty = jl_typeof(a); + if (!jl_is_bitstype(aty)) + jl_errorf("%s: value is not a bitstype", name); + if (!jl_is_bitstype(ty)) + jl_errorf("%s: type is not a bitstype", name); + void *pa = jl_data_ptr(a); + unsigned isize = jl_datatype_size(aty); + unsigned osize = jl_datatype_size(ty); + if (check_op && check_op(isize, osize, pa)) + jl_throw(jl_inexact_exception); + jl_value_t *newv = newstruct((jl_datatype_t*)ty); + op(aty == (jl_value_t*)jl_bool_type ? 1 : isize * host_char_bit, pa, + osize * host_char_bit, jl_data_ptr(newv)); + if (ty == (jl_value_t*)jl_bool_type) + return *(uint8_t*)jl_data_ptr(newv) & 1 ? jl_true : jl_false; + return newv; +} + +// floating point + +#define un_fintrinsic_withtype(OP, name) \ +un_fintrinsic_ctype(OP, jl_##name##32, float) \ +un_fintrinsic_ctype(OP, jl_##name##64, double) \ +DLLEXPORT jl_value_t *jl_##name(jl_value_t *ty, jl_value_t *a) \ +{ \ + return jl_fintrinsic_1(ty, a, #name, jl_##name##32, jl_##name##64); \ +} + +#define un_fintrinsic(OP, name) \ +un_fintrinsic_withtype(OP, name##_withtype) \ +DLLEXPORT jl_value_t *jl_##name(jl_value_t *a) \ +{ \ + return jl_##name##_withtype(jl_typeof(a), a); \ +} + +typedef void (fintrinsic_op1)(unsigned, void*, void*); + +static inline jl_value_t *jl_fintrinsic_1(jl_value_t *ty, jl_value_t *a, const char *name, fintrinsic_op1 *floatop, fintrinsic_op1 *doubleop) +{ + if (!jl_is_bitstype(jl_typeof(a))) + jl_errorf("%s: value is not a bitstype", name); + if (!jl_is_bitstype(ty)) + jl_errorf("%s: type is not a bitstype", name); + jl_value_t *newv = newstruct((jl_datatype_t*)ty); + void *pa = jl_data_ptr(a), *pr = jl_data_ptr(newv); + unsigned sz = jl_datatype_size(jl_typeof(a)); + unsigned sz2 = jl_datatype_size(ty); + switch (sz) { + /* choose the right size c-type operation based on the input */ + case 4: + floatop(sz2 * host_char_bit, pa, pr); + break; + case 8: + doubleop(sz2 * host_char_bit, pa, pr); + break; + default: + jl_errorf("%s: runtime floating point intrinsics are not implemented for bit sizes other than 32 and 64", name); + } + return newv; +} + +// binary operator generator // + +// integer + +typedef void (*intrinsic_2_t)(unsigned, void*, void*, void*); +SELECTOR_FUNC(intrinsic_2) +#define bi_iintrinsic(name, u, cvtb) \ +DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b) \ +{ \ + return jl_iintrinsic_2(a, b, #name, u##signbitbyte, jl_intrinsiclambda_2, name##_list, cvtb); \ +} +#define bi_iintrinsic_cnvtb_fast(LLVMOP, OP, name, u, cvtb) \ +bi_intrinsic_ctype(OP, name, 8, u##int##8_t) \ +bi_intrinsic_ctype(OP, name, 16, u##int##16_t) \ +bi_intrinsic_ctype(OP, name, 32, u##int##32_t) \ +bi_intrinsic_ctype(OP, name, 64, u##int##64_t) \ +static select_intrinsic_2_t name##_list = { \ + LLVMOP, \ + jl_##name##8, \ + jl_##name##16, \ + jl_##name##32, \ + jl_##name##64, \ +}; \ +bi_iintrinsic(name, u, cvtb) +#define bi_iintrinsic_fast(LLVMOP, OP, name, u) \ + bi_iintrinsic_cnvtb_fast(LLVMOP, OP, name, u, 0) + +typedef int (*intrinsic_cmp_t)(unsigned, void*, void*); +SELECTOR_FUNC(intrinsic_cmp) +#define cmp_iintrinsic(name, u) \ +DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b) \ +{ \ + return jl_iintrinsic_2(a, b, #name, u##signbitbyte, jl_intrinsiclambda_cmp, name##_list, 0); \ +} +#define bool_iintrinsic_fast(LLVMOP, OP, name, u) \ +bool_intrinsic_ctype(OP, name, 8, u##int##8_t) \ +bool_intrinsic_ctype(OP, name, 16, u##int##16_t) \ +bool_intrinsic_ctype(OP, name, 32, u##int##32_t) \ +bool_intrinsic_ctype(OP, name, 64, u##int##64_t) \ +static select_intrinsic_cmp_t name##_list = { \ + LLVMOP, \ + jl_##name##8, \ + jl_##name##16, \ + jl_##name##32, \ + jl_##name##64, \ +}; \ +cmp_iintrinsic(name, u) + +typedef int (*intrinsic_checked_t)(unsigned, void*, void*, void*); +SELECTOR_FUNC(intrinsic_checked) +#define checked_iintrinsic(name, u) \ +DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b) \ +{ \ + return jl_iintrinsic_2(a, b, #name, u##signbitbyte, jl_intrinsiclambda_checked, name##_list, 0); \ +} +#define checked_iintrinsic_fast(LLVMOP, CHECK_OP, OP, name, u) \ +checked_intrinsic_ctype(CHECK_OP, OP, name, 8, u##int##8_t) \ +checked_intrinsic_ctype(CHECK_OP, OP, name, 16, u##int##16_t) \ +checked_intrinsic_ctype(CHECK_OP, OP, name, 32, u##int##32_t) \ +checked_intrinsic_ctype(CHECK_OP, OP, name, 64, u##int##64_t) \ +static select_intrinsic_checked_t name##_list = { \ + LLVMOP, \ + jl_##name##8, \ + jl_##name##16, \ + jl_##name##32, \ + jl_##name##64, \ +}; \ +checked_iintrinsic(name, u) +#define checked_iintrinsic_slow(LLVMOP, name, u) \ +static select_intrinsic_checked_t name##_list = { \ + LLVMOP \ +}; \ +checked_iintrinsic(name, u) + +static inline jl_value_t *jl_iintrinsic_2(jl_value_t *a, jl_value_t *b, const char *name, char (*getsign)(void*, unsigned), + jl_value_t* (*lambda2)(jl_value_t*, void*, void*, unsigned, unsigned, void*), + void *list, int cvtb) +{ + jl_value_t *ty = jl_typeof(a); + jl_value_t *tyb = jl_typeof(b); + if (tyb != ty) { + if (!cvtb) + jl_errorf("%s: types of a and b must match", name); + if (!jl_is_bitstype(tyb)) + jl_errorf("%s: b is not a bitstypes", name); + } + if (!jl_is_bitstype(ty)) + jl_errorf("%s: a is not a bitstypes", name); + void *pa = jl_data_ptr(a), *pb = jl_data_ptr(b); + unsigned sz = jl_datatype_size(ty); + unsigned sz2 = next_power_of_two(sz); + unsigned szb = jl_datatype_size(tyb); + if (sz2 > sz) { + /* round type up to the appropriate c-type and set/clear the unused bits */ + void *pa2 = alloca(sz2); + memcpy(pa2, pa, sz); + memset((char*)pa2 + sz, getsign(pa, sz), sz2 - sz); + pa = pa2; + } + if (sz2 > szb) { + /* round type up to the appropriate c-type and set/clear/truncate the unused bits */ + void *pb2 = alloca(sz2); + memcpy(pb2, pb, szb); + memset((char*)pb2 + szb, getsign(pb, sz), sz2 - szb); + pb = pb2; + } + jl_value_t *newv = lambda2(ty, pa, pb, sz, sz2, list); + return newv; +} + +static inline jl_value_t *jl_intrinsiclambda_2(jl_value_t *ty, void *pa, void *pb, unsigned sz, unsigned sz2, void *voidlist) +{ + jl_value_t *newv = newstruct((jl_datatype_t*)ty); + intrinsic_2_t op = select_intrinsic_2(sz2, (intrinsic_2_t*)voidlist); + op(sz * host_char_bit, pa, pb, jl_data_ptr(newv)); + if (ty == (jl_value_t*)jl_bool_type) + return *(uint8_t*)jl_data_ptr(newv) & 1 ? jl_true : jl_false; + return newv; +} + +static inline jl_value_t *jl_intrinsiclambda_cmp(jl_value_t *ty, void *pa, void *pb, unsigned sz, unsigned sz2, void *voidlist) +{ + intrinsic_cmp_t op = select_intrinsic_cmp(sz2, (intrinsic_cmp_t*)voidlist); + int cmp = op(sz * host_char_bit, pa, pb); + return cmp ? jl_true : jl_false; +} + +static inline jl_value_t *jl_intrinsiclambda_checked(jl_value_t *ty, void *pa, void *pb, unsigned sz, unsigned sz2, void *voidlist) +{ + jl_value_t *newv = newstruct((jl_datatype_t*)ty); + intrinsic_checked_t op = select_intrinsic_checked(sz2, (intrinsic_checked_t*)voidlist); + int ovflw = op(sz * host_char_bit, pa, pb, jl_data_ptr(newv)); + if (ovflw) + jl_throw(jl_overflow_exception); + if (ty == (jl_value_t*)jl_bool_type) + return *(uint8_t*)jl_data_ptr(newv) & 1 ? jl_true : jl_false; + return newv; +} + +// floating point + +#define bi_fintrinsic(OP, name) \ + bi_intrinsic_ctype(OP, name, 32, float) \ + bi_intrinsic_ctype(OP, name, 64, double) \ +DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b) \ +{ \ + jl_value_t *ty = jl_typeof(a); \ + if (jl_typeof(b) != ty) \ + jl_error(#name ": types of a and b must match"); \ + if (!jl_is_bitstype(ty)) \ + jl_error(#name ": values are not bitstypes"); \ + jl_value_t *newv = newstruct((jl_datatype_t*)ty); \ + void *pa = jl_data_ptr(a), *pb = jl_data_ptr(b), *pr = jl_data_ptr(newv); \ + int sz = jl_datatype_size(ty); \ + switch (sz) { \ + /* choose the right size c-type operation */ \ + case 4: \ + jl_##name##32(32, pa, pb, pr); \ + break; \ + case 8: \ + jl_##name##64(64, pa, pb, pr); \ + break; \ + default: \ + jl_error(#name ": runtime floating point intrinsics are not implemented for bit sizes other than 32 and 64"); \ + } \ + return newv; \ +} + +#define bool_fintrinsic(OP, name) \ + bool_intrinsic_ctype(OP, name, 32, float) \ + bool_intrinsic_ctype(OP, name, 64, double) \ +DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b) \ +{ \ + jl_value_t *ty = jl_typeof(a); \ + if (jl_typeof(b) != ty) \ + jl_error(#name ": types of a and b must match"); \ + if (!jl_is_bitstype(ty)) \ + jl_error(#name ": values are not bitstypes"); \ + void *pa = jl_data_ptr(a), *pb = jl_data_ptr(b); \ + int sz = jl_datatype_size(ty); \ + int cmp; \ + switch (sz) { \ + /* choose the right size c-type operation */ \ + case 4: \ + cmp = jl_##name##32(32, pa, pb); \ + break; \ + case 8: \ + cmp = jl_##name##64(64, pa, pb); \ + break; \ + default: \ + jl_error(#name ": runtime floating point intrinsics are not implemented for bit sizes other than 32 and 64"); \ + } \ + return cmp ? jl_true : jl_false; \ +} + +#define ter_fintrinsic(OP, name) \ + ter_intrinsic_ctype(OP, name, 32, float) \ + ter_intrinsic_ctype(OP, name, 64, double) \ +DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b, jl_value_t *c) \ +{ \ + jl_value_t *ty = jl_typeof(a); \ + if (jl_typeof(b) != ty || jl_typeof(c) != ty) \ + jl_error(#name ": types of a, b, and c must match"); \ + if (!jl_is_bitstype(ty)) \ + jl_error(#name ": values are not bitstypes"); \ + jl_value_t *newv = newstruct((jl_datatype_t*)ty); \ + void *pa = jl_data_ptr(a), *pb = jl_data_ptr(b), *pc = jl_data_ptr(c), *pr = jl_data_ptr(newv); \ + int sz = jl_datatype_size(ty); \ + switch (sz) { \ + /* choose the right size c-type operation */ \ + case 4: \ + jl_##name##32(32, pa, pb, pc, pr); \ + break; \ + case 8: \ + jl_##name##64(64, pa, pb, pc, pr); \ + break; \ + default: \ + jl_error(#name ": runtime floating point intrinsics are not implemented for bit sizes other than 32 and 64"); \ + } \ + return newv; \ +} + +// arithmetic +#define neg(a) -a +#define neg_float(pr, a) *pr = -a +un_iintrinsic_fast(LLVMNeg, neg, neg_int, u) +#define add(a,b) a + b +bi_iintrinsic_fast(LLVMAdd, add, add_int, u) +#define sub(a,b) a - b +bi_iintrinsic_fast(LLVMSub, sub, sub_int, u) +#define mul(a,b) a * b +bi_iintrinsic_fast(LLVMMul, mul, mul_int, u) +#define div(a,b) a / b +bi_iintrinsic_fast(LLVMSDiv, div, sdiv_int, ) +bi_iintrinsic_fast(LLVMUDiv, div, udiv_int, u) +#define rem(a,b) a % b +bi_iintrinsic_fast(LLVMSRem, rem, srem_int, ) +bi_iintrinsic_fast(LLVMURem, rem, urem_int, u) +#define smod(a,b) ((a < 0) == (b < 0)) ? a % b : (b + (a % b)) % b +bi_iintrinsic_fast(jl_LLVMSMod, smod, smod_int, ) +#define frem(a, b) \ + fp_select2(a, b, fmod) + +un_fintrinsic(neg_float,neg_float) +bi_fintrinsic(add,add_float) +bi_fintrinsic(sub,sub_float) +bi_fintrinsic(mul,mul_float) +bi_fintrinsic(div,div_float) +bi_fintrinsic(frem,rem_float) + +// ternary operators // +#define fma(a, b, c) \ + sizeof(a) == sizeof(float) ? fmaf(a, b, c) : fma(a, b, c) +#define muladd(a, b, c) a * b + c +ter_fintrinsic(fma,fma_float) +ter_fintrinsic(muladd,muladd_float) + +// same-type comparisons +#define eq(a,b) a == b +bool_iintrinsic_fast(LLVMICmpEQ, eq, eq_int, u) +#define ne(a,b) a != b +bool_iintrinsic_fast(LLVMICmpNE, ne, ne_int, u) +#define lt(a,b) a < b +bool_iintrinsic_fast(LLVMICmpSLT, lt, slt_int, ) +bool_iintrinsic_fast(LLVMICmpULT, lt, ult_int, u) +#define le(a,b) a <= b +bool_iintrinsic_fast(LLVMICmpSLE, le, sle_int, ) +bool_iintrinsic_fast(LLVMICmpULE, le, ule_int, u) + +typedef union { + float f; + int32_t d; + uint32_t ud; +} bits32; +typedef union { + double f; + int64_t d; + uint64_t ud; +} bits64; + +#define fpiseq_n(c_type, nbits) \ +static inline int fpiseq##nbits(c_type a, c_type b) { \ + bits##nbits ua, ub; \ + ua.f = a; \ + ub.f = b; \ + return (isnan(a) && isnan(b)) || ua.d == ub.d; \ +} +fpiseq_n(float, 32) +fpiseq_n(double, 64) +#define fpiseq(a,b) \ + sizeof(a) == sizeof(float) ? fpiseq32(a, b) : fpiseq64(a, b) + +#define fpislt_n(c_type, nbits) \ +static inline int fpislt##nbits(c_type a, c_type b) { \ + bits##nbits ua, ub; \ + ua.f = a; \ + ub.f = b; \ + if (!isnan(a) && isnan(b)) \ + return 1; \ + if (isnan(a) || isnan(b)) \ + return 0; \ + if (ua.d >= 0 && ua.d < ub.d) \ + return 1; \ + if (ua.d < 0 && ua.ud > ub.ud) \ + return 1; \ + return 0; \ +} +fpislt_n(float, 32) +fpislt_n(double, 64) +#define fpislt(a, b) \ + sizeof(a) == sizeof(float) ? fpislt32(a, b) : fpislt64(a, b) + +bool_fintrinsic(eq,eq_float) +bool_fintrinsic(ne,ne_float) +bool_fintrinsic(lt,lt_float) +bool_fintrinsic(le,le_float) +bool_fintrinsic(fpiseq,fpiseq) +bool_fintrinsic(fpislt,fpislt) + +// bitwise operators +#define and(a,b) a & b +bi_iintrinsic_fast(LLVMAnd, and, and_int, u) +#define or(a,b) a | b +bi_iintrinsic_fast(LLVMOr, or, or_int, u) +#define xor(a,b) a ^ b +bi_iintrinsic_fast(LLVMXor, xor, xor_int, u) +#define shl(a,b) b >= 8 * sizeof(a) ? 0 : a << b +bi_iintrinsic_cnvtb_fast(LLVMShl, shl, shl_int, u, 1) +#define lshr(a,b) (b >= 8 * sizeof(a)) ? 0 : a >> b +bi_iintrinsic_cnvtb_fast(LLVMLShr, lshr, lshr_int, u, 1) +#define ashr(a,b) \ + /* if ((signed)a > 0) [in two's complement] ? ... : ...) */ \ + (a >> (host_char_bit * sizeof(a) - 1)) ? ~(b >= 8 * sizeof(a) ? 0 : (~a) >> b) : (b >= 8 * sizeof(a) ? 0 : a >> b) +bi_iintrinsic_cnvtb_fast(LLVMAShr, ashr, ashr_int, u, 1) +//#define bswap(a) __builtin_bswap(a) +//un_iintrinsic_fast(LLVMByteSwap, bswap, bswap_int, u) +un_iintrinsic_slow(LLVMByteSwap, bswap_int, u) +//#define ctpop(a) __builtin_ctpop(a) +//uu_iintrinsic_fast(LLVMCountPopulation, ctpop, ctpop_int, u) +uu_iintrinsic_slow(LLVMCountPopulation, ctpop_int, u) +//#define ctlz(a) __builtin_ctlz(a) +//uu_iintrinsic_fast(LLVMCountLeadingZeros, ctlz_int, u) +uu_iintrinsic_slow(LLVMCountLeadingZeros, ctlz_int, u) +//#define cttz(a) __builtin_cttz(a) +//uu_iintrinsic_fast(LLVMCountTrailingZeros, cttz, cttz_int, u) +uu_iintrinsic_slow(LLVMCountTrailingZeros, cttz_int, u) +#define not(a) ~a +un_iintrinsic_fast(LLVMFlipAllBits, not, not_int, u) + +// conversions +cvt_iintrinsic(LLVMTrunc, trunc_int) +cvt_iintrinsic(LLVMSExt, sext_int) +cvt_iintrinsic(LLVMZExt, zext_int) +cvt_iintrinsic(LLVMSItoFP, sitofp) +cvt_iintrinsic(LLVMUItoFP, uitofp) +cvt_iintrinsic(LLVMFPtoSI, fptosi) +cvt_iintrinsic(LLVMFPtoUI, fptoui) + +#define fpcvt(pr, a) \ + if (osize == 32) \ + *(float*)pr = a; \ + else if (osize == 64) \ + *(double*)pr = a; \ + else \ + jl_error("fptrunc/fpext: runtime floating point intrinsics are not implemented for bit sizes other than 32 and 64"); +un_fintrinsic_withtype(fpcvt,fptrunc) +un_fintrinsic_withtype(fpcvt,fpext) + +DLLEXPORT jl_value_t *jl_fptoui_auto(jl_value_t *a) +{ + jl_datatype_t *ty; + switch (jl_datatype_size(jl_typeof(a))) { + case 4: + ty = jl_uint32_type; + break; + case 8: + ty = jl_uint64_type; + break; + default: + jl_error("fptoui: runtime floating point intrinsics are not implemented for bit sizes other than 32 and 64"); + } + return jl_fptoui((jl_value_t*)ty, a); +} +DLLEXPORT jl_value_t *jl_fptosi_auto(jl_value_t *a) +{ + jl_datatype_t *ty; + switch (jl_datatype_size(jl_typeof(a))) { + case 4: + ty = jl_int32_type; + break; + case 8: + ty = jl_int64_type; + break; + default: + jl_error("fptoui: runtime floating point intrinsics are not implemented for bit sizes other than 32 and 64"); + } + return jl_fptosi((jl_value_t*)ty, a); +} + +// checked conversion +static inline int all_eq(char *p, char n, char v) +{ + // computes p[0:n] == v + while (n--) + if (*p++ != v) + return 0; + return 1; +} +static unsigned check_trunc_sint(unsigned isize, unsigned osize, void *pa) +{ + return !all_eq((char*)pa + osize, isize - osize, signbitbyte(pa, isize)); // TODO: assumes little-endian +} +cvt_iintrinsic_checked(LLVMTrunc, check_trunc_sint, checked_trunc_sint) +static unsigned check_trunc_uint(unsigned isize, unsigned osize, void *pa) +{ + return !all_eq((char*)pa + osize, isize - osize, 0); // TODO: assumes little-endian +} +cvt_iintrinsic_checked(LLVMTrunc, check_trunc_uint, checked_trunc_uint) + +#define checked_fptosi(pr, a) \ + if (!LLVMFPtoSI_exact(sizeof(a) * host_char_bit, pa, osize, pr)) \ + jl_throw(jl_inexact_exception); +un_fintrinsic_withtype(checked_fptosi, checked_fptosi) +#define checked_fptoui(pr, a) \ + if (!LLVMFPtoUI_exact(sizeof(a) * host_char_bit, pa, osize, pr)) \ + jl_throw(jl_inexact_exception); +un_fintrinsic_withtype(checked_fptoui, checked_fptoui) + +DLLEXPORT jl_value_t *jl_check_top_bit(jl_value_t *a) +{ + jl_value_t *ty = jl_typeof(a); + if (!jl_is_bitstype(ty)) + jl_error("check_top_bit: value is not a bitstype"); + if (signbitbyte(jl_data_ptr(a), jl_datatype_size(ty))) + jl_throw(jl_inexact_exception); + return a; +} + +// checked arithmetic +#define check_sadd(a,b) \ + /* this test is a reduction of (b > 0) ? (a + b >= typemin(a)) : (a + b < typemin(a)) ==> overflow */ \ + (b > 0) == (a >= (((typeof(a))1) << (8 * sizeof(a) - 1)) - b) +checked_iintrinsic_fast(LLVMAdd_sov, check_sadd, add, checked_sadd, ) +#define check_uadd(a,b) \ + /* this test checks for (a + b) > typemax(a) ==> overflow */ \ + a >= -b +checked_iintrinsic_fast(LLVMAdd_uov, check_uadd, add, checked_uadd, u) +#define check_ssub(a,b) check_sadd(a,-b) +checked_iintrinsic_fast(LLVMSub_sov, check_ssub, sub, checked_ssub, ) +#define check_usub(a,b) \ + /* this test checks for (a - b) < 0 ==> overflow */ \ + a < b +checked_iintrinsic_fast(LLVMSub_uov, check_usub, sub, checked_usub, u) +checked_iintrinsic_slow(LLVMMul_sov, checked_smul, ) +checked_iintrinsic_slow(LLVMMul_uov, checked_umul, u) + +DLLEXPORT jl_value_t *jl_nan_dom_err(jl_value_t *a, jl_value_t *b) +{ + jl_value_t *ty = jl_typeof(a); + if (jl_typeof(b) != ty) + jl_error("nan_dom_err: types of a and b must match"); + if (!jl_is_bitstype(ty)) + jl_error("nan_dom_err: values are not bitstypes"); + switch (jl_datatype_size(ty)) { + case 4: + if (isnan(*(float*)a) && !isnan(*(float*)b)) + jl_throw(jl_domain_exception); + break; + case 8: + if (isnan(*(double*)a) && !isnan(*(double*)b)) + jl_throw(jl_domain_exception); + break; + default: + jl_error("nan_dom_err: runtime floating point intrinsics are not implemented for bit sizes other than 32 and 64"); + } + return a; +} + +// functions +#define flipsign(a, b) \ + (b >= 0) ? a : -a +bi_iintrinsic_fast(jl_LLVMFlipSign, flipsign, flipsign_int, ) +#define abs_float(pr, a) *pr = fp_select(a, fabs) +#define ceil_float(pr, a) *pr = fp_select(a, ceil) +#define floor_float(pr, a) *pr = fp_select(a, floor) +#define trunc_float(pr, a) *pr = fp_select(a, trunc) +#define rint_float(pr, a) *pr = fp_select(a, rint) +#define sqrt_float(pr, a) \ + if (a < 0) \ + jl_throw(jl_domain_exception); \ + *pr = fp_select(a, sqrt) +#define copysign_float(a, b) \ + fp_select2(a, b, copysign) + +un_fintrinsic(abs_float,abs_float) +bi_fintrinsic(copysign_float,copysign_float) +un_fintrinsic(ceil_float,ceil_llvm) +un_fintrinsic(floor_float,floor_llvm) +un_fintrinsic(trunc_float,trunc_llvm) +un_fintrinsic(rint_float,rint_llvm) +un_fintrinsic(sqrt_float,sqrt_llvm) + +DLLEXPORT jl_value_t *jl_powi_llvm(jl_value_t *a, jl_value_t *b) +{ + jl_value_t *ty = jl_typeof(a); + if (!jl_is_bitstype(ty)) + jl_error("powi_llvm: a is not a bitstype"); + if (!jl_is_bitstype(jl_typeof(b)) || jl_datatype_size(jl_typeof(b)) != 4) + jl_error("powi_llvm: b is not a 32-bit bitstype"); + jl_value_t *newv = newstruct((jl_datatype_t*)ty); + void *pa = jl_data_ptr(a), *pr = jl_data_ptr(newv); + int sz = jl_datatype_size(ty); + switch (sz) { + /* choose the right size c-type operation */ + case 4: + *(float*)pr = powf(*(float*)pa, (float)jl_unbox_int32(b)); + break; + case 8: + *(double*)pr = pow(*(double*)pa, (double)jl_unbox_int32(b)); + break; + default: + jl_error("powi_llvm: runtime floating point intrinsics are not implemented for bit sizes other than 32 and 64"); + } + return newv; +} + +DLLEXPORT jl_value_t *jl_select_value(jl_value_t *isfalse, jl_value_t *a, jl_value_t *b) +{ + JL_TYPECHK(isfalse, bool, isfalse); + return (isfalse == jl_false ? b : a); +} From 971747232bbc5f5846486e8add2e3bcea6dcb988 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 10 Aug 2015 00:04:34 -0400 Subject: [PATCH 0473/1938] wip: cleanup intrinsics --- src/intrinsics.cpp | 178 ++++++++++++++++++++++----------------------- 1 file changed, 87 insertions(+), 91 deletions(-) diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index d8db10bfe7288..e92583d126fe6 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -330,7 +330,7 @@ static Value *emit_unbox(Type *to, const jl_cgval_t &x, jl_value_t *jt) } // unbox, trying to determine correct bitstype automatically -// returns some sort of raw, unboxed numeric type (in registers) +// returns some sort of raw, unboxed numeric type (e.g. in registers) static Value *auto_unbox(const jl_cgval_t &v, jl_codectx_t *ctx) { jl_value_t *bt = v.typ; @@ -831,9 +831,6 @@ static Value *emit_smod(Value *x, Value *den, jl_codectx_t *ctx) return ret; } -#define HANDLE(intr,n) \ - case intr: - static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, size_t nargs, jl_codectx_t *ctx, jl_datatype_t* *newtyp); static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, @@ -857,20 +854,21 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, default: int ldepth = ctx->gc.argDepth; Value *r; + Value *func = prepare_call(runtime_func[f]); if (nargs == 1) { Value *x = emit_boxed_rooted(args[1], ctx).V; - r = builder.CreateCall(runtime_func[f], x); + r = builder.CreateCall(func, x); } else if (nargs == 2) { Value *x = emit_boxed_rooted(args[1], ctx).V; Value *y = emit_boxed_rooted(args[2], ctx).V; - r = builder.CreateCall2(runtime_func[f], x, y); + r = builder.CreateCall2(func, x, y); } else if (nargs == 3) { Value *x = emit_boxed_rooted(args[1], ctx).V; Value *y = emit_boxed_rooted(args[2], ctx).V; Value *z = emit_boxed_rooted(args[3], ctx).V; - r = builder.CreateCall3(runtime_func[f], x, y, z); + r = builder.CreateCall3(func, x, y, z); } else { assert(0); @@ -886,22 +884,22 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, return generic_box(args[1], args[2], ctx); case unbox: return generic_box(args[1], args[2], ctx); - HANDLE(trunc_int,2) + case trunc_int: return generic_trunc(args[1], args[2], ctx, false, false); - HANDLE(checked_trunc_sint,2) + case checked_trunc_sint: return generic_trunc(args[1], args[2], ctx, true, true); - HANDLE(checked_trunc_uint,2) + case checked_trunc_uint: return generic_trunc(args[1], args[2], ctx, true, false); - HANDLE(sext_int,2) + case sext_int: return generic_sext(args[1], args[2], ctx); - HANDLE(zext_int,2) + case zext_int: return generic_zext(args[1], args[2], ctx); - HANDLE(checked_fptosi,2) + case checked_fptosi: return emit_checked_fptosi(args[1], args[2], ctx); - HANDLE(checked_fptoui,2) + case checked_fptoui: return emit_checked_fptoui(args[1], args[2], ctx); - HANDLE(uitofp,2) { + case uitofp: { jl_value_t *bt = staticeval_bitstype(args[1], "uitofp", ctx); if (!bt) return jl_cgval_t(); int nb = get_bitstype_nbits(bt); @@ -910,7 +908,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, return mark_julia_type(builder.CreateUIToFP(xi, FTnbits(nb)), false, bt); } - HANDLE(sitofp,2) { + case sitofp: { jl_value_t *bt = staticeval_bitstype(args[1], "sitofp", ctx); if (!bt) return jl_cgval_t(); int nb = get_bitstype_nbits(bt); @@ -952,7 +950,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, return mark_julia_type(builder.CreateFPToSI(xf, Type::getIntNTy(jl_LLVMContext, nb)), false, bt); } - HANDLE(fptrunc,2) { + case fptrunc: { jl_value_t *bt = staticeval_bitstype(args[1], "sitofp", ctx); if (!bt) return jl_cgval_t(); int nb = get_bitstype_nbits(bt); @@ -961,7 +959,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, return mark_julia_type(builder.CreateFPTrunc(xf, FTnbits(nb)), false, bt); } - HANDLE(fpext,2) { + case fpext: { jl_value_t *bt = staticeval_bitstype(args[1], "sitofp", ctx); if (!bt) return jl_cgval_t(); int nb = get_bitstype_nbits(bt); @@ -980,7 +978,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, return mark_julia_type(builder.CreateFPExt(x, FTnbits(nb)), false, bt); } - HANDLE(select_value,3) { + case select_value: { Value *isfalse = emit_condition(args[1], "select_value", ctx); // emit the first argument jl_value_t *t1 = expr_type(args[2], ctx); jl_value_t *t2 = expr_type(args[3], ctx); @@ -1067,11 +1065,11 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, Value *den; Value *typemin; switch (f) { - HANDLE(neg_int,1) return builder.CreateSub(ConstantInt::get(t, 0), JL_INT(x)); - HANDLE(add_int,2) return builder.CreateAdd(JL_INT(x), JL_INT(y)); - HANDLE(sub_int,2) return builder.CreateSub(JL_INT(x), JL_INT(y)); - HANDLE(mul_int,2) return builder.CreateMul(JL_INT(x), JL_INT(y)); - HANDLE(sdiv_int,2) + case neg_int: return builder.CreateSub(ConstantInt::get(t, 0), JL_INT(x)); + case add_int: return builder.CreateAdd(JL_INT(x), JL_INT(y)); + case sub_int: return builder.CreateSub(JL_INT(x), JL_INT(y)); + case mul_int: return builder.CreateMul(JL_INT(x), JL_INT(y)); + case sdiv_int: den = JL_INT(y); t = den->getType(); x = JL_INT(x); @@ -1089,24 +1087,24 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, prepare_global(jldiverr_var), ctx); return builder.CreateSDiv(x, den); - HANDLE(udiv_int,2) + case udiv_int: den = JL_INT(y); t = den->getType(); raise_exception_unless(builder.CreateICmpNE(den, ConstantInt::get(t,0)), prepare_global(jldiverr_var), ctx); return builder.CreateUDiv(JL_INT(x), den); - HANDLE(srem_int,2) + case srem_int: return emit_srem(JL_INT(x), JL_INT(y), ctx); - HANDLE(urem_int,2) + case urem_int: den = JL_INT(y); t = den->getType(); raise_exception_unless(builder.CreateICmpNE(den, ConstantInt::get(t,0)), prepare_global(jldiverr_var), ctx); return builder.CreateURem(JL_INT(x), den); - HANDLE(smod_int,2) + case smod_int: return emit_smod(JL_INT(x), JL_INT(y), ctx); // Implements IEEE negate. Unfortunately there is no compliant way @@ -1114,25 +1112,25 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, // that do the correct thing on LLVM <= 3.3 and >= 3.5 respectively. // See issue #7868 #ifdef LLVM35 - HANDLE(neg_float,1) return math_builder(ctx)().CreateFSub(ConstantFP::get(FT(t), -0.0), FP(x)); - HANDLE(neg_float_fast,1) return math_builder(ctx, true)().CreateFNeg(FP(x)); + case neg_float: return math_builder(ctx)().CreateFSub(ConstantFP::get(FT(t), -0.0), FP(x)); + case neg_float_fast: return math_builder(ctx, true)().CreateFNeg(FP(x)); #else - HANDLE(neg_float,1) + case neg_float: return math_builder(ctx)().CreateFMul(ConstantFP::get(FT(t), -1.0), FP(x)); - HANDLE(neg_float_fast,1) + case neg_float_fast: return math_builder(ctx, true)().CreateFMul(ConstantFP::get(FT(t), -1.0), FP(x)); #endif - HANDLE(add_float,2) return math_builder(ctx)().CreateFAdd(FP(x), FP(y)); - HANDLE(sub_float,2) return math_builder(ctx)().CreateFSub(FP(x), FP(y)); - HANDLE(mul_float,2) return math_builder(ctx)().CreateFMul(FP(x), FP(y)); - HANDLE(div_float,2) return math_builder(ctx)().CreateFDiv(FP(x), FP(y)); - HANDLE(rem_float,2) return math_builder(ctx)().CreateFRem(FP(x), FP(y)); - HANDLE(add_float_fast,2) return math_builder(ctx, true)().CreateFAdd(FP(x), FP(y)); - HANDLE(sub_float_fast,2) return math_builder(ctx, true)().CreateFSub(FP(x), FP(y)); - HANDLE(mul_float_fast,2) return math_builder(ctx, true)().CreateFMul(FP(x), FP(y)); - HANDLE(div_float_fast,2) return math_builder(ctx, true)().CreateFDiv(FP(x), FP(y)); - HANDLE(rem_float_fast,2) return math_builder(ctx, true)().CreateFRem(FP(x), FP(y)); - HANDLE(fma_float,3) { + case add_float: return math_builder(ctx)().CreateFAdd(FP(x), FP(y)); + case sub_float: return math_builder(ctx)().CreateFSub(FP(x), FP(y)); + case mul_float: return math_builder(ctx)().CreateFMul(FP(x), FP(y)); + case div_float: return math_builder(ctx)().CreateFDiv(FP(x), FP(y)); + case rem_float: return math_builder(ctx)().CreateFRem(FP(x), FP(y)); + case add_float_fast: return math_builder(ctx, true)().CreateFAdd(FP(x), FP(y)); + case sub_float_fast: return math_builder(ctx, true)().CreateFSub(FP(x), FP(y)); + case mul_float_fast: return math_builder(ctx, true)().CreateFMul(FP(x), FP(y)); + case div_float_fast: return math_builder(ctx, true)().CreateFDiv(FP(x), FP(y)); + case rem_float_fast: return math_builder(ctx, true)().CreateFRem(FP(x), FP(y)); + case fma_float: { assert(y->getType() == x->getType()); assert(z->getType() == y->getType()); Value *fmaintr = Intrinsic::getDeclaration(jl_Module, Intrinsic::fma, @@ -1143,7 +1141,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, return builder.CreateCall3(fmaintr, FP(x), FP(y), FP(z)); #endif } - HANDLE(muladd_float,3) + case muladd_float: #ifdef LLVM34 { assert(y->getType() == x->getType()); @@ -1167,12 +1165,12 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, CreateFAdd(builder.CreateFMul(FP(x), FP(y)), FP(z)); #endif - HANDLE(checked_sadd,2) - HANDLE(checked_uadd,2) - HANDLE(checked_ssub,2) - HANDLE(checked_usub,2) - HANDLE(checked_smul,2) - HANDLE(checked_umul,2) { + case checked_sadd: + case checked_uadd: + case checked_ssub: + case checked_usub: + case checked_smul: + case checked_umul: { Value *ix = JL_INT(x); Value *iy = JL_INT(y); assert(ix->getType() == iy->getType()); Value *intr = @@ -1199,7 +1197,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, return builder.CreateExtractValue(res, ArrayRef(0)); } - HANDLE(check_top_bit,1) + case check_top_bit: // raise InexactError if argument's top bit is set x = JL_INT(x); raise_exception_if(builder. @@ -1209,24 +1207,24 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, prepare_global(jlinexacterr_var), ctx); return x; - HANDLE(eq_int,2) *newtyp = jl_bool_type; return builder.CreateICmpEQ(JL_INT(x), JL_INT(y)); - HANDLE(ne_int,2) *newtyp = jl_bool_type; return builder.CreateICmpNE(JL_INT(x), JL_INT(y)); - HANDLE(slt_int,2) *newtyp = jl_bool_type; return builder.CreateICmpSLT(JL_INT(x), JL_INT(y)); - HANDLE(ult_int,2) *newtyp = jl_bool_type; return builder.CreateICmpULT(JL_INT(x), JL_INT(y)); - HANDLE(sle_int,2) *newtyp = jl_bool_type; return builder.CreateICmpSLE(JL_INT(x), JL_INT(y)); - HANDLE(ule_int,2) *newtyp = jl_bool_type; return builder.CreateICmpULE(JL_INT(x), JL_INT(y)); + case eq_int: *newtyp = jl_bool_type; return builder.CreateICmpEQ(JL_INT(x), JL_INT(y)); + case ne_int: *newtyp = jl_bool_type; return builder.CreateICmpNE(JL_INT(x), JL_INT(y)); + case slt_int: *newtyp = jl_bool_type; return builder.CreateICmpSLT(JL_INT(x), JL_INT(y)); + case ult_int: *newtyp = jl_bool_type; return builder.CreateICmpULT(JL_INT(x), JL_INT(y)); + case sle_int: *newtyp = jl_bool_type; return builder.CreateICmpSLE(JL_INT(x), JL_INT(y)); + case ule_int: *newtyp = jl_bool_type; return builder.CreateICmpULE(JL_INT(x), JL_INT(y)); - HANDLE(eq_float,2) *newtyp = jl_bool_type; return math_builder(ctx)().CreateFCmpOEQ(FP(x), FP(y)); - HANDLE(ne_float,2) *newtyp = jl_bool_type; return math_builder(ctx)().CreateFCmpUNE(FP(x), FP(y)); - HANDLE(lt_float,2) *newtyp = jl_bool_type; return math_builder(ctx)().CreateFCmpOLT(FP(x), FP(y)); - HANDLE(le_float,2) *newtyp = jl_bool_type; return math_builder(ctx)().CreateFCmpOLE(FP(x), FP(y)); + case eq_float: *newtyp = jl_bool_type; return math_builder(ctx)().CreateFCmpOEQ(FP(x), FP(y)); + case ne_float: *newtyp = jl_bool_type; return math_builder(ctx)().CreateFCmpUNE(FP(x), FP(y)); + case lt_float: *newtyp = jl_bool_type; return math_builder(ctx)().CreateFCmpOLT(FP(x), FP(y)); + case le_float: *newtyp = jl_bool_type; return math_builder(ctx)().CreateFCmpOLE(FP(x), FP(y)); - HANDLE(eq_float_fast,2) *newtyp = jl_bool_type; return math_builder(ctx, true)().CreateFCmpOEQ(FP(x), FP(y)); - HANDLE(ne_float_fast,2) *newtyp = jl_bool_type; return math_builder(ctx, true)().CreateFCmpUNE(FP(x), FP(y)); - HANDLE(lt_float_fast,2) *newtyp = jl_bool_type; return math_builder(ctx, true)().CreateFCmpOLT(FP(x), FP(y)); - HANDLE(le_float_fast,2) *newtyp = jl_bool_type; return math_builder(ctx, true)().CreateFCmpOLE(FP(x), FP(y)); + case eq_float_fast: *newtyp = jl_bool_type; return math_builder(ctx, true)().CreateFCmpOEQ(FP(x), FP(y)); + case ne_float_fast: *newtyp = jl_bool_type; return math_builder(ctx, true)().CreateFCmpUNE(FP(x), FP(y)); + case lt_float_fast: *newtyp = jl_bool_type; return math_builder(ctx, true)().CreateFCmpOLT(FP(x), FP(y)); + case le_float_fast: *newtyp = jl_bool_type; return math_builder(ctx, true)().CreateFCmpOLE(FP(x), FP(y)); - HANDLE(fpiseq,2) { + case fpiseq: { *newtyp = jl_bool_type; Value *xi = JL_INT(x); Value *yi = JL_INT(y); @@ -1237,7 +1235,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, builder.CreateICmpEQ(xi, yi)); } - HANDLE(fpislt,2) { + case fpislt: { *newtyp = jl_bool_type; Value *xi = JL_INT(x); Value *yi = JL_INT(y); @@ -1264,11 +1262,11 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, ); } - HANDLE(and_int,2) return builder.CreateAnd(JL_INT(x), JL_INT(y)); - HANDLE(or_int,2) return builder.CreateOr(JL_INT(x), JL_INT(y)); - HANDLE(xor_int,2) return builder.CreateXor(JL_INT(x), JL_INT(y)); - HANDLE(not_int,1) return builder.CreateXor(JL_INT(x), ConstantInt::get(t, -1, true)); - HANDLE(shl_int,2) + case and_int: return builder.CreateAnd(JL_INT(x), JL_INT(y)); + case or_int: return builder.CreateOr(JL_INT(x), JL_INT(y)); + case xor_int: return builder.CreateXor(JL_INT(x), JL_INT(y)); + case not_int: return builder.CreateXor(JL_INT(x), ConstantInt::get(t, -1, true)); + case shl_int: x = JL_INT(x); y = JL_INT(y); return builder. CreateSelect(builder. @@ -1276,7 +1274,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, x->getType()->getPrimitiveSizeInBits())), ConstantInt::get(x->getType(),0), builder.CreateShl(x, uint_cnvt(t,y))); - HANDLE(lshr_int,2) + case lshr_int: x = JL_INT(x); y = JL_INT(y); return builder. CreateSelect(builder. @@ -1284,7 +1282,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, x->getType()->getPrimitiveSizeInBits())), ConstantInt::get(x->getType(),0), builder.CreateLShr(x, uint_cnvt(t,y))); - HANDLE(ashr_int,2) + case ashr_int: x = JL_INT(x); y = JL_INT(y); return builder. CreateSelect(builder. @@ -1293,17 +1291,17 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, builder.CreateAShr(x, ConstantInt::get(x->getType(), x->getType()->getPrimitiveSizeInBits()-1)), builder.CreateAShr(x, uint_cnvt(t,y))); - HANDLE(bswap_int,1) + case bswap_int: x = JL_INT(x); return builder.CreateCall( Intrinsic::getDeclaration(jl_Module, Intrinsic::bswap, ArrayRef(x->getType())), x); - HANDLE(ctpop_int,1) + case ctpop_int: x = JL_INT(x); return builder.CreateCall( Intrinsic::getDeclaration(jl_Module, Intrinsic::ctpop, ArrayRef(x->getType())), x); - HANDLE(ctlz_int,1) { + case ctlz_int: { x = JL_INT(x); Type *types[1] = {x->getType()}; Value *ctlz = Intrinsic::getDeclaration(jl_Module, Intrinsic::ctlz, @@ -1314,7 +1312,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, return builder.CreateCall2(ctlz, x, ConstantInt::get(T_int1,0)); #endif } - HANDLE(cttz_int,1) { + case cttz_int: { x = JL_INT(x); Type *types[1] = {x->getType()}; Value *cttz = Intrinsic::getDeclaration(jl_Module, Intrinsic::cttz, ArrayRef(types)); @@ -1325,7 +1323,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, #endif } - HANDLE(nan_dom_err,2) { + case nan_dom_err: { // nan_dom_err(f, x) throw DomainError if isnan(f)&&!isnan(x) Value *f = FP(x); x = FP(y); raise_exception_unless(builder.CreateOr(builder.CreateFCmpORD(f,f), @@ -1334,7 +1332,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, return f; } - HANDLE(abs_float,1) + case abs_float: { x = FP(x); #ifdef LLVM34 @@ -1350,7 +1348,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, return builder.CreateBitCast(absbits, x->getType()); #endif } - HANDLE(copysign_float,2) + case copysign_float: { x = FP(x); fy = FP(y); @@ -1369,7 +1367,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, signbit0))); return builder.CreateBitCast(rbits, x->getType()); } - HANDLE(flipsign_int,2) + case flipsign_int: { x = JL_INT(x); fy = JL_INT(y); @@ -1388,31 +1386,31 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, Value *tmp = builder.CreateAShr(fy, ConstantInt::get(intt,((IntegerType*)intt)->getBitWidth()-1)); return builder.CreateXor(builder.CreateAdd(x,tmp),tmp); } - HANDLE(ceil_llvm,1) { + case ceil_llvm: { x = FP(x); return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::ceil, ArrayRef(x->getType())), x); } - HANDLE(floor_llvm,1) { + case floor_llvm: { x = FP(x); return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::floor, ArrayRef(x->getType())), x); } - HANDLE(trunc_llvm,1) { + case trunc_llvm: { x = FP(x); return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::trunc, ArrayRef(x->getType())), x); } - HANDLE(rint_llvm,1) { + case rint_llvm: { x = FP(x); return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::rint, ArrayRef(x->getType())), x); } - HANDLE(sqrt_llvm,1) { + case sqrt_llvm: { x = FP(x); raise_exception_unless(builder.CreateFCmpUGE(x, ConstantFP::get(x->getType(),0.0)), prepare_global(jldomerr_var), ctx); @@ -1420,7 +1418,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, ArrayRef(x->getType())), x); } - HANDLE(powi_llvm,2) { + case powi_llvm: { x = FP(x); y = JL_INT(y); Type *tx = x->getType(); // TODO: LLVM expects this to be i32 @@ -1441,7 +1439,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, return builder.CreateCall2(pow, x, builder.CreateSIToFP(y, tx)); #endif } - HANDLE(sqrt_llvm_fast,1) { + case sqrt_llvm_fast: { x = FP(x); return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::sqrt, ArrayRef(x->getType())), @@ -1455,8 +1453,6 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, return NULL; } -#undef HANDLE - static Function *boxfunc_llvm(FunctionType *ft, const std::string &cname, void *addr, Module *m) { From 5f327735e63e99734deb856eb4512f92d77e56a6 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 5 Oct 2015 18:34:16 -0400 Subject: [PATCH 0474/1938] introduce a call method for (runtime versions of) intrinsics. use it to have "reinterpret" do runtime error checking (fix #12832) --- src/cgutils.cpp | 49 +++++++++++++-- src/codegen.cpp | 14 +++-- src/dump.c | 1 + src/intrinsics.cpp | 139 ++++++++++++++++++++++++++++++++++++++++++- src/julia_internal.h | 1 + test/numbers.jl | 6 ++ 6 files changed, 198 insertions(+), 12 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 7f0d2c5046bb3..8966741ae4a65 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -871,24 +871,22 @@ static Value *emit_typeof(const jl_cgval_t &p) return literal_pointer_val(aty); } -static Value *emit_datatype_types(const jl_cgval_t &dt) +static Value *emit_datatype_types(Value *dt) { - assert(dt.isboxed); return builder. CreateLoad(builder. CreateBitCast(builder. - CreateGEP(builder.CreateBitCast(dt.V, T_pint8), + CreateGEP(builder.CreateBitCast(dt, T_pint8), ConstantInt::get(T_size, offsetof(jl_datatype_t, types))), T_ppjlvalue)); } -static Value *emit_datatype_nfields(const jl_cgval_t &dt) +static Value *emit_datatype_nfields(Value *dt) { - assert(dt.isboxed); Value *nf = builder. CreateLoad(builder. CreateBitCast(builder. - CreateGEP(builder.CreateBitCast(dt.V, T_pint8), + CreateGEP(builder.CreateBitCast(dt, T_pint8), ConstantInt::get(T_size, offsetof(jl_datatype_t, nfields))), T_pint32)); #ifdef _P64 @@ -897,6 +895,45 @@ static Value *emit_datatype_nfields(const jl_cgval_t &dt) return nf; } +static Value *emit_datatype_size(Value *dt) +{ + Value *size = builder. + CreateLoad(builder. + CreateBitCast(builder. + CreateGEP(builder.CreateBitCast(dt, T_pint8), + ConstantInt::get(T_size, offsetof(jl_datatype_t, size))), + T_pint32)); + return size; +} + +static Value *emit_datatype_mutabl(Value *dt) +{ + Value *mutabl = builder. + CreateLoad(builder.CreateGEP(builder.CreateBitCast(dt, T_pint8), + ConstantInt::get(T_size, offsetof(jl_datatype_t, mutabl)))); + return builder.CreateTrunc(mutabl, T_int1); +} + +static Value *emit_datatype_abstract(Value *dt) +{ + Value *abstract = builder. + CreateLoad(builder.CreateGEP(builder.CreateBitCast(dt, T_pint8), + ConstantInt::get(T_size, offsetof(jl_datatype_t, abstract)))); + return builder.CreateTrunc(abstract, T_int1); +} + +static Value *emit_datatype_isbitstype(Value *dt) +{ + Value *immut = builder.CreateXor(emit_datatype_mutabl(dt), ConstantInt::get(T_int1, -1)); + Value *nofields = builder.CreateICmpEQ(emit_datatype_nfields(dt), ConstantInt::get(T_size, 0)); + Value *isbitstype = builder.CreateAnd(immut, builder.CreateAnd(nofields, + builder.CreateXor(builder.CreateAnd(emit_datatype_abstract(dt), + builder.CreateICmpSGT(emit_datatype_size(dt), ConstantInt::get(T_int32, 0))), + ConstantInt::get(T_int1, -1)))); + return isbitstype; +} + + // --- generating various error checks --- static void just_emit_error(const std::string &txt, jl_codectx_t *ctx) diff --git a/src/codegen.cpp b/src/codegen.cpp index d88be7d4be5ed..220b6573f3b62 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2647,10 +2647,13 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, else if (jl_is_leaf_type(aty)) { jl_cgval_t arg1 = emit_expr(args[1], ctx); Value *sz; - if (aty == (jl_value_t*)jl_datatype_type) - sz = emit_datatype_nfields(arg1); - else + if (aty == (jl_value_t*)jl_datatype_type) { + assert(arg1.isboxed); + sz = emit_datatype_nfields(arg1.V); + } + else { sz = ConstantInt::get(T_size, jl_datatype_nfields(aty)); + } *ret = mark_julia_type(sz, false, jl_long_type); JL_GC_POP(); return true; @@ -2664,8 +2667,9 @@ static bool emit_known_call(jl_cgval_t *ret, jl_value_t *ff, rt2 = expr_type(args[2], ctx); // index argument type if (rt2 == (jl_value_t*)jl_long_type) { jl_cgval_t ty = emit_expr(args[1], ctx); - Value *types_svec = emit_datatype_types(ty); - Value *types_len = emit_datatype_nfields(ty); + assert(ty.isboxed); + Value *types_svec = emit_datatype_types(ty.V); + Value *types_len = emit_datatype_nfields(ty.V); Value *idx = emit_unbox(T_size, emit_unboxed(args[2], ctx), (jl_value_t*)jl_long_type); emit_bounds_check(ty, (jl_value_t*)jl_datatype_type, idx, types_len, ctx); Value *fieldtyp = builder.CreateLoad(builder.CreateGEP(builder.CreateBitCast(types_svec, T_ppjlvalue), idx)); diff --git a/src/dump.c b/src/dump.c index e87b27aac7184..da91aa1f86e95 100644 --- a/src/dump.c +++ b/src/dump.c @@ -78,6 +78,7 @@ static jl_fptr_t id_to_fptrs[] = { jl_f_instantiate_type, jl_f_kwcall, jl_trampoline, jl_f_methodexists, jl_f_applicable, jl_f_invoke, jl_apply_generic, jl_unprotect_stack, jl_f_sizeof, jl_f_new_expr, + jl_f_intrinsic_call, NULL }; // pointers to non-AST-ish objects in a compressed tree diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index e92583d126fe6..2802a51867322 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -52,6 +52,7 @@ namespace JL_I { using namespace JL_I; Function *runtime_func[num_intrinsics]; +void* runtime_fp[num_intrinsics]; unsigned intrinsic_nargs[num_intrinsics]; #include "ccall.cpp" @@ -409,6 +410,94 @@ int get_bitstype_nbits(jl_value_t *bt) // put a bits type tag on some value (despite the name, this doesn't necessarily actually "box" the value however) static jl_cgval_t generic_box(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ctx) +{ + // Examine the first argument // + jl_value_t *bt = static_eval(targ, ctx, true, true); + if (bt && !jl_is_leaf_type(bt)) { + jl_add_linfo_root(ctx->linfo, bt); + } + + if (!bt || !jl_is_bitstype(bt)) { + // it's easier to throw a good error from C than llvm + if (bt) targ = bt; + int last_depth = ctx->gc.argDepth; + Value *arg1 = emit_boxed_rooted(targ, ctx).V; + Value *arg2 = emit_boxed_rooted(x, ctx).V; + Value *func = prepare_call(runtime_func[box]); +#ifdef LLVM37 + Value *r = builder.CreateCall(func, {arg1, arg2}); +#else + Value *r = builder.CreateCall2(func, arg1, arg2); +#endif + ctx->gc.argDepth = last_depth; + jl_value_t *et = expr_type(targ, ctx); + return mark_julia_type(r, true, jl_is_type_type(et) ? jl_tparam0(et) : (jl_value_t*)jl_any_type); + } + + Type *llvmt = staticeval_bitstype(bt); + int nb = jl_datatype_size(bt); + + // Examine the second argument // + jl_cgval_t v = emit_unboxed(x, ctx); + bool isboxed; + Type *vxt = julia_type_to_llvm(v.typ, &isboxed); + + if (!jl_is_datatype(v.typ) + || !jl_is_bitstype(v.typ) + || jl_datatype_size(v.typ) != nb) { + Value *typ = emit_typeof(v); + if (!jl_is_bitstype(v.typ)) { + if (isboxed) { + Value *isbits = emit_datatype_isbitstype(typ); + error_unless(isbits, "reinterpret: expected bitstype value for second argument", ctx); + } + else { + emit_error("reinterpet: expected bitstype value for second argument", ctx); + return jl_cgval_t(); + } + } + if (jl_datatype_size(v.typ) != nb) { + if (isboxed) { + Value *size = emit_datatype_size(typ); + error_unless(builder.CreateICmpEQ(size, ConstantInt::get(T_int32, nb)), + "reinterpet: argument size does not match size of target type", ctx); + } + else { + emit_error("reinterpet: argument size does not match size of target type", ctx); + return jl_cgval_t(); + } + } + } + + Value *vx = v.V; + if (v.ispointer) { + vx = v.V; + if (isboxed) // try to load as original Type, to preserve llvm optimizations + vxt = llvmt; // but if the v.typ is not well known, use T + if (vx->getType()->getPointerElementType() != vxt) + vx = builder.CreatePointerCast(vx, vxt->getPointerTo()); + vx = builder.CreateLoad(vx); + } + + vxt = vx->getType(); + if (vxt != llvmt) { + if (llvmt == T_int1) + vx = builder.CreateTrunc(vx, llvmt); + else if (vxt == T_int1 && llvmt == T_int8) + vx = builder.CreateZExt(vx, llvmt); + else if (vxt->isPointerTy() && !llvmt->isPointerTy()) + vx = builder.CreatePtrToInt(vx, llvmt); + else if (!vxt->isPointerTy() && llvmt->isPointerTy()) + vx = builder.CreateIntToPtr(vx, llvmt); + else + vx = builder.CreateBitCast(vx, llvmt); + } + + return mark_julia_type(vx, false, bt); +} + +// put a bits type tag on some value +static jl_cgval_t generic_unbox(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ctx) { // Examine the first argument // jl_value_t *bt = staticeval_bitstype(targ, NULL, ctx); @@ -857,18 +946,30 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, Value *func = prepare_call(runtime_func[f]); if (nargs == 1) { Value *x = emit_boxed_rooted(args[1], ctx).V; +#ifdef LLVM37 + r = builder.CreateCall(func, {x}); +#else r = builder.CreateCall(func, x); +#endif } else if (nargs == 2) { Value *x = emit_boxed_rooted(args[1], ctx).V; Value *y = emit_boxed_rooted(args[2], ctx).V; +#ifdef LLVM37 + r = builder.CreateCall(func, {x, y}); +#else r = builder.CreateCall2(func, x, y); +#endif } else if (nargs == 3) { Value *x = emit_boxed_rooted(args[1], ctx).V; Value *y = emit_boxed_rooted(args[2], ctx).V; Value *z = emit_boxed_rooted(args[3], ctx).V; +#ifdef LLVM37 + r = builder.CreateCall(func, {x, y, z}); +#else r = builder.CreateCall3(func, x, y, z); +#endif } else { assert(0); @@ -883,7 +984,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, case box: return generic_box(args[1], args[2], ctx); case unbox: - return generic_box(args[1], args[2], ctx); + return generic_unbox(args[1], args[2], ctx); // TODO: replace with generic_box case trunc_int: return generic_trunc(args[1], args[2], ctx, false, false); case checked_trunc_sint: @@ -1453,6 +1554,36 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, return NULL; } +typedef jl_value_t *(*intrinsic_call_1_arg)(jl_value_t*); +typedef jl_value_t *(*intrinsic_call_2_arg)(jl_value_t*, jl_value_t*); +typedef jl_value_t *(*intrinsic_call_3_arg)(jl_value_t*, jl_value_t*, jl_value_t*); +#define jl_is_intrinsic(v) jl_typeis(v,jl_intrinsic_type) + +JL_CALLABLE(jl_f_intrinsic_call) +{ + JL_NARGSV(intrinsic_call, 1); + JL_TYPECHK(intrinsic_call, intrinsic, args[0]); + intrinsic f = (intrinsic)*(uint32_t*)jl_data_ptr(args[0]); + if (f == fptoui && nargs == 1) + f = fptoui_auto; + if (f == fptosi && nargs == 1) + f = fptosi_auto; + unsigned fargs = intrinsic_nargs[f]; + JL_NARGS(intrinsic_call, 1 + fargs, 1 + fargs); + switch (fargs) { + case 1: + return ((intrinsic_call_1_arg)runtime_fp[f])(args[1]); + case 2: + return ((intrinsic_call_2_arg)runtime_fp[f])(args[1], args[2]); + case 3: + return ((intrinsic_call_3_arg)runtime_fp[f])(args[1], args[2], args[3]); + default: + assert(0 && "unexpected number of arguments to an intrinsic function"); + } + abort(); +} + + static Function *boxfunc_llvm(FunctionType *ft, const std::string &cname, void *addr, Module *m) { @@ -1540,6 +1671,9 @@ extern "C" void jl_init_intrinsic_functions(void) //ADD_I(fptosi_auto); ADD_I(fptoui_auto); // these intrinsics are "hidden" in fpto*i ADD_I(ccall); ADD_I(cglobal); ADD_I(llvmcall); + + jl_set_const(inm, jl_symbol("intrinsic_call"), + (jl_value_t*)jl_new_closure(jl_f_intrinsic_call, (jl_value_t*)jl_symbol("intrinsic_call"), NULL)); } #undef ADD_I @@ -1550,12 +1684,14 @@ static void add_intrinsic_to_codegen(Module *m, const std::string &name, intrins runtime_func[f] = func; add_named_global(func, pfunc); intrinsic_nargs[f] = nargs; + runtime_fp[f] = pfunc; } static void add_intrinsic_to_codegen(intrinsic alias, intrinsic base) { runtime_func[alias] = runtime_func[base]; intrinsic_nargs[alias] = intrinsic_nargs[base]; + runtime_fp[alias] = runtime_fp[base]; } #define ADD_I(name, nargs) add_intrinsic_to_codegen(m, "jl_" #name, name, nargs, args##nargs, (void*)&jl_##name) @@ -1613,6 +1749,7 @@ static void jl_init_intrinsic_functions_codegen(Module *m) ADD_I(check_top_bit, 1); ADD_I(nan_dom_err, 2); ADD_I(fptosi_auto, 1); ADD_I(fptoui_auto, 1); + } #undef ADD_I diff --git a/src/julia_internal.h b/src/julia_internal.h index d3798c1964f96..86490022e2340 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -75,6 +75,7 @@ JL_CALLABLE(jl_apply_generic); JL_CALLABLE(jl_unprotect_stack); JL_CALLABLE(jl_f_no_function); JL_CALLABLE(jl_f_tuple); +JL_CALLABLE(jl_f_intrinsic_call); extern jl_function_t *jl_unprotect_stack_func; extern jl_function_t *jl_bottom_func; void jl_install_default_signal_handlers(void); diff --git a/test/numbers.jl b/test/numbers.jl index 76b10e3f0fe32..aa78abd831b93 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2353,6 +2353,12 @@ end # issue #7508 @test_throws ErrorException reinterpret(Int, 0x01) +# issue #12832 +@test_throws ErrorException reinterpret(Float64, Complex{Int64}(1)) +@test_throws ErrorException reinterpret(Float64, Complex64(1)) +@test_throws ErrorException reinterpret(Complex64, Float64(1)) +@test_throws ErrorException reinterpret(Int32, false) + # issue #41 ndigf(n) = Float64(log(Float32(n))) @test Float64(log(Float32(256))) == ndigf(256) == 5.545177459716797 From 8d840bde783301f10db4b7ab0e6ffe59fe25b719 Mon Sep 17 00:00:00 2001 From: Kiran Pamnany Date: Wed, 7 Oct 2015 09:48:30 -0700 Subject: [PATCH 0475/1938] Change Lock to AbstractLock and fix compile error --- base/locks.jl | 8 ++++---- src/threading.c | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/base/locks.jl b/base/locks.jl index 232ccf1a105a7..0629f844850a9 100644 --- a/base/locks.jl +++ b/base/locks.jl @@ -2,12 +2,12 @@ export SpinLock, Mutex, init_lock!, destroy_lock!, lock!, trylock!, unlock! -abstract Lock +abstract AbstractLock # Test-and-test-and-set spin locks are quickest up to about 30ish # contending threads. If you have more contention than that, perhaps # a lock is the wrong way to synchronize. -type TatasLock <: Lock +type TatasLock <: AbstractLock handle::Atomic{Int} TatasLock() = new(Atomic{Int}(0)) end @@ -40,7 +40,7 @@ end # Recursive test-and-test-and-set lock. Slower. -type RecursiveTatasLock <: Lock +type RecursiveTatasLock <: AbstractLock ownertid::Atomic{Int16} handle::Atomic{Int} RecursiveTatasLock() = new(0, Atomic{Int}(0)) @@ -98,7 +98,7 @@ end const UV_MUTEX_SIZE = ccall(:jl_sizeof_uv_mutex, Cint, ()) -type Mutex <: Lock +type Mutex <: AbstractLock ownertid::Int16 handle::Array{Int8} Mutex() = (m = new(zero(Int16), zeros(Int8, UV_MUTEX_SIZE)); diff --git a/src/threading.c b/src/threading.c index 670670185fb07..808a0ec95a04c 100644 --- a/src/threading.c +++ b/src/threading.c @@ -324,7 +324,7 @@ DLLEXPORT jl_value_t *jl_threading_run(jl_function_t *f, jl_svec_t *args) if ((jl_value_t*)args == jl_emptytuple) args = jl_emptysvec; JL_TYPECHK(jl_threading_run, function, (jl_value_t*)f); - JL_TYPECHK(jl_threading_run, svec, (jl_value_t*)args); + JL_TYPECHK(jl_threading_run, simplevector, (jl_value_t*)args); JL_GC_PUSH2(&argtypes, &fun); if (jl_svec_len(args) == 0) From ac66b65212c273cbbf2934b219b25eafb384e2d6 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 6 Oct 2015 16:09:47 -0400 Subject: [PATCH 0476/1938] create a compile option for a Julia that is not linked against llvm (other than libLLVMSupport.a for APInt support) --- Make.inc | 1 + src/Makefile | 39 +++--- src/alloc.c | 9 ++ src/anticodegen.c | 50 ++++++++ src/ccall.cpp | 128 +------------------ src/codegen.cpp | 20 --- src/intrinsics.cpp | 264 ++++++--------------------------------- src/intrinsics.h | 207 ++++++++++++++++++++++++++++++ src/julia_internal.h | 10 +- src/runtime_ccall.cpp | 140 +++++++++++++++++++++ src/runtime_intrinsics.c | 2 +- src/sys.c | 14 +-- 12 files changed, 485 insertions(+), 399 deletions(-) create mode 100644 src/anticodegen.c create mode 100644 src/intrinsics.h create mode 100644 src/runtime_ccall.cpp diff --git a/Make.inc b/Make.inc index 9e234c2ac1be1..4c1804c8fc707 100644 --- a/Make.inc +++ b/Make.inc @@ -266,6 +266,7 @@ EXE := endif JULIAGC := MARKSWEEP +JULIACODEGEN := LLVM USE_COPY_STACKS := 1 # flag for disabling assertions diff --git a/src/Makefile b/src/Makefile index 2731b897445e2..5f3cab29a04e2 100644 --- a/src/Makefile +++ b/src/Makefile @@ -11,16 +11,6 @@ override CFLAGS += $(JCFLAGS) override CXXFLAGS += $(JCXXFLAGS) override CPPFLAGS += $(JCPPFLAGS) -SRCS := \ - jltypes gf ast builtins module codegen disasm debuginfo interpreter \ - alloc dlload sys init task array dump toplevel jl_uv jlapi signal-handling \ - llvm-simdloop simplevector APInt-C runtime_intrinsics -ifeq ($(JULIAGC),MARKSWEEP) -SRCS += gc -endif - -HEADERS := $(addprefix $(SRCDIR)/,julia.h julia_internal.h options.h) $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(LIBUV_INC)/uv.h - # -I BUILDDIR comes before -I SRCDIR so that the user can override on a per-build-directory basis # for gcc/clang, suggested content is: # #include_next @@ -28,17 +18,37 @@ HEADERS := $(addprefix $(SRCDIR)/,julia.h julia_internal.h options.h) $(BUILDDIR FLAGS := \ -D_GNU_SOURCE -I$(BUILDDIR) -I$(SRCDIR) \ -I$(SRCDIR)/flisp -I$(SRCDIR)/support \ - -I$(shell $(LLVM_CONFIG_HOST) --includedir) \ -I$(LIBUV_INC) -I$(build_includedir) -DLIBRARY_EXPORTS \ -I$(JULIAHOME)/deps/valgrind ifneq ($(USEMSVC), 1) FLAGS += -Wall -Wno-strict-aliasing -fno-omit-frame-pointer -fvisibility=hidden -fno-common endif + +SRCS := \ + jltypes gf ast builtins module interpreter \ + alloc dlload sys init task array dump toplevel jl_uv jlapi signal-handling \ + simplevector APInt-C runtime_intrinsics runtime_ccall +ifeq ($(JULIAGC),MARKSWEEP) +SRCS += gc +endif + +ifeq ($(JULIACODEGEN),LLVM) +SRCS += codegen disasm debuginfo llvm-simdloop +FLAGS += -I$(shell $(LLVM_CONFIG_HOST) --includedir) +LLVM_LIBS := all +else +SRCS += anticodegen +LLVM_LIBS := support +endif + +HEADERS := $(addprefix $(SRCDIR)/,julia.h julia_internal.h options.h) $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(LIBUV_INC)/uv.h + # In LLVM < 3.4, --ldflags includes both options and libraries, so use it both before and after --libs # In LLVM >= 3.4, --ldflags has only options, and --system-libs has the libraries. -LLVMLINK := $(shell $(LLVM_CONFIG_HOST) --ldflags) $(shell $(LLVM_CONFIG_HOST) --libs) $(shell $(LLVM_CONFIG_HOST) --ldflags) $(shell $(LLVM_CONFIG_HOST) --system-libs 2> /dev/null) -ifeq ($(USE_LLVM_SHLIB),1) +ifneq ($(USE_LLVM_SHLIB),1) +LLVMLINK := $(shell $(LLVM_CONFIG_HOST) --ldflags) $(shell $(LLVM_CONFIG_HOST) --libs $(LLVM_LIBS)) $(shell $(LLVM_CONFIG_HOST) --ldflags) $(shell $(LLVM_CONFIG_HOST) --system-libs 2> /dev/null) +else ifeq ($(LLVM_USE_CMAKE),1) LLVMLINK := $(shell $(LLVM_CONFIG_HOST) --ldflags) -lLLVM else @@ -102,7 +112,8 @@ $(BUILDDIR)/julia_flisp.boot: $(addprefix $(SRCDIR)/,jlfrontend.scm \ $(call cygpath_w,$(SRCDIR)/mk_julia_flisp_boot.scm) $(call cygpath_w,$(dir $<)) $(notdir $<) $(call cygpath_w,$@)) $(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc $(SRCDIR)/flisp/*.h -$(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,intrinsics.cpp cgutils.cpp ccall.cpp abi_*.cpp) +$(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,intrinsics.cpp intrinsics.h cgutils.cpp ccall.cpp abi_*.cpp) +$(BUILDDIR)/anticodegen.o $(BUILDDIR)/anticodegen.dbg.obj: $(SRCDIR)/intrinsics.h $(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: $(SRCDIR)/table.c $(BUILDDIR)/gc.o $(BUILDDIR)/gc.dbg.obj: $(SRCDIR)/gc-debug.c $(BUILDDIR)/signal-handling.o $(BUILDDIR)/signal-handling.dbg.obj: $(addprefix $(SRCDIR)/,signals-*.c) diff --git a/src/alloc.c b/src/alloc.c index 52be1ed3b2a2b..02b51a8dcf350 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -819,6 +819,15 @@ jl_value_t *jl_box_bool(int8_t x) return jl_false; } +DLLEXPORT jl_value_t *jl_new_box(jl_value_t *v) +{ + jl_value_t *box = (jl_value_t*)jl_gc_alloc_1w(); + jl_set_typeof(box, jl_box_any_type); + // if (v) jl_gc_wb(box, v); // write block not needed: box was just allocated + box->fieldptr[0] = v; + return box; +} + // Expr constructor for internal use ------------------------------------------ jl_expr_t *jl_exprn(jl_sym_t *head, size_t n) diff --git a/src/anticodegen.c b/src/anticodegen.c new file mode 100644 index 0000000000000..dc78e22d85487 --- /dev/null +++ b/src/anticodegen.c @@ -0,0 +1,50 @@ +#include "julia.h" +#include "julia_internal.h" + +#include "intrinsics.h" + +int globalUnique = 0; + +#define UNAVAILABLE { jl_errorf("%s: not available in this build of Julia", __func__); } + +void jl_dump_bitcode(char *fname, const char *sysimg_data, size_t sysimg_len) UNAVAILABLE +void jl_dump_objfile(char *fname, int jit_model, const char *sysimg_data, size_t sysimg_len) UNAVAILABLE +int32_t jl_get_llvm_gv(jl_value_t *p) UNAVAILABLE +void jl_write_malloc_log(void) UNAVAILABLE +void jl_write_coverage_data(void) UNAVAILABLE +void jl_generate_fptr(jl_function_t *f) { + jl_lambda_info_t *li = f->linfo; + if (li->fptr == &jl_trampoline) UNAVAILABLE + f->fptr = li->fptr; +} + +DLLEXPORT void jl_clear_malloc_data(void) UNAVAILABLE +DLLEXPORT void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) UNAVAILABLE +DLLEXPORT void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt) UNAVAILABLE +DLLEXPORT const jl_value_t *jl_dump_function_asm(void *f, int raw_mc) UNAVAILABLE +DLLEXPORT const jl_value_t *jl_dump_function_ir(void *f, uint8_t strip_ir_metadata, uint8_t dump_module) UNAVAILABLE + +void jl_init_codegen(void) { } +void jl_compile(jl_function_t *f) { } +void jl_fptr_to_llvm(void *fptr, jl_lambda_info_t *lam, int specsig) +{ + if (!specsig) + lam->fptr = (jl_fptr_t)fptr; +} +void jl_getFunctionInfo(char **name, char **filename, size_t *line, + char **inlinedat_file, size_t *inlinedat_line, + size_t pointer, int *fromC, int skipC, int skipInline) +{ + *name = NULL; + *line = -1; + *filename = NULL; + *inlinedat_file = NULL; + *inlinedat_line = -1; + *fromC = 0; +} + +jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod, + jl_value_t *sp, jl_expr_t *ast, int sparams, int allow_alloc) +{ + return NULL; +} diff --git a/src/ccall.cpp b/src/ccall.cpp index 819818f1bb9f9..7c14ce160750d 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1,126 +1,6 @@ // This file is a part of Julia. License is MIT: http://julialang.org/license -// --- the ccall intrinsic --- - -// --- library symbol lookup --- - -// map from "libX" to full soname "libX.so.ver" -#if defined(__linux__) || defined(__FreeBSD__) -static std::map sonameMap; -static bool got_sonames = false; - -extern "C" DLLEXPORT void jl_read_sonames(void) -{ - char *line=NULL; - size_t sz=0; -#if defined(__linux__) - FILE *ldc = popen("/sbin/ldconfig -p", "r"); -#else - FILE *ldc = popen("/sbin/ldconfig -r", "r"); -#endif - - while (!feof(ldc)) { - ssize_t n = getline(&line, &sz, ldc); - if (n == -1) - break; - if (n > 2 && isspace((unsigned char)line[0])) { -#ifdef __linux__ - int i = 0; - while (isspace((unsigned char)line[++i])) ; - char *name = &line[i]; - char *dot = strstr(name, ".so"); - i = 0; -#else - char *name = strstr(line, ":-l"); - if (name == NULL) continue; - strncpy(name, "lib", 3); - char *dot = strchr(name, '.'); -#endif - - if (NULL == dot) - continue; - -#ifdef __linux__ - // Detect if this entry is for the current architecture - while (!isspace((unsigned char)dot[++i])) ; - while (isspace((unsigned char)dot[++i])) ; - int j = i; - while (!isspace((unsigned char)dot[++j])) ; - char *arch = strstr(dot+i,"x86-64"); - if (arch != NULL && arch < dot + j) { -#ifdef _P32 - continue; -#endif - } - else { -#ifdef _P64 - continue; -#endif - } -#endif // __linux__ - - char *abslibpath = strrchr(line, ' '); - if (dot != NULL && abslibpath != NULL) { - std::string pfx(name, dot - name); - // Do not include ' ' in front and '\n' at the end - std::string soname(abslibpath+1, line+n-(abslibpath+1)-1); - sonameMap[pfx] = soname; - } - } - } - - free(line); - pclose(ldc); -} - -extern "C" DLLEXPORT const char *jl_lookup_soname(const char *pfx, size_t n) -{ - if (!got_sonames) { - jl_read_sonames(); - got_sonames = true; - } - std::string str(pfx, n); - if (sonameMap.find(str) != sonameMap.end()) { - return sonameMap[str].c_str(); - } - return NULL; -} -#endif - -// map from user-specified lib names to handles -static std::map libMap; - -static uv_lib_t *get_library(char *lib) -{ - uv_lib_t *hnd; -#ifdef _OS_WINDOWS_ - if ((intptr_t)lib == 1) - return jl_exe_handle; - if ((intptr_t)lib == 2) - return jl_dl_handle; -#endif - if (lib == NULL) - return jl_RTLD_DEFAULT_handle; - hnd = libMap[lib]; - if (hnd != NULL) - return hnd; - hnd = (uv_lib_t *) jl_load_dynamic_library(lib, JL_RTLD_DEFAULT); - if (hnd != NULL) - libMap[lib] = hnd; - return hnd; -} - -extern "C" DLLEXPORT -void *jl_load_and_lookup(char *f_lib, char *f_name, uv_lib_t **hnd) -{ - uv_lib_t *handle = *hnd; - if (!handle) - *hnd = handle = get_library(f_lib); - void *ptr = jl_dlsym_e(handle, f_name); - if (!ptr) - jl_errorf("symbol \"%s\" could not be found: %s", f_name, uv_dlerror(handle)); - return ptr; -} +// --- the ccall, cglobal, and llvm intrinsics --- static std::map libMapGV; static std::map symMapGV; @@ -161,7 +41,7 @@ static Value *runtime_sym_lookup(PointerType *funcptype, char *f_lib, char *f_na false, GlobalVariable::PrivateLinkage, initnul, f_lib); libMapGV[f_lib] = libptrgv; - libsym = get_library(f_lib); + libsym = jl_get_library(f_lib); assert(libsym != NULL); #ifdef USE_MCJIT jl_llvm_to_jl_value[libptrgv] = libsym; @@ -565,7 +445,7 @@ static jl_cgval_t emit_cglobal(jl_value_t **args, size_t nargs, jl_codectx_t *ct res = runtime_sym_lookup((PointerType*)lrt, sym.f_lib, sym.f_name, ctx); } else { - void *symaddr = jl_dlsym_e(get_library(sym.f_lib), sym.f_name); + void *symaddr = jl_dlsym_e(jl_get_library(sym.f_lib), sym.f_name); if (symaddr == NULL) { std::stringstream msg; msg << "cglobal: could not find symbol "; @@ -1370,7 +1250,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) llvmf = runtime_sym_lookup(funcptype, f_lib, f_name, ctx); } else { - void *symaddr = jl_dlsym_e(get_library(f_lib), f_name); + void *symaddr = jl_dlsym_e(jl_get_library(f_lib), f_name); if (symaddr == NULL) { JL_GC_POP(); std::stringstream msg; diff --git a/src/codegen.cpp b/src/codegen.cpp index 220b6573f3b62..923bc06f99e1c 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -658,17 +658,6 @@ extern "C" { int globalUnique = 0; } -extern "C" DLLEXPORT -jl_value_t *jl_get_cpu_name(void) -{ -#if LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR < 5 - std::string HostCPUName = llvm::sys::getHostCPUName(); -#else - StringRef HostCPUName = llvm::sys::getHostCPUName(); -#endif - return jl_pchar_to_string(HostCPUName.data(), HostCPUName.size()); -} - static void emit_write_barrier(jl_codectx_t*, Value*, Value*); #include "cgutils.cpp" @@ -5208,15 +5197,6 @@ extern "C" void jl_fptr_to_llvm(void *fptr, jl_lambda_info_t *lam, int specsig) } } -extern "C" DLLEXPORT jl_value_t *jl_new_box(jl_value_t *v) -{ - jl_value_t *box = (jl_value_t*)jl_gc_alloc_1w(); - jl_set_typeof(box, jl_box_any_type); - // if (v) jl_gc_wb(box, v); // write block not needed: box was just allocated - box->fieldptr[0] = v; - return box; -} - #if LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR == 3 && SYSTEM_LLVM #define INSTCOMBINE_BUG #define V128_BUG diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 2802a51867322..88f7e1734d877 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -1,61 +1,39 @@ // This file is a part of Julia. License is MIT: http://julialang.org/license namespace JL_I { - enum intrinsic { - // wrap and unwrap - box=0, unbox, - // arithmetic - neg_int, add_int, sub_int, mul_int, - sdiv_int, udiv_int, srem_int, urem_int, smod_int, - neg_float, add_float, sub_float, mul_float, div_float, rem_float, - fma_float, muladd_float, - // fast arithmetic - neg_float_fast, add_float_fast, sub_float_fast, - mul_float_fast, div_float_fast, rem_float_fast, - // same-type comparisons - eq_int, ne_int, - slt_int, ult_int, - sle_int, ule_int, - eq_float, ne_float, - lt_float, le_float, - eq_float_fast, ne_float_fast, - lt_float_fast, le_float_fast, - fpiseq, fpislt, - // bitwise operators - and_int, or_int, xor_int, not_int, shl_int, lshr_int, ashr_int, - bswap_int, ctpop_int, ctlz_int, cttz_int, - // conversion - sext_int, zext_int, trunc_int, - fptoui, fptosi, uitofp, sitofp, - fptrunc, fpext, - // checked conversion - checked_fptosi, checked_fptoui, - checked_trunc_sint, checked_trunc_uint, check_top_bit, - // checked arithmetic - checked_sadd, checked_uadd, checked_ssub, checked_usub, - checked_smul, checked_umul, - nan_dom_err, - // functions - abs_float, copysign_float, flipsign_int, select_value, - ceil_llvm, floor_llvm, trunc_llvm, rint_llvm, - sqrt_llvm, powi_llvm, - sqrt_llvm_fast, - // pointer access - pointerref, pointerset, - // c interface - ccall, cglobal, llvmcall, - // terminator - fptoui_auto, fptosi_auto, - num_intrinsics - }; +#include "intrinsics.h" } +#include "ccall.cpp" + using namespace JL_I; -Function *runtime_func[num_intrinsics]; -void* runtime_fp[num_intrinsics]; -unsigned intrinsic_nargs[num_intrinsics]; +static Function *runtime_func[num_intrinsics]; +static void jl_init_intrinsic_functions_codegen(Module *m) +{ + std::vector args1(0); \ + args1.push_back(T_pjlvalue); \ + std::vector args2(0); \ + args2.push_back(T_pjlvalue); \ + args2.push_back(T_pjlvalue); \ + std::vector args3(0); \ + args3.push_back(T_pjlvalue); \ + args3.push_back(T_pjlvalue); \ + args3.push_back(T_pjlvalue); -#include "ccall.cpp" +#define ADD_I(name, nargs) do { \ + Function *func = Function::Create(FunctionType::get(T_pjlvalue, args##nargs, false), \ + Function::ExternalLinkage, "jl_"#name, m); \ + runtime_func[name] = func; \ + add_named_global(func, (void*)&jl_##name); \ + } while (0); +#define ADD_HIDDEN ADD_I +#define ALIAS(alias, base) runtime_func[alias] = runtime_func[base]; + ADD_HIDDEN(reinterpret, 2); + INTRINSICS +#undef ADD_I +#undef ADD_HIDDEN +#undef ALIAS +} /* low-level intrinsics design: TODO: fix description below @@ -423,7 +401,7 @@ static jl_cgval_t generic_box(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ctx int last_depth = ctx->gc.argDepth; Value *arg1 = emit_boxed_rooted(targ, ctx).V; Value *arg2 = emit_boxed_rooted(x, ctx).V; - Value *func = prepare_call(runtime_func[box]); + Value *func = prepare_call(runtime_func[reinterpret]); #ifdef LLVM37 Value *r = builder.CreateCall(func, {arg1, arg2}); #else @@ -932,7 +910,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, f = fptosi_auto; unsigned expected_nargs = intrinsic_nargs[f]; if (expected_nargs && expected_nargs != nargs) { - jl_errorf("intrinsic #%d: wrong number of arguments", f); + jl_errorf("intrinsic #%d %s: wrong number of arguments", f, jl_intrinsic_name((int)f)); } switch (f) { @@ -1554,35 +1532,12 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, return NULL; } -typedef jl_value_t *(*intrinsic_call_1_arg)(jl_value_t*); -typedef jl_value_t *(*intrinsic_call_2_arg)(jl_value_t*, jl_value_t*); -typedef jl_value_t *(*intrinsic_call_3_arg)(jl_value_t*, jl_value_t*, jl_value_t*); -#define jl_is_intrinsic(v) jl_typeis(v,jl_intrinsic_type) - -JL_CALLABLE(jl_f_intrinsic_call) -{ - JL_NARGSV(intrinsic_call, 1); - JL_TYPECHK(intrinsic_call, intrinsic, args[0]); - intrinsic f = (intrinsic)*(uint32_t*)jl_data_ptr(args[0]); - if (f == fptoui && nargs == 1) - f = fptoui_auto; - if (f == fptosi && nargs == 1) - f = fptosi_auto; - unsigned fargs = intrinsic_nargs[f]; - JL_NARGS(intrinsic_call, 1 + fargs, 1 + fargs); - switch (fargs) { - case 1: - return ((intrinsic_call_1_arg)runtime_fp[f])(args[1]); - case 2: - return ((intrinsic_call_2_arg)runtime_fp[f])(args[1], args[2]); - case 3: - return ((intrinsic_call_3_arg)runtime_fp[f])(args[1], args[2], args[3]); - default: - assert(0 && "unexpected number of arguments to an intrinsic function"); - } - abort(); -} +#define BOX_F(ct,jl_ct) \ + box_##ct##_func = boxfunc_llvm(ft1arg(T_pjlvalue, T_##jl_ct), \ + "jl_box_"#ct, (void*)&jl_box_##ct, m); +#define SBOX_F(ct,jl_ct) BOX_F(ct,jl_ct); box_##ct##_func->addAttribute(1, Attribute::SExt); +#define UBOX_F(ct,jl_ct) BOX_F(ct,jl_ct); box_##ct##_func->addAttribute(1, Attribute::ZExt); static Function *boxfunc_llvm(FunctionType *ft, const std::string &cname, void *addr, Module *m) @@ -1607,150 +1562,3 @@ static FunctionType *ft2arg(Type *ret, Type *arg1, Type *arg2) args2.push_back(arg2); return FunctionType::get(ret, args2, false); } - -#define BOX_F(ct,jl_ct) \ - box_##ct##_func = boxfunc_llvm(ft1arg(T_pjlvalue, T_##jl_ct), \ - "jl_box_"#ct, (void*)&jl_box_##ct, m); - -#define SBOX_F(ct,jl_ct) BOX_F(ct,jl_ct); box_##ct##_func->addAttribute(1, Attribute::SExt); -#define UBOX_F(ct,jl_ct) BOX_F(ct,jl_ct); box_##ct##_func->addAttribute(1, Attribute::ZExt); - -static void add_intrinsic(jl_module_t *m, const std::string &name, intrinsic f) -{ - jl_value_t *i = jl_box32(jl_intrinsic_type, (int32_t)f); - jl_sym_t *sym = jl_symbol(const_cast(name.c_str())); - jl_set_const(m, sym, i); - jl_module_export(m, sym); -} - -#define ADD_I(name) add_intrinsic(inm, #name, name) - -extern "C" void jl_init_intrinsic_functions(void) -{ - jl_module_t *inm = jl_new_module(jl_symbol("Intrinsics")); - inm->parent = jl_core_module; - jl_set_const(jl_core_module, jl_symbol("Intrinsics"), (jl_value_t*)inm); - - ADD_I(box); ADD_I(unbox); - ADD_I(neg_int); ADD_I(add_int); ADD_I(sub_int); ADD_I(mul_int); - ADD_I(sdiv_int); ADD_I(udiv_int); ADD_I(srem_int); ADD_I(urem_int); - ADD_I(smod_int); - ADD_I(neg_float); ADD_I(add_float); ADD_I(sub_float); ADD_I(mul_float); - ADD_I(div_float); ADD_I(rem_float); ADD_I(fma_float); ADD_I(muladd_float); - ADD_I(neg_float_fast); ADD_I(add_float_fast); ADD_I(sub_float_fast); - ADD_I(mul_float_fast); ADD_I(div_float_fast); ADD_I(rem_float_fast); - ADD_I(eq_int); ADD_I(ne_int); - ADD_I(slt_int); ADD_I(ult_int); - ADD_I(sle_int); ADD_I(ule_int); - ADD_I(eq_float); ADD_I(ne_float); - ADD_I(lt_float); ADD_I(le_float); - ADD_I(eq_float_fast); ADD_I(ne_float_fast); - ADD_I(lt_float_fast); ADD_I(le_float_fast); - ADD_I(fpiseq); ADD_I(fpislt); - ADD_I(and_int); ADD_I(or_int); ADD_I(xor_int); ADD_I(not_int); - ADD_I(shl_int); ADD_I(lshr_int); ADD_I(ashr_int); ADD_I(bswap_int); - ADD_I(ctpop_int); ADD_I(ctlz_int); ADD_I(cttz_int); - ADD_I(sext_int); ADD_I(zext_int); ADD_I(trunc_int); - ADD_I(fptoui); ADD_I(fptosi); - ADD_I(uitofp); ADD_I(sitofp); - ADD_I(fptrunc); ADD_I(fpext); - ADD_I(abs_float); ADD_I(copysign_float); - ADD_I(flipsign_int); ADD_I(select_value); - ADD_I(ceil_llvm); ADD_I(floor_llvm); ADD_I(trunc_llvm); ADD_I(rint_llvm); - ADD_I(sqrt_llvm); ADD_I(powi_llvm); - ADD_I(sqrt_llvm_fast); - ADD_I(pointerref); ADD_I(pointerset); - ADD_I(checked_sadd); ADD_I(checked_uadd); - ADD_I(checked_ssub); ADD_I(checked_usub); - ADD_I(checked_smul); ADD_I(checked_umul); - ADD_I(checked_fptosi); ADD_I(checked_fptoui); - ADD_I(checked_trunc_sint); - ADD_I(checked_trunc_uint); - ADD_I(check_top_bit); - ADD_I(nan_dom_err); - //ADD_I(fptosi_auto); ADD_I(fptoui_auto); // these intrinsics are "hidden" in fpto*i - ADD_I(ccall); ADD_I(cglobal); - ADD_I(llvmcall); - - jl_set_const(inm, jl_symbol("intrinsic_call"), - (jl_value_t*)jl_new_closure(jl_f_intrinsic_call, (jl_value_t*)jl_symbol("intrinsic_call"), NULL)); -} -#undef ADD_I - -static void add_intrinsic_to_codegen(Module *m, const std::string &name, intrinsic f, - unsigned nargs, std::vector args, void *pfunc) { - Function *func = Function::Create(FunctionType::get(T_pjlvalue, args, false), - Function::ExternalLinkage, name, m); - runtime_func[f] = func; - add_named_global(func, pfunc); - intrinsic_nargs[f] = nargs; - runtime_fp[f] = pfunc; -} - -static void add_intrinsic_to_codegen(intrinsic alias, intrinsic base) -{ - runtime_func[alias] = runtime_func[base]; - intrinsic_nargs[alias] = intrinsic_nargs[base]; - runtime_fp[alias] = runtime_fp[base]; -} - -#define ADD_I(name, nargs) add_intrinsic_to_codegen(m, "jl_" #name, name, nargs, args##nargs, (void*)&jl_##name) -#define ALIAS(alias, base) add_intrinsic_to_codegen(alias, base) - -static void jl_init_intrinsic_functions_codegen(Module *m) -{ - std::vector args1(0); - args1.push_back(T_pjlvalue); - std::vector args2(0); - args2.push_back(T_pjlvalue); - args2.push_back(T_pjlvalue); - std::vector args3(0); - args3.push_back(T_pjlvalue); - args3.push_back(T_pjlvalue); - args3.push_back(T_pjlvalue); - - add_intrinsic_to_codegen(m, "jl_reinterpret", box, - 2, args2, (void*)&jl_reinterpret); - ALIAS(unbox, box); - ADD_I(neg_int, 1); ADD_I(add_int, 2); ADD_I(sub_int, 2); ADD_I(mul_int, 2); - ADD_I(sdiv_int, 2); ADD_I(udiv_int, 2); ADD_I(srem_int, 2); ADD_I(urem_int, 2); - ADD_I(smod_int, 2); - ADD_I(neg_float, 1); ADD_I(add_float, 2); ADD_I(sub_float, 2); ADD_I(mul_float, 2); - ADD_I(div_float, 2); ADD_I(rem_float, 2); ADD_I(fma_float, 3); ADD_I(muladd_float, 3); - ALIAS(neg_float_fast, neg_float); ALIAS(add_float_fast, add_float); ALIAS(sub_float_fast, sub_float); - ALIAS(mul_float_fast, mul_float); ALIAS(div_float_fast, div_float); ALIAS(rem_float_fast, rem_float); - ADD_I(eq_int, 2); ADD_I(ne_int, 2); - ADD_I(slt_int, 2); ADD_I(ult_int, 2); - ADD_I(sle_int, 2); ADD_I(ule_int, 2); - ADD_I(eq_float, 2); ADD_I(ne_float, 2); - ADD_I(lt_float, 2); ADD_I(le_float, 2); - ALIAS(eq_float_fast, eq_float); ALIAS(ne_float_fast, ne_float); - ALIAS(lt_float_fast, lt_float); ALIAS(le_float_fast, le_float); - ADD_I(fpiseq, 2); ADD_I(fpislt, 2); - ADD_I(and_int, 2); ADD_I(or_int, 2); ADD_I(xor_int, 2); ADD_I(not_int, 1); - ADD_I(shl_int, 2); ADD_I(lshr_int, 2); ADD_I(ashr_int, 2); ADD_I(bswap_int, 1); - ADD_I(ctpop_int, 1); ADD_I(ctlz_int, 1); ADD_I(cttz_int, 1); - ADD_I(sext_int, 2); ADD_I(zext_int, 2); ADD_I(trunc_int, 2); - ADD_I(fptoui, 2); ADD_I(fptosi, 2); - ADD_I(uitofp, 2); ADD_I(sitofp, 2); - ADD_I(fptrunc, 2); ADD_I(fpext, 2); - ADD_I(abs_float, 1); ADD_I(copysign_float, 2); - ADD_I(flipsign_int, 2); ADD_I(select_value, 3); - ADD_I(ceil_llvm, 1); ADD_I(floor_llvm, 1); ADD_I(trunc_llvm, 1); ADD_I(rint_llvm, 1); - ADD_I(sqrt_llvm, 1); ADD_I(powi_llvm, 2); - ALIAS(sqrt_llvm_fast, sqrt_llvm); - ADD_I(pointerref, 2); ADD_I(pointerset, 3); - ADD_I(checked_sadd, 2); ADD_I(checked_uadd, 2); - ADD_I(checked_ssub, 2); ADD_I(checked_usub, 2); - ADD_I(checked_smul, 2); ADD_I(checked_umul, 2); - ADD_I(checked_fptosi, 2); ADD_I(checked_fptoui, 2); - ADD_I(checked_trunc_sint, 2); - ADD_I(checked_trunc_uint, 2); - ADD_I(check_top_bit, 1); - ADD_I(nan_dom_err, 2); - ADD_I(fptosi_auto, 1); ADD_I(fptoui_auto, 1); - -} - -#undef ADD_I -#undef ALIAS diff --git a/src/intrinsics.h b/src/intrinsics.h new file mode 100644 index 0000000000000..3692ec2d260e5 --- /dev/null +++ b/src/intrinsics.h @@ -0,0 +1,207 @@ +// This file is a part of Julia. License is MIT: http://julialang.org/license + +#define INTRINSICS \ + /* wrap and unwrap */ \ + ALIAS(box, reinterpret) \ + ALIAS(unbox, reinterpret) \ + /* arithmetic */ \ + ADD_I(neg_int, 1) \ + ADD_I(add_int, 2) \ + ADD_I(sub_int, 2) \ + ADD_I(mul_int, 2) \ + ADD_I(sdiv_int, 2) \ + ADD_I(udiv_int, 2) \ + ADD_I(srem_int, 2) \ + ADD_I(urem_int, 2) \ + ADD_I(smod_int, 2) \ + ADD_I(neg_float, 1) \ + ADD_I(add_float, 2) \ + ADD_I(sub_float, 2) \ + ADD_I(mul_float, 2) \ + ADD_I(div_float, 2) \ + ADD_I(rem_float, 2) \ + ADD_I(fma_float, 3) \ + ADD_I(muladd_float, 3) \ + /* fast arithmetic */ \ + ALIAS(neg_float_fast, neg_float) \ + ALIAS(add_float_fast, add_float) \ + ALIAS(sub_float_fast, sub_float) \ + ALIAS(mul_float_fast, mul_float) \ + ALIAS(div_float_fast, div_float) \ + ALIAS(rem_float_fast, rem_float) \ + /* same-type comparisons */ \ + ADD_I(eq_int, 2) \ + ADD_I(ne_int, 2) \ + ADD_I(slt_int, 2) \ + ADD_I(ult_int, 2) \ + ADD_I(sle_int, 2) \ + ADD_I(ule_int, 2) \ + ADD_I(eq_float, 2) \ + ADD_I(ne_float, 2) \ + ADD_I(lt_float, 2) \ + ADD_I(le_float, 2) \ + ALIAS(eq_float_fast, eq_float) \ + ALIAS(ne_float_fast, ne_float) \ + ALIAS(lt_float_fast, lt_float) \ + ALIAS(le_float_fast, le_float) \ + ADD_I(fpiseq, 2) \ + ADD_I(fpislt, 2) \ + /* bitwise operators */ \ + ADD_I(and_int, 2) \ + ADD_I(or_int, 2) \ + ADD_I(xor_int, 2) \ + ADD_I(not_int, 1) \ + ADD_I(shl_int, 2) \ + ADD_I(lshr_int, 2) \ + ADD_I(ashr_int, 2) \ + ADD_I(bswap_int, 1) \ + ADD_I(ctpop_int, 1) \ + ADD_I(ctlz_int, 1) \ + ADD_I(cttz_int, 1) \ + /* conversion */ \ + ADD_I(sext_int, 2) \ + ADD_I(zext_int, 2) \ + ADD_I(trunc_int, 2) \ + ADD_I(fptoui, 2) \ + ADD_I(fptosi, 2) \ + ADD_I(uitofp, 2) \ + ADD_I(sitofp, 2) \ + ADD_I(fptrunc, 2) \ + ADD_I(fpext, 2) \ + /* checked conversion */ \ + ADD_I(checked_fptosi, 2) \ + ADD_I(checked_fptoui, 2) \ + ADD_I(checked_trunc_sint, 2) \ + ADD_I(checked_trunc_uint, 2) \ + ADD_I(check_top_bit, 1) \ + /* checked arithmetic */ \ + ADD_I(checked_sadd, 2) \ + ADD_I(checked_uadd, 2) \ + ADD_I(checked_ssub, 2) \ + ADD_I(checked_usub, 2) \ + ADD_I(checked_smul, 2) \ + ADD_I(checked_umul, 2) \ + ADD_I(nan_dom_err, 2) \ + /* functions */ \ + ADD_I(abs_float, 1) \ + ADD_I(copysign_float, 2) \ + ADD_I(flipsign_int, 2) \ + ADD_I(select_value, 3) \ + ADD_I(ceil_llvm, 1) \ + ADD_I(floor_llvm, 1) \ + ADD_I(trunc_llvm, 1) \ + ADD_I(rint_llvm, 1) \ + ADD_I(sqrt_llvm, 1) \ + ADD_I(powi_llvm, 2) \ + ALIAS(sqrt_llvm_fast, sqrt_llvm) \ + /* pointer access */ \ + ADD_I(pointerref, 2) \ + ADD_I(pointerset, 3) \ + /* c interface */ \ + ALIAS(ccall, ccall) \ + ALIAS(cglobal, cglobal) \ + ALIAS(llvmcall, llvmcall) \ + /* hidden intrinsics */ \ + ADD_HIDDEN(fptoui_auto, 1) \ + ADD_HIDDEN(fptosi_auto, 1) + +enum intrinsic { +#define ADD_I(func, nargs) func, +#define ADD_HIDDEN ADD_I +#define ALIAS ADD_I + INTRINSICS +#undef ADD_I +#undef ADD_HIDDEN +#undef ALIAS + num_intrinsics, + reinterpret = box +}; + +#ifdef __cplusplus +extern "C" +#endif +const char* jl_intrinsic_name(int f) +{ + switch ((enum intrinsic)f) { + default: return "invalid"; +#define ADD_I(func, nargs) case func: return #func; +#define ADD_HIDDEN ADD_I +#define ALIAS ADD_I + INTRINSICS +#undef ADD_I +#undef ADD_HIDDEN +#undef ALIAS + } +} + +static void* runtime_fp[num_intrinsics]; +static unsigned intrinsic_nargs[num_intrinsics]; + +typedef jl_value_t *(*intrinsic_call_1_arg)(jl_value_t*); +typedef jl_value_t *(*intrinsic_call_2_arg)(jl_value_t*, jl_value_t*); +typedef jl_value_t *(*intrinsic_call_3_arg)(jl_value_t*, jl_value_t*, jl_value_t*); +#define jl_is_intrinsic(v) jl_typeis(v,jl_intrinsic_type) + +#ifdef __cplusplus +extern "C" +#endif +JL_CALLABLE(jl_f_intrinsic_call) +{ + JL_NARGSV(intrinsic_call, 1); + JL_TYPECHK(intrinsic_call, intrinsic, args[0]); + enum intrinsic f = (enum intrinsic)*(uint32_t*)jl_data_ptr(args[0]); + if (f == fptoui && nargs == 1) + f = fptoui_auto; + if (f == fptosi && nargs == 1) + f = fptosi_auto; + unsigned fargs = intrinsic_nargs[f]; + JL_NARGS(intrinsic_call, 1 + fargs, 1 + fargs); + switch (fargs) { + case 1: + return ((intrinsic_call_1_arg)runtime_fp[f])(args[1]); + case 2: + return ((intrinsic_call_2_arg)runtime_fp[f])(args[1], args[2]); + case 3: + return ((intrinsic_call_3_arg)runtime_fp[f])(args[1], args[2], args[3]); + default: + assert(0 && "unexpected number of arguments to an intrinsic function"); + } + abort(); +} + +static void add_intrinsic_properties(enum intrinsic f, unsigned nargs, void *pfunc) +{ + intrinsic_nargs[f] = nargs; + runtime_fp[f] = pfunc; +} + +static void add_intrinsic(jl_module_t *inm, const char *name, enum intrinsic f) +{ + jl_value_t *i = jl_box32(jl_intrinsic_type, (int32_t)f); + jl_sym_t *sym = jl_symbol(name); + jl_set_const(inm, sym, i); + jl_module_export(inm, sym); +} + + +#ifdef __cplusplus +extern "C" +#endif +void jl_init_intrinsic_functions() +{ + jl_module_t *inm = jl_new_module(jl_symbol("Intrinsics")); + inm->parent = jl_core_module; + jl_set_const(jl_core_module, jl_symbol("Intrinsics"), (jl_value_t*)inm); + +#define ADD_I(name, nargs) add_intrinsic(inm, #name, name); add_intrinsic_properties(name, nargs, (void*)&jl_##name); +#define ADD_HIDDEN(name, nargs) add_intrinsic_properties(name, nargs, (void*)&jl_##name); +#define ALIAS(alias, base) add_intrinsic(inm, #alias, alias); add_intrinsic_properties(alias, intrinsic_nargs[base], runtime_fp[base]); + ADD_HIDDEN(reinterpret, 2); + INTRINSICS +#undef ADD_I +#undef ADD_HIDDEN +#undef ALIAS + + jl_set_const(inm, jl_symbol("intrinsic_call"), + (jl_value_t*)jl_new_closure(jl_f_intrinsic_call, (jl_value_t*)jl_symbol("intrinsic_call"), NULL)); +} diff --git a/src/julia_internal.h b/src/julia_internal.h index 86490022e2340..fe9f2ac488256 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -116,6 +116,7 @@ jl_value_t *jl_nth_slot_type(jl_tupletype_t *sig, size_t i); void jl_compute_field_offsets(jl_datatype_t *st); jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims, size_t *dims, int isunboxed, int elsz); +DLLEXPORT jl_value_t *jl_new_box(jl_value_t *v); extern jl_array_t *jl_module_init_order; #ifdef JL_USE_INTEL_JITEVENTS @@ -145,10 +146,6 @@ void jl_dump_objfile(char *fname, int jit_model, const char *sysimg_data, size_t int32_t jl_get_llvm_gv(jl_value_t *p); void jl_idtable_rehash(jl_array_t **pa, size_t newsz); -#ifdef _OS_LINUX_ -DLLEXPORT void jl_read_sonames(void); -#endif - jl_lambda_info_t *jl_add_static_parameters(jl_lambda_info_t *l, jl_svec_t *sp); jl_function_t *jl_get_specialization(jl_function_t *f, jl_tupletype_t *types); jl_function_t *jl_module_get_initializer(jl_module_t *m); @@ -214,6 +211,10 @@ extern uv_lib_t *jl_crtdll_handle; extern uv_lib_t *jl_winsock_handle; #endif +uv_lib_t *jl_get_library(char *f_lib); +DLLEXPORT void *jl_load_and_lookup(char *f_lib, char *f_name, uv_lib_t **hnd); + + // libuv wrappers: DLLEXPORT int jl_fs_rename(const char *src_path, const char *dst_path); @@ -226,6 +227,7 @@ extern DLLEXPORT jl_value_t *jl_segv_exception; #endif // Runtime intrinsics // +const char* jl_intrinsic_name(int f); DLLEXPORT jl_value_t *jl_reinterpret(jl_value_t *ty, jl_value_t *v); DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i); diff --git a/src/runtime_ccall.cpp b/src/runtime_ccall.cpp new file mode 100644 index 0000000000000..f1623fc6ac08f --- /dev/null +++ b/src/runtime_ccall.cpp @@ -0,0 +1,140 @@ +// This file is a part of Julia. License is MIT: http://julialang.org/license + +#include +#include +#include "julia.h" +#include "julia_internal.h" + +// --- library symbol lookup --- + +// map from "libX" to full soname "libX.so.ver" +#if defined(__linux__) || defined(__FreeBSD__) +static std::map sonameMap; +static bool got_sonames = false; + +static void jl_read_sonames(void) +{ + char *line=NULL; + size_t sz=0; +#if defined(__linux__) + FILE *ldc = popen("/sbin/ldconfig -p", "r"); +#else + FILE *ldc = popen("/sbin/ldconfig -r", "r"); +#endif + + while (!feof(ldc)) { + ssize_t n = getline(&line, &sz, ldc); + if (n == -1) + break; + if (n > 2 && isspace((unsigned char)line[0])) { +#ifdef __linux__ + int i = 0; + while (isspace((unsigned char)line[++i])) ; + char *name = &line[i]; + char *dot = strstr(name, ".so"); + i = 0; +#else + char *name = strstr(line, ":-l"); + if (name == NULL) continue; + strncpy(name, "lib", 3); + char *dot = strchr(name, '.'); +#endif + + if (NULL == dot) + continue; + +#ifdef __linux__ + // Detect if this entry is for the current architecture + while (!isspace((unsigned char)dot[++i])) ; + while (isspace((unsigned char)dot[++i])) ; + int j = i; + while (!isspace((unsigned char)dot[++j])) ; + char *arch = strstr(dot+i,"x86-64"); + if (arch != NULL && arch < dot + j) { +#ifdef _P32 + continue; +#endif + } + else { +#ifdef _P64 + continue; +#endif + } +#endif // __linux__ + + char *abslibpath = strrchr(line, ' '); + if (dot != NULL && abslibpath != NULL) { + std::string pfx(name, dot - name); + // Do not include ' ' in front and '\n' at the end + std::string soname(abslibpath+1, line+n-(abslibpath+1)-1); + sonameMap[pfx] = soname; + } + } + } + + free(line); + pclose(ldc); +} + +extern "C" DLLEXPORT const char *jl_lookup_soname(const char *pfx, size_t n) +{ + if (!got_sonames) { + jl_read_sonames(); + got_sonames = true; + } + std::string str(pfx, n); + if (sonameMap.find(str) != sonameMap.end()) { + return sonameMap[str].c_str(); + } + return NULL; +} +#endif + +// map from user-specified lib names to handles +static std::map libMap; + +extern "C" +uv_lib_t *jl_get_library(char *f_lib) +{ + uv_lib_t *hnd; +#ifdef _OS_WINDOWS_ + if ((intptr_t)f_lib == 1) + return jl_exe_handle; + if ((intptr_t)f_lib == 2) + return jl_dl_handle; +#endif + if (f_lib == NULL) + return jl_RTLD_DEFAULT_handle; + hnd = libMap[f_lib]; + if (hnd != NULL) + return hnd; + hnd = (uv_lib_t *) jl_load_dynamic_library(f_lib, JL_RTLD_DEFAULT); + if (hnd != NULL) + libMap[f_lib] = hnd; + return hnd; +} + +extern "C" DLLEXPORT +void *jl_load_and_lookup(char *f_lib, char *f_name, uv_lib_t **hnd) +{ + uv_lib_t *handle = *hnd; + if (!handle) + *hnd = handle = jl_get_library(f_lib); + void *ptr = jl_dlsym_e(handle, f_name); + if (!ptr) + jl_errorf("symbol \"%s\" could not be found: %s", f_name, uv_dlerror(handle)); + return ptr; +} + +// miscellany +#include +extern "C" DLLEXPORT +jl_value_t *jl_get_cpu_name(void) +{ +#if LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR < 5 + std::string HostCPUName = llvm::sys::getHostCPUName(); +#else + StringRef HostCPUName = llvm::sys::getHostCPUName(); +#endif + return jl_pchar_to_string(HostCPUName.data(), HostCPUName.size()); +} diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index bd2c296b99f87..a7910152bcd23 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -1,5 +1,5 @@ // This file is a part of Julia. License is MIT: http://julialang.org/license -// + // This is in implementation of the Julia intrinsic functions against boxed types // excluding the c interface (ccall, cglobal, llvmcall) // diff --git a/src/sys.c b/src/sys.c index 4d88301b8aed3..df5d373501382 100644 --- a/src/sys.c +++ b/src/sys.c @@ -540,14 +540,12 @@ DLLEXPORT int32_t jl_set_zero_subnormals(int8_t isZero) DLLEXPORT void jl_native_alignment(uint_t *int8align, uint_t *int16align, uint_t *int32align, uint_t *int64align, uint_t *float32align, uint_t *float64align) { - LLVMTargetDataRef tgtdata = LLVMCreateTargetData(""); - *int8align = LLVMPreferredAlignmentOfType(tgtdata, LLVMInt8Type()); - *int16align = LLVMPreferredAlignmentOfType(tgtdata, LLVMInt16Type()); - *int32align = LLVMPreferredAlignmentOfType(tgtdata, LLVMInt32Type()); - *int64align = LLVMPreferredAlignmentOfType(tgtdata, LLVMInt64Type()); - *float32align = LLVMPreferredAlignmentOfType(tgtdata, LLVMFloatType()); - *float64align = LLVMPreferredAlignmentOfType(tgtdata, LLVMDoubleType()); - LLVMDisposeTargetData(tgtdata); + *int8align = __alignof(uint8_t); + *int16align = __alignof(uint16_t); + *int32align = __alignof(uint32_t); + *int64align = __alignof(uint64_t); + *float32align = __alignof(float); + *float64align = __alignof(double); } DLLEXPORT jl_value_t *jl_is_char_signed() From d23dc8531fb335c4a7d4e1c463f67cba1d1efd41 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 7 Oct 2015 13:12:40 -0400 Subject: [PATCH 0477/1938] improve jl_ printing for functions --- src/builtins.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/builtins.c b/src/builtins.c index 0dd497981bc6a..c4691dc1126d8 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1400,12 +1400,14 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, n += jl_printf(out, "%s", jl_gf_name(v)->name); } else { - n += jl_printf(out, "#"); + n += jl_printf(out, "#linfo, depth); + n += jl_printf(out, ">"); } } else if (vt == jl_intrinsic_type) { - n += jl_printf(out, "#", - *(uint32_t*)jl_data_ptr(v)); + int f = *(uint32_t*)jl_data_ptr(v); + n += jl_printf(out, "#", f, jl_intrinsic_name(f)); } else if (vt == jl_int64_type) { n += jl_printf(out, "%" PRId64, *(int64_t*)v); From 789ce2f2c6fd774a06b4ecb0a1266e539a9ec4b3 Mon Sep 17 00:00:00 2001 From: peter1000 Date: Wed, 7 Oct 2015 16:07:10 -0300 Subject: [PATCH 0478/1938] Fixes typo [skip ci] --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 0cae4b2c39771..90f776040c1a0 100644 --- a/NEWS.md +++ b/NEWS.md @@ -22,7 +22,7 @@ Library improvements tests together and delay throwing an error until the end ([#13062]). * The function `remotecall`, `remotecall_fetch`, and `remotecall_wait` now have the - the function argument as the first argument to allow for do-block syntax ([#13338]). + function argument as the first argument to allow for do-block syntax ([#13338]). Deprecated or removed --------------------- From d0a50e0de3de549e3a9a477533e4bdacf2ac23a5 Mon Sep 17 00:00:00 2001 From: Jiahao Chen Date: Wed, 7 Oct 2015 15:32:54 -0400 Subject: [PATCH 0479/1938] NEWS: Minor reformatting and grammatical fixes --- NEWS.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/NEWS.md b/NEWS.md index 90f776040c1a0..cf410b47bf7b6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -16,12 +16,14 @@ Compiler/Runtime improvements Library improvements -------------------- - * The package system (`Pkg`) is now based on the libgit2 library, rather than running the `git` program, increasing performance (especially on Windows) ([#11196]). + * The package system (`Pkg`) is now based on the `libgit2` library, rather + than running the `git` program, increasing performance (especially on + Windows) ([#11196]). * The `Base.Test` module now has a `@testset` feature to bundle tests together and delay throwing an error until the end ([#13062]). - * The function `remotecall`, `remotecall_fetch`, and `remotecall_wait` now have the + * The functions `remotecall`, `remotecall_fetch`, and `remotecall_wait` now have the function argument as the first argument to allow for do-block syntax ([#13338]). Deprecated or removed @@ -83,7 +85,7 @@ New language features * Support for inter-task communication using `Channels` ([#12264]). See http://docs.julialang.org/en/latest/manual/parallel-computing/#channels for details. - * RemoteRefs now point to remote channels. The remote channels can be of length greater than 1. + * `RemoteRef`s now point to remote channels. The remote channels can be of length greater than 1. Default continues to be of length 1 ([#12385]). See http://docs.julialang.org/en/latest/manual/parallel-computing/#remoterefs-and-abstractchannels for details. @@ -419,7 +421,7 @@ Library improvements * Other improvements - * You can now tab-complete Emoji characters via their [short names](http://www.emoji-cheat-sheet.com/), using `\:name:` ([#10709]). + * You can now tab-complete emoji via their [short names](http://www.emoji-cheat-sheet.com/), using `\:name:` ([#10709]). * `gc_enable` subsumes `gc_disable`, and also returns the previous GC state. @@ -433,9 +435,9 @@ Library improvements * Added `recvfrom` to get source address of UDP packets ([#9418]). - * ClusterManager performance improvements ([#9309]) and support for changing transports([#9434]). + * `ClusterManager` performance improvements ([#9309]) and support for changing transports([#9434]). - * Added Base.get_process_title / Base.set_process_title ([#9957]). + * Added `Base.get_process_title` / `Base.set_process_title` ([#9957]). * `readavailable` now returns a byte vector instead of a string. @@ -479,7 +481,7 @@ Deprecated or removed end ``` - * indexing with Reals that are not subtypes of Integers (Rationals, AbstractFloat, etc.) has been deprecated ([#10458]). + * indexing with `Real`s that are not subtypes of `Integer` (`Rational`, `AbstractFloat`, etc.) has been deprecated ([#10458]). * `push!(A)` has been deprecated, use `append!` instead of splatting arguments to `push!` ([#10400]). @@ -492,7 +494,7 @@ Deprecated or removed * The `Graphics` module has been removed from `Base` and is now a standalone package ([#10150], [#9862]). - * The `Woodbury` special matrix type has been removed from LinAlg ([#10024]). + * The `Woodbury` special matrix type has been removed from `LinAlg` ([#10024]). * `median` and `median!` no longer accept a `checknan` keyword argument ([#8605]). From 05fb6ad22608c34ad03a24474db94c9da7ef0ebc Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 7 Oct 2015 23:22:50 +0200 Subject: [PATCH 0480/1938] improve docs for A_mul_B! --- base/docs/helpdb.jl | 8 ++++---- doc/stdlib/math.rst | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index aecf883f77206..311a06c1255e2 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -6086,14 +6086,14 @@ values doc""" A_mul_B!(Y, A, B) -> Y - Calculates the matrix-matrix or matrix-vector product $A⋅B$ and stores the -result in $Y$, overwriting the existing value of $Y$. +result in `Y`, overwriting the existing value of `Y`. Note that `Y` must not +be aliased with either `A` or `B`. ```jldoctest -julia> A=[1.0 2.0; 3.0 4.0]; B=[1.0 1.0; 1.0 1.0]; A_mul_B!(B, A, B); +julia> A=[1.0 2.0; 3.0 4.0]; B=[1.0 1.0; 1.0 1.0]; Y = similar(B); A_mul_B!(Y, A, B); -julia> B +julia> Y 2x2 Array{Float64,2}: 3.0 3.0 7.0 7.0 diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index c3fa039b1d8c3..9e3734e067011 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -425,13 +425,13 @@ Mathematical Operators .. Docstring generated from Julia source - Calculates the matrix-matrix or matrix-vector product :math:`A⋅B` and stores the result in :math:`Y`\ , overwriting the existing value of :math:`Y`\ . + Calculates the matrix-matrix or matrix-vector product :math:`A⋅B` and stores the result in ``Y``\ , overwriting the existing value of ``Y``\ . Note that ``Y`` must not be aliased with either ``A`` or ``B``\ . .. doctest:: - julia> A=[1.0 2.0; 3.0 4.0]; B=[1.0 1.0; 1.0 1.0]; A_mul_B!(B, A, B); + julia> A=[1.0 2.0; 3.0 4.0]; B=[1.0 1.0; 1.0 1.0]; Y = similar(B); A_mul_B!(Y, A, B); - julia> B + julia> Y 2x2 Array{Float64,2}: 3.0 3.0 7.0 7.0 From a70d7cfafe049588a3d9d9b77efc331e629aabdd Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 7 Oct 2015 18:04:49 -0400 Subject: [PATCH 0481/1938] fix disasm printing to handle inlining info --- src/disasm.cpp | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/disasm.cpp b/src/disasm.cpp index aea9c565c4f90..02a21d3d88895 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -247,6 +247,23 @@ int OpInfoLookup(void *DisInfo, uint64_t PC, } } // namespace +#ifndef USE_MCJIT +static void print_source_line(raw_ostream &stream, DebugLoc Loc) +{ + MDNode *inlinedAt = Loc.getInlinedAt(jl_LLVMContext); + if (inlinedAt != NULL) { + DebugLoc inlineloc = DebugLoc::getFromDILocation(inlinedAt); + stream << "Source line: " << inlineloc.getLine() << "\n"; + + DILexicalBlockFile innerscope = DILexicalBlockFile(Loc.getScope(jl_LLVMContext)); + stream << "Source line: [inline] " << innerscope.getFilename().str().c_str() << ':' << Loc.getLine() << "\n"; + } + else { + stream << "Source line: " << Loc.getLine() << "\n"; + } +} +#endif + extern "C" void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, size_t slide, #ifndef USE_MCJIT @@ -465,10 +482,21 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, size_t slide, if (lineIter != lineEnd) { nextLineAddr = (*lineIter).Address; - DISubprogram debugscope = DISubprogram((*lineIter).Loc.getScope(jl_LLVMContext)); if (pass != 0) { - stream << "Filename: " << debugscope.getFilename() << "\n"; - stream << "Source line: " << (*lineIter).Loc.getLine() << "\n"; + DebugLoc Loc = (*lineIter).Loc; + MDNode *outer = Loc.getInlinedAt(jl_LLVMContext); + StringRef FileName; + if (!outer) { + DISubprogram debugscope = DISubprogram(Loc.getScope(jl_LLVMContext)); + FileName = debugscope.getFilename(); + } + else { + DebugLoc inlineloc = DebugLoc::getFromDILocation(outer); + DILexicalBlockFile debugscope = DILexicalBlockFile(inlineloc.getScope(jl_LLVMContext)); + FileName = debugscope.getFilename(); + } + stream << "Filename: " << FileName << "\n"; + print_source_line(stream, (*lineIter).Loc); } } #endif @@ -490,8 +518,9 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, size_t slide, #endif nextLineAddr = (++lineIter)->first; #else - if (pass != 0) - stream << "Source line: " << (*lineIter).Loc.getLine() << "\n"; + if (pass != 0) { + print_source_line(stream, (*lineIter).Loc); + } nextLineAddr = (*++lineIter).Address; #endif } From a33cd07f8a0fe2dd4ef3fecf809add2bb2a493c9 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 7 Oct 2015 16:35:53 -0400 Subject: [PATCH 0482/1938] preserve inline line numbers during inlining --- base/inference.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index fc190395ebb30..e4e5fa17d4a67 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2248,8 +2248,7 @@ function inlineable(f::ANY, e::Expr, atype::ANY, sv::StaticVarInfo, enclosing_as end body = Expr(:block) - body.args = filter(x->!((isa(x,Expr) && is(x.head,:line)) || isa(x,LineNumberNode)), - ast.args[3].args::Array{Any,1}) + body.args = ast.args[3].args::Array{Any,1} cost::Int = 1000 if incompletematch cost *= 4 From e2ee7283c204b974a6e08f01c1e1041dc2b9bcb5 Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Wed, 7 Oct 2015 19:13:29 -0400 Subject: [PATCH 0483/1938] tweaks docstrings and adds stdlib docs --- base/test.jl | 31 +++++++++++++--- doc/stdlib/test.rst | 88 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 109 insertions(+), 10 deletions(-) diff --git a/base/test.jl b/base/test.jl index d3a2ab0a33f99..02de51a8bfae2 100644 --- a/base/test.jl +++ b/base/test.jl @@ -238,6 +238,26 @@ end # Called after the test set has been popped from the test set stack abstract AbstractTestSet +""" + record(ts::AbstractTestSet, res::Result) + +Record a result to a testset. This function is called by the `@testset` +infrastructure each time a contained `@test` macro completes, and is given the +test result (which could be an `Error`). This will also be called with an `Error` +if an exception is thrown inside the test block but outside of a `@test` context. +""" +function record end + +""" + finish(ts::AbstractTestSet) + +Do any final processing necessary for the given testset. This is called by the +`@testset` infrastructure after a test block executes. One common use for this +function is to record the testset to the parent's results list, using +`get_testset`. +""" +function finish end + """ TestSetException @@ -248,6 +268,7 @@ type TestSetException <: Exception fail::Int error::Int end + function Base.show(io::IO, ex::TestSetException) print(io, "Some tests did not pass: ") print(io, ex.pass, " passed, ") @@ -470,8 +491,9 @@ end """ @testset [CustomTestSet] [option=val ...] ["description"] begin ... end -Starts a new test set. The test results will be recorded, and if there -are any `Fail`s or `Error`s, an exception will be thrown at the end of the +Starts a new test set. If no custom testset type is given it defaults to creating +a `DefaultTestSet`. `DefaultTestSet` records all the results and, and if there +are any `Fail`s or `Error`s, throws an exception at the end of the top-level (non-nested) test set, along with a summary of the test results. Any custom testset type (subtype of `AbstractTestSet`) can be given and it will @@ -530,8 +552,9 @@ also be used for any nested `@testset` or `@testloop` invocations. The given options are only applied to the test sets where they are given. The default test set type does not take any options. -By default the `@testset` macro will return a list of the testset objects used -in each iteration, though this behavior can be customized in other testset types. +The `@testloop` macro collects and returns a list of the return values of the +`finish` method, which by default will return a list of the testset objects used +in each iteration. """ macro testloop(args...) length(args) > 0 || error("no arguments to @testloop") diff --git a/doc/stdlib/test.rst b/doc/stdlib/test.rst index 142d2efd3e2cd..bb5e4a471a38b 100644 --- a/doc/stdlib/test.rst +++ b/doc/stdlib/test.rst @@ -115,16 +115,18 @@ and at the end of the test set a summary will be printed. If any of the tests failed, or could not be evaluated due to an error, the test set will then throw a ``TestSetException``. - -.. function:: @testset "description" begin ... end - @testset begin ... end +.. function:: @testset [CustomTestSet] [option=val ...] ["description"] begin ... end .. Docstring generated from Julia source - Starts a new test set. The test results will be recorded, and if there are any ``Fail``\ s or ``Error``\ s, an exception will be thrown only at the end, along with a summary of the test results. + Starts a new test set. If no custom testset type is given it defaults to creating a ``DefaultTestSet``\ . ``DefaultTestSet`` records all the results and, and if there are any ``Fail``\ s or ``Error``\ s, throws an exception at the end of the top-level (non-nested) test set, along with a summary of the test results. + + Any custom testset type (subtype of ``AbstractTestSet``\ ) can be given and it will also be used for any nested ``@testset`` or ``@testloop`` invocations. The given options are only applied to the test set where they are given. The default test set type does not take any options. + + By default the ``@testset`` macro will return the testset object itself, though this behavior can be customized in other testset types. -.. function:: @testloop "description $v" for v in (...) ... end - @testloop for x in (...), y in (...) ... end +.. function:: @testloop [CustomTestSet] [option=val ...] ["description \$v"] for v in (...) ... end + @testloop [CustomTestSet] [option=val ...] ["description \$v, \$w"] for v in (...), w in (...) ... end .. Docstring generated from Julia source @@ -244,3 +246,77 @@ writing new tests. Test two floating point numbers ``a`` and ``b`` for equality taking in account a margin of tolerance given by ``tol``\ . +Creating Custom ``AbstractTestSet`` Types +----------------------------------------- + +Packages can create their own ``AbstractTestSet`` subtypes by implementing the +``record`` and ``finish`` methods. The subtype should have a one-argument +constructor taking a description string, with any options passed in as keyword +arguments. + +.. function:: record(ts::AbstractTestSet, res::Result) + + .. Docstring generated from Julia source + + Record a result to a testset. This function is called by the ``@testset`` infrastructure each time a contained ``@test`` macro completes, and is given the test result (which could be an ``Error``\ ). This will also be called with an ``Error`` if an exception is thrown inside the test block but outside of a ``@test`` context. + +.. function:: finish(ts::AbstractTestSet) + + .. Docstring generated from Julia source + + Do any final processing necessary for the given testset. This is called by the ``@testset`` infrastructure after a test block executes. One common use for this function is to record the testset to the parent's results list, using ``get_testset``\ . + +``Base.Test`` takes responsibility for maintaining a stack of nested testsets as +they are executed, but any result accumulation is the responsibility of the +``AbstractTestSet`` subtype. You can access this stack with the ``get_testset`` and +``get_testset_depth`` methods. Note that these functions are not exported. + +.. function:: get_testset() + + .. Docstring generated from Julia source + + Retrieve the active test set from the task's local storage. If no test set is active, use the fallback default test set. + +.. function:: get_testset_depth() + + .. Docstring generated from Julia source + + Returns the number of active test sets, not including the defaut test set + +``Base.Test`` also makes sure that nested ``@testset`` invocations use the same +``AbstractTestSet`` subtype as their parent unless it is set explicitly. It does +not propagate any properties of the testset option inheritance behavior can be +implemented by packages using the stack infrastructure that ``Base.Test`` +provides. + +Defining a basic ``AbstractTestSet`` subtype might look like:: + + import Base.Test: record, finish + using Base.Test: AbstractTestSet, Result, Pass, Fail, Error + using Base.Test: get_testset_depth, get_testset + immutable CustomTestSet <: Base.Test.AbstractTestSet + description::AbstractString + foo::Int + results::Vector + # constructor takes a description string and options keyword arguments + CustomTestSet(desc; foo=1) = new(desc, foo, []) + end + + record(ts::CustomTestSet, child::AbstractTestSet) = push!(ts.results, child) + record(ts::CustomTestSet, res::Result) = push!(ts.results, res) + function finish(ts::CustomTestSet) + # just record if we're not the top-level parent + if get_testset_depth() > 0 + record(get_testset(), ts) + end + ts + end + +And using that testset looks like:: + + @testset CustomTestSet foo=4 "custom testset inner 2" begin + # this testset should inherit the type, but not the argument. + @testset "custom testset inner" begin + @test true + end + end From af81d6c7fe09f57ab268ccd731bc2d2f4aee9a95 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 7 Oct 2015 19:22:01 -0400 Subject: [PATCH 0484/1938] fix #922 (wrong line numbers in error expressions) and remove old hacks around this issue --- src/builtins.c | 6 ------ src/cgutils.cpp | 14 +++++-------- src/codegen.cpp | 52 ++++++++++++++++++----------------------------- src/julia.h | 3 --- src/task.c | 5 ----- test/backtrace.jl | 3 ++- 6 files changed, 27 insertions(+), 56 deletions(-) diff --git a/src/builtins.c b/src/builtins.c index 0dd497981bc6a..6fd90d141a513 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -103,12 +103,6 @@ void NORETURN jl_type_error_rt(const char *fname, const char *context, jl_throw(ex); } -void NORETURN jl_type_error_rt_line(const char *fname, const char *context, - jl_value_t *ty, jl_value_t *got, int line) -{ - jl_type_error_rt(fname, context, ty, got); -} - void NORETURN jl_type_error(const char *fname, jl_value_t *expected, jl_value_t *got) { jl_type_error_rt(fname, "", expected, got); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 7f0d2c5046bb3..dade30965558f 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -935,11 +935,9 @@ static void raise_exception_unless(Value *cond, Value *exc, jl_codectx_t *ctx) builder.CreateCondBr(cond, passBB, failBB); builder.SetInsertPoint(failBB); #ifdef LLVM37 - builder.CreateCall(prepare_call(jlthrow_line_func), { exc, - ConstantInt::get(T_int32, ctx->lineno) }); + builder.CreateCall(prepare_call(jlthrow_func), { exc }); #else - builder.CreateCall2(prepare_call(jlthrow_line_func), exc, - ConstantInt::get(T_int32, ctx->lineno)); + builder.CreateCall(prepare_call(jlthrow_func), exc); #endif builder.CreateUnreachable(); ctx->f->getBasicBlockList().push_back(passBB); @@ -983,13 +981,11 @@ static void emit_type_error(const jl_cgval_t &x, jl_value_t *type, const std::st #ifdef LLVM37 builder.CreateCall(prepare_call(jltypeerror_func), { fname_val, msg_val, - literal_pointer_val(type), boxed(x,ctx), - ConstantInt::get(T_int32, ctx->lineno) }); + literal_pointer_val(type), boxed(x,ctx)}); #else - builder.CreateCall5(prepare_call(jltypeerror_func), + builder.CreateCall4(prepare_call(jltypeerror_func), fname_val, msg_val, - literal_pointer_val(type), boxed(x,ctx), - ConstantInt::get(T_int32, ctx->lineno)); + literal_pointer_val(type), boxed(x,ctx)); #endif } diff --git a/src/codegen.cpp b/src/codegen.cpp index 3e88bdfc56bf0..2894f650058df 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -315,7 +315,6 @@ extern RTDyldMemoryManager* createRTDyldMemoryManagerOSX(); // important functions static Function *jlnew_func; static Function *jlthrow_func; -static Function *jlthrow_line_func; static Function *jlerror_func; static Function *jltypeerror_func; static Function *jlundefvarerror_func; @@ -539,7 +538,6 @@ typedef struct { bool vaStack; // varargs stack-allocated bool sret; int nReqArgs; - int lineno; std::vector boundsCheck; jl_gcinfo_t gc; @@ -1437,7 +1435,7 @@ extern "C" void jl_write_malloc_log(void) static void show_source_loc(JL_STREAM *out, jl_codectx_t *ctx) { if (ctx == NULL) return; - jl_printf(out, "in %s at %s:%d", ctx->linfo->name->name, ctx->linfo->file->name, ctx->lineno); + jl_printf(out, "in %s at %s", ctx->linfo->name->name, ctx->linfo->file->name); } extern "C" void jl_binding_deprecation_warning(jl_binding_t *b); @@ -4417,7 +4415,6 @@ static Function *emit_function(jl_lambda_info_t *lam) } } } - ctx.lineno = lno; int toplineno = lno; DIBuilder dbuilder(*m); @@ -4429,6 +4426,7 @@ static Function *emit_function(jl_lambda_info_t *lam) DIFile topfile; DISubprogram SP; #endif + DebugLoc inlineLoc; BasicBlock *b0 = BasicBlock::Create(jl_LLVMContext, "top", f); builder.SetInsertPoint(b0); @@ -4503,7 +4501,8 @@ static Function *emit_function(jl_lambda_info_t *lam) true, // isOptimized f); // Fn // set initial line number - builder.SetCurrentDebugLocation(DebugLoc::get(lno, 0, (MDNode*)SP, NULL)); + inlineLoc = DebugLoc::get(lno, 0, (MDNode*)SP, NULL); + builder.SetCurrentDebugLocation(inlineLoc); #ifndef LLVM37 assert(SP.Verify() && SP.describes(f) && SP.getFunction() == f); #endif @@ -4520,7 +4519,7 @@ static Function *emit_function(jl_lambda_info_t *lam) argname->name, // Variable name ctx.sret + i + 1, // Argument number (1-based) topfile, // File - ctx.lineno == -1 ? 0 : ctx.lineno, // Line + toplineno == -1 ? 0 : toplineno, // Line // Variable type julia_type_to_di(varinfo.value.typ,ctx.dbuilder,specsig)); #else @@ -4529,7 +4528,7 @@ static Function *emit_function(jl_lambda_info_t *lam) SP, // Scope (current function will be fill in later) argname->name, // Variable name topfile, // File - ctx.lineno == -1 ? 0 : ctx.lineno, // Line (for now, use lineno of the function) + toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) julia_type_to_di(varinfo.value.typ, ctx.dbuilder, specsig), // Variable type false, // May be optimized out 0, // Flags (TODO: Do we need any) @@ -4543,7 +4542,7 @@ static Function *emit_function(jl_lambda_info_t *lam) ctx.vaName->name, // Variable name ctx.sret + nreq + 1, // Argument number (1-based) topfile, // File - ctx.lineno == -1 ? 0 : ctx.lineno, // Line (for now, use lineno of the function) + toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) julia_type_to_di(ctx.vars[ctx.vaName].value.typ, ctx.dbuilder, false)); #else ctx.vars[ctx.vaName].dinfo = ctx.dbuilder->createLocalVariable( @@ -4551,7 +4550,7 @@ static Function *emit_function(jl_lambda_info_t *lam) SP, // Scope (current function will be fill in later) ctx.vaName->name, // Variable name topfile, // File - ctx.lineno == -1 ? 0 : ctx.lineno, // Line (for now, use lineno of the function) + toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) julia_type_to_di(ctx.vars[ctx.vaName].value.typ, ctx.dbuilder, false), // Variable type false, // May be optimized out 0, // Flags (TODO: Do we need any) @@ -4572,7 +4571,7 @@ static Function *emit_function(jl_lambda_info_t *lam) SP, // Scope (current function will be fill in later) s->name, // Variable name topfile, // File - ctx.lineno == -1 ? 0 : ctx.lineno, // Line (for now, use lineno of the function) + toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) julia_type_to_di(varinfo.value.typ, ctx.dbuilder, specsig), // Variable type false, // May be optimized out 0 // Flags (TODO: Do we need any) @@ -4596,7 +4595,7 @@ static Function *emit_function(jl_lambda_info_t *lam) SP, // Scope (current function will be filled in later) vname->name, // Variable name topfile, // File - ctx.lineno == -1 ? 0 : ctx.lineno, // Line (for now, use lineno of the function) + toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) julia_type_to_di(varinfo.value.typ, ctx.dbuilder, specsig), // Variable type false, // May be optimized out 0 // Flags (TODO: Do we need any) @@ -4996,7 +4995,6 @@ static Function *emit_function(jl_lambda_info_t *lam) } DebugLoc loc; if (ctx.debug_enabled) { - MDNode *funcscope = (MDNode*)dbuilder.createLexicalBlockFile(SP, topfile); MDNode *scope; if ((dfil == topfile || dfil == NULL) && lno >= toplineno) @@ -5004,19 +5002,17 @@ static Function *emit_function(jl_lambda_info_t *lam) // for sequentially-defined code, // set location to line in top file. // TODO: improve handling of nested inlines - loc = DebugLoc::get(lno, 1, SP, NULL); + loc = inlineLoc = DebugLoc::get(lno, 1, SP, NULL); } else { // otherwise, we are compiling inlined code, // so set the DebugLoc "inlinedAt" parameter // to the current line, then use source loc. #ifdef LLVM37 scope = (MDNode*)dbuilder.createLexicalBlockFile(SP,dfil); - MDNode *inlineLocMd = DebugLoc::get(toplineno, 1, funcscope, NULL). - getAsMDNode(); + MDNode *inlineLocMd = inlineLoc.getAsMDNode(); #else scope = (MDNode*)dbuilder.createLexicalBlockFile(SP,DIFile(dfil)); - MDNode *inlineLocMd = DebugLoc::get(toplineno, 1, funcscope, NULL). - getAsMDNode(jl_LLVMContext); + MDNode *inlineLocMd = inlineLoc.getAsMDNode(jl_LLVMContext); #endif loc = DebugLoc::get(lno, 1, scope, inlineLocMd); } @@ -5024,7 +5020,6 @@ static Function *emit_function(jl_lambda_info_t *lam) } if (do_coverage) coverageVisitLine(filename, lno); - ctx.lineno = lno; // NOO TOUCHIE; NO TOUCH! See #922 } if (jl_is_labelnode(stmt)) { if (prevlabel) continue; @@ -5478,15 +5473,6 @@ static void init_julia_llvm_env(Module *m) jluboundserror_func->setDoesNotReturn(); add_named_global(jluboundserror_func, (void*)&jl_bounds_error_unboxed_int); - std::vector args2_throw(0); - args2_throw.push_back(T_pjlvalue); - args2_throw.push_back(T_int32); - jlthrow_line_func = - (Function*)m->getOrInsertFunction("jl_throw_with_superfluous_argument", - FunctionType::get(T_void, args2_throw, false)); - jlthrow_line_func->setDoesNotReturn(); - add_named_global(jlthrow_line_func, (void*)&jl_throw_with_superfluous_argument); - jlnew_func = Function::Create(jl_func_sig, Function::ExternalLinkage, "jl_new_structv", m); @@ -5517,13 +5503,12 @@ static void init_julia_llvm_env(Module *m) te_args.push_back(T_pint8); te_args.push_back(T_pjlvalue); te_args.push_back(T_pjlvalue); - te_args.push_back(T_int32); jltypeerror_func = Function::Create(FunctionType::get(T_void, te_args, false), Function::ExternalLinkage, - "jl_type_error_rt_line", m); + "jl_type_error_rt", m); jltypeerror_func->setDoesNotReturn(); - add_named_global(jltypeerror_func, (void*)&jl_type_error_rt_line); + add_named_global(jltypeerror_func, (void*)&jl_type_error_rt); std::vector args_2ptrs(0); args_2ptrs.push_back(T_pjlvalue); @@ -5923,13 +5908,16 @@ static void init_julia_llvm_env(Module *m) extern "C" void jl_init_codegen(void) { + const char *const argv_tailmerge[] = {"", "-enable-tail-merge=0"}; // NOO TOUCHIE; NO TOUCH! See #922 + cl::ParseCommandLineOptions(sizeof(argv_tailmerge)/sizeof(argv_tailmerge[0]), argv_tailmerge, "disable-tail-merge\n"); #if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_) - const char *const argv[] = {"", "-disable-copyprop"}; // llvm bug 21743 - cl::ParseCommandLineOptions(sizeof(argv)/sizeof(argv[0]), argv, "disable-copyprop\n"); + const char *const argv_copyprop[] = {"", "-disable-copyprop"}; // llvm bug 21743 + cl::ParseCommandLineOptions(sizeof(argv_copyprop)/sizeof(argv_copyprop[0]), argv_copyprop, "disable-copyprop\n"); #endif #ifdef JL_DEBUG_BUILD cl::ParseEnvironmentOptions("Julia", "JULIA_LLVM_ARGS"); #endif + #if defined(_CPU_PPC_) || defined(_CPU_PPC64_) imaging_mode = true; // LLVM seems to JIT bad TOC tables for the optimizations we attempt in non-imaging_mode #else diff --git a/src/julia.h b/src/julia.h index ec05783766d70..43da2db6963c7 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1150,8 +1150,6 @@ DLLEXPORT void NORETURN jl_too_many_args(const char *fname, int max); DLLEXPORT void NORETURN jl_type_error(const char *fname, jl_value_t *expected, jl_value_t *got); DLLEXPORT void NORETURN jl_type_error_rt(const char *fname, const char *context, jl_value_t *ty, jl_value_t *got); -DLLEXPORT void NORETURN jl_type_error_rt_line(const char *fname, const char *context, - jl_value_t *ty, jl_value_t *got, int line); DLLEXPORT void NORETURN jl_undefined_var_error(jl_sym_t *var); DLLEXPORT void NORETURN jl_bounds_error(jl_value_t *v, jl_value_t *t); DLLEXPORT void NORETURN jl_bounds_error_v(jl_value_t *v, jl_value_t **idxs, size_t nidxs); @@ -1405,7 +1403,6 @@ extern DLLEXPORT JL_THREAD jl_value_t *jl_exception_in_transit; DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize); DLLEXPORT jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg); DLLEXPORT void NORETURN jl_throw(jl_value_t *e); -DLLEXPORT void NORETURN jl_throw_with_superfluous_argument(jl_value_t *e, int); DLLEXPORT void NORETURN jl_rethrow(void); DLLEXPORT void NORETURN jl_rethrow_other(jl_value_t *e); diff --git a/src/task.c b/src/task.c index 31c3dc8b91085..cf9f3715befdc 100644 --- a/src/task.c +++ b/src/task.c @@ -828,11 +828,6 @@ DLLEXPORT void jl_rethrow_other(jl_value_t *e) throw_internal(e); } -DLLEXPORT void jl_throw_with_superfluous_argument(jl_value_t *e, int line) -{ - jl_throw(e); -} - DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize) { size_t pagesz = jl_page_size; diff --git a/test/backtrace.jl b/test/backtrace.jl index d5b9f3ad97c5f..4f6b2eb6d194b 100644 --- a/test/backtrace.jl +++ b/test/backtrace.jl @@ -99,5 +99,6 @@ let ind2 = find(:test_throw_commoning .== map(b->code_loc(b)[1], b2)) @test !isempty(ind1) @test !isempty(ind2) - @test code_loc(b1[ind1[1]])[3] != code_loc(b2[ind2[1]])[3] + @test code_loc(b1[ind1[1]])[3]::Int == code_loc(b2[ind2[1]])[3]::Int && # source line, for example: essentials.jl:58 + code_loc(b1[ind1[1]])[5]::Int != code_loc(b2[ind2[1]])[5]::Int # inlined line, for example: backtrace.jl:82 end From 4dd3364c6e4a3128f271ee12fad3591b20335a4b Mon Sep 17 00:00:00 2001 From: wildart Date: Wed, 30 Sep 2015 15:14:50 -0400 Subject: [PATCH 0485/1938] moving development related functions to `PkgDev` package --- base/docs/helpdb.jl | 28 --- base/pkg.jl | 26 +-- base/pkg/entry.jl | 259 --------------------- base/pkg/generate.jl | 536 ------------------------------------------- 4 files changed, 3 insertions(+), 846 deletions(-) delete mode 100644 base/pkg/generate.jl diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index aecf883f77206..5bd3b4126224d 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -11648,13 +11648,6 @@ Initialize `Pkg.dir()` as a package directory. This will be done automatically w """ Pkg.init() -doc""" - publish() - -For each new package version tagged in `METADATA` not already published, make sure that the tagged package commits have been pushed to the repo at the registered URL for the package and if they all have, open a pull request to `METADATA`. -""" -Pkg.publish() - doc""" pin(pkg) @@ -11690,13 +11683,6 @@ Returns the version numbers available for package `pkg`. """ Pkg.available(pkg) -doc""" - register(pkg, [url]) - -Register `pkg` at the git URL `url`, defaulting to the configured origin URL of the git repo `Pkg.dir(pkg)`. -""" -Pkg.register(pkg, url=?) - doc""" rm(pkg) @@ -11762,13 +11748,6 @@ Add a requirement entry for `pkg` to `Pkg.dir("REQUIRE")` and call `Pkg.resolve( """ Pkg.add(pkg, vers...) -doc""" - tag(pkg, [ver, [commit]]) - -Tag `commit` as version `ver` of package `pkg` and create a version entry in `METADATA`. If not provided, `commit` defaults to the current commit of the `pkg` repo. If `ver` is one of the symbols `:patch`, `:minor`, `:major` the next patch, minor or major version is used. If `ver` is not provided, it defaults to `:patch`. -""" -Pkg.tag(pkg) - doc""" test() @@ -11783,13 +11762,6 @@ Run the tests for each package in `pkgs` ensuring that each package's test depen """ Pkg.test(pkgs...) -doc""" - generate(pkg,license) - -Generate a new package named `pkg` with one of these license keys: `"MIT"`, `"BSD"` or `"ASL"`. If you want to make a package with a different license, you can edit it afterwards. Generate creates a git repo at `Pkg.dir(pkg)` for the package and inside it `LICENSE.md`, `README.md`, `REQUIRE`, the julia entrypoint `$pkg/src/$pkg.jl`, and Travis and AppVeyor CI configuration files `.travis.yml` and `appveyor.yml`. -""" -Pkg.generate(pkg,license) - doc""" dir() -> AbstractString diff --git a/base/pkg.jl b/base/pkg.jl index 5c49c5ff26860..73596f6779315 100644 --- a/base/pkg.jl +++ b/base/pkg.jl @@ -2,10 +2,9 @@ module Pkg -export Git, Dir, GitHub, Types, Reqs, Cache, Read, Query, Resolve, Write, Generate, Entry, Git +export Git, Dir, GitHub, Types, Reqs, Cache, Read, Query, Resolve, Write, Entry, Git export dir, init, rm, add, available, installed, status, clone, checkout, - update, resolve, register, tag, publish, generate, test, - build, free, pin, PkgError + update, resolve, test, build, free, pin, PkgError const DEFAULT_META = "https://github.com/JuliaLang/METADATA.jl" const META_BRANCH = "metadata-v2" @@ -14,7 +13,7 @@ type PkgError <: Exception msg::AbstractString end -for file in split("dir github types reqs cache read query resolve write generate entry git") +for file in split("dir github types reqs cache read query resolve write entry git") include("pkg/$file.jl") end const cd = Dir.cd @@ -49,28 +48,9 @@ pin(pkg::AbstractString, ver::VersionNumber) = cd(Entry.pin,pkg,ver) update() = cd(Entry.update,Dir.getmetabranch()) resolve() = cd(Entry.resolve) -register(pkg::AbstractString) = cd(Entry.register,pkg) -register(pkg::AbstractString, url::AbstractString) = cd(Entry.register,pkg,url) - -tag(pkg::AbstractString, sym::Symbol=:patch) = cd(Entry.tag,pkg,sym) -tag(pkg::AbstractString, sym::Symbol, commit::AbstractString) = cd(Entry.tag,pkg,sym,false,commit) - -tag(pkg::AbstractString, ver::VersionNumber; force::Bool=false) = cd(Entry.tag,pkg,ver,force) -tag(pkg::AbstractString, ver::VersionNumber, commit::AbstractString; force::Bool=false) = - cd(Entry.tag,pkg,ver,force,commit) - -submit(pkg::AbstractString) = cd(Entry.submit,pkg) -submit(pkg::AbstractString, commit::AbstractString) = cd(Entry.submit,pkg,commit) - -publish() = cd(Entry.publish,Dir.getmetabranch()) - build() = cd(Entry.build) build(pkgs::AbstractString...) = cd(Entry.build,[pkgs...]) -generate(pkg::AbstractString, license::AbstractString; force::Bool=false, authors::Union{AbstractString,Array} = [], config::Dict=Dict()) = - cd(Generate.package,pkg,license,force=force,authors=authors,config=config) - - test(;coverage::Bool=false) = cd(Entry.test; coverage=coverage) test(pkgs::AbstractString...; coverage::Bool=false) = cd(Entry.test,AbstractString[pkgs...]; coverage=coverage) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 8e25da91524b1..c9dec1a77e572 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -371,96 +371,6 @@ function update(branch::AbstractString) updatehook(sort!(collect(keys(installed())))) end -function pull_request(dir::AbstractString, commit::AbstractString="", url::AbstractString="") - with(GitRepo, dir) do repo - if isempty(commit) - commit = string(LibGit2.head_oid(repo)) - else - !LibGit2.iscommit(commit, repo) && throw(PkgError("Cannot find pull commit: $commit")) - end - if isempty(url) - url = LibGit2.getconfig(repo, "remote.origin.url", "") - end - - m = match(LibGit2.GITHUB_REGEX, url) - m === nothing && throw(PkgError("not a GitHub repo URL, can't make a pull request: $url")) - owner, owner_repo = m.captures[2:3] - user = GitHub.user() - info("Forking $owner/$owner_repo to $user") - response = GitHub.fork(owner,owner_repo) - fork = response["ssh_url"] - branch = "pull-request/$(commit[1:8])" - info("Pushing changes as branch $branch") - refspecs = ["HEAD:refs/heads/$branch"] # workaround for $commit:refs/heads/$branch - LibGit2.push(repo, remoteurl=fork, refspecs=refspecs) - pr_url = "$(response["html_url"])/compare/$branch" - info("To create a pull-request, open:\n\n $pr_url\n") - end -end - -function submit(pkg::AbstractString, commit::AbstractString="") - urlpath = joinpath("METADATA",pkg,"url") - url = ispath(urlpath) ? readchomp(urlpath) : "" - pull_request(pkg, commit, url) -end - -function publish(branch::AbstractString) - tags = Dict{ByteString,Vector{ASCIIString}}() - - with(GitRepo, "METADATA") do repo - LibGit2.branch(repo) == branch || - throw(PkgError("METADATA must be on $branch to publish changes")) - LibGit2.fetch(repo) - - ahead_remote, ahead_local = LibGit2.revcount(repo, "origin/$branch", branch) - ahead_remote > 0 && throw(PkgError("METADATA is behind origin/$branch – run `Pkg.update()` before publishing")) - ahead_local == 0 && throw(PkgError("There are no METADATA changes to publish")) - - # get changed files - for path in LibGit2.diff_files(repo, "origin/$branch", LibGit2.Consts.HEAD_FILE) - m = match(r"^(.+?)/versions/([^/]+)/sha1$", path) - m !== nothing && ismatch(Base.VERSION_REGEX, m.captures[2]) || continue - pkg, ver = m.captures; ver = convert(VersionNumber,ver) - sha1 = readchomp(joinpath("METADATA",path)) - old = LibGit2.cat(repo, LibGit2.GitBlob, "origin/$branch:$path") - old !== nothing && old != sha1 && throw(PkgError("$pkg v$ver SHA1 changed in METADATA – refusing to publish")) - with(GitRepo, pkg) do pkg_repo - tag_name = "v$ver" - tag_commit = LibGit2.revparseid(pkg_repo, "$(tag_name)^{commit}") - LibGit2.iszero(tag_commit) || string(tag_commit) == sha1 || return false - haskey(tags,pkg) || (tags[pkg] = ASCIIString[]) - push!(tags[pkg], tag_name) - return true - end || throw(PkgError("$pkg v$ver is incorrectly tagged – $sha1 expected")) - end - isempty(tags) && info("No new package versions to publish") - info("Validating METADATA") - check_metadata(Set(keys(tags))) - end - - for pkg in sort!(collect(keys(tags))) - with(GitRepo, pkg) do pkg_repo - forced = ASCIIString[] - unforced = ASCIIString[] - for tag in tags[pkg] - ver = convert(VersionNumber,tag) - push!(isrewritable(ver) ? forced : unforced, tag) - end - if !isempty(forced) - info("Pushing $pkg temporary tags: ", join(forced,", ")) - LibGit2.push(pkg_repo, remote="origin", force=true, - refspecs=["refs/tags/$tag:refs/tags/$tag" for tag in forced]) - end - if !isempty(unforced) - info("Pushing $pkg permanent tags: ", join(unforced,", ")) - LibGit2.push(pkg_repo, remote="origin", - refspecs=["refs/tags/$tag:refs/tags/$tag" for tag in unforced]) - end - end - end - info("Submitting METADATA changes") - pull_request("METADATA") -end function resolve( reqs :: Dict = Reqs.parse("REQUIRE"), @@ -547,175 +457,6 @@ function resolve( build(map(x->x[1], filter(x -> x[2][2] !== nothing, changes))) end -function write_tag_metadata(repo::GitRepo, pkg::AbstractString, ver::VersionNumber, commit::AbstractString, force::Bool=false) - content = with(GitRepo,pkg) do pkg_repo - LibGit2.cat(pkg_repo, LibGit2.GitBlob, "$commit:REQUIRE") - end - reqs = content !== nothing ? Reqs.read(split(content, '\n', keep=false)) : Reqs.Line[] - cd("METADATA") do - d = joinpath(pkg,"versions",string(ver)) - mkpath(d) - sha1file = joinpath(d,"sha1") - if !force && ispath(sha1file) - current = readchomp(sha1file) - current == commit || - throw(PkgError("$pkg v$ver is already registered as $current, bailing")) - end - open(io->println(io,commit), sha1file, "w") - LibGit2.add!(repo, sha1file) - reqsfile = joinpath(d,"requires") - if isempty(reqs) - ispath(reqsfile) && LibGit2.remove!(repo, reqsfile) - else - Reqs.write(reqsfile,reqs) - LibGit2.add!(repo, reqsfile) - end - end - return nothing -end - -function register(pkg::AbstractString, url::AbstractString) - ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) - isfile("METADATA",pkg,"url") && throw(PkgError("$pkg already registered")) - LibGit2.transact(GitRepo("METADATA")) do repo - # Get versions from package repo - versions = with(GitRepo, pkg) do pkg_repo - tags = filter(t->startswith(t,"v"), LibGit2.tag_list(pkg_repo)) - filter!(tag->ismatch(Base.VERSION_REGEX,tag), tags) - [ - convert(VersionNumber,tag) => string(LibGit2.revparseid(pkg_repo, "$tag^{commit}")) - for tag in tags - ] - end - # Register package url in METADATA - cd("METADATA") do - info("Registering $pkg at $url") - mkdir(pkg) - path = joinpath(pkg,"url") - open(io->println(io,url), path, "w") - LibGit2.add!(repo, path) - end - # Register package version in METADATA - vers = sort!(collect(keys(versions))) - for ver in vers - info("Tagging $pkg v$ver") - write_tag_metadata(repo, pkg,ver,versions[ver]) - end - # Commit changes in METADATA - if LibGit2.isdirty(repo) - info("Committing METADATA for $pkg") - msg = "Register $pkg" - if !isempty(versions) - msg *= ": $(join(map(v->"v$v", vers),", "))" - end - LibGit2.commit(repo, msg) - else - info("No METADATA changes to commit") - end - end - return -end - -function register(pkg::AbstractString) - url = "" - try - url = LibGit2.getconfig(pkg, "remote.origin.url", "") - catch err - throw(PkgError("$pkg: $err")) - end - !isempty(url) || throw(PkgError("$pkg: no URL configured")) - register(pkg, GitHub.normalize_url(url)) -end - -function isrewritable(v::VersionNumber) - thispatch(v)==v"0" || - length(v.prerelease)==1 && isempty(v.prerelease[1]) || - length(v.build)==1 && isempty(v.build[1]) -end - -nextbump(v::VersionNumber) = isrewritable(v) ? v : nextpatch(v) - -function tag(pkg::AbstractString, ver::Union{Symbol,VersionNumber}, force::Bool=false, commitish::AbstractString="HEAD") - ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) - with(GitRepo,"METADATA") do repo - LibGit2.isdirty(repo, pkg) && throw(PkgError("METADATA/$pkg is dirty – commit or stash changes to tag")) - end - with(GitRepo,pkg) do repo - LibGit2.isdirty(repo) && throw(PkgError("$pkg is dirty – commit or stash changes to tag")) - commit = string(LibGit2.revparseid(repo, commitish)) - registered = isfile("METADATA",pkg,"url") - - if !force - if registered - avail = Read.available(pkg) - existing = VersionNumber[keys(Read.available(pkg))...] - ancestors = filter(v->LibGit2.is_ancestor_of(avail[v].sha1, commit, repo), existing) - else - tags = filter(t->startswith(t,"v"), LibGit2.tag_list(repo)) - filter!(tag->ismatch(Base.VERSION_REGEX,tag), tags) - existing = VersionNumber[tags...] - filter!(tags) do tag - sha1 = string(LibGit2.revparseid(repo, "$tag^{commit}")) - LibGit2.is_ancestor_of(sha1, commit, repo) - end - ancestors = VersionNumber[tags...] - end - sort!(existing) - if isa(ver,Symbol) - prv = isempty(existing) ? v"0" : - isempty(ancestors) ? maximum(existing) : maximum(ancestors) - ver = (ver == :bump ) ? nextbump(prv) : - (ver == :patch) ? nextpatch(prv) : - (ver == :minor) ? nextminor(prv) : - (ver == :major) ? nextmajor(prv) : - throw(PkgError("invalid version selector: $ver")) - end - isrewritable(ver) && filter!(v->v!=ver,existing) - check_new_version(existing,ver) - end - # TODO: check that SHA1 isn't the same as another version - info("Tagging $pkg v$ver") - LibGit2.tag_create(repo, "v$ver", commit, - msg=(!isrewritable(ver) ? "$pkg v$ver [$(commit[1:10])]" : ""), - force=(force || isrewritable(ver)) ) - registered || return - try - LibGit2.transact(GitRepo("METADATA")) do repo - write_tag_metadata(repo, pkg, ver, commit, force) - if LibGit2.isdirty(repo) - info("Committing METADATA for $pkg") - LibGit2.commit(repo, "Tag $pkg v$ver") - else - info("No METADATA changes to commit") - end - end - catch - LibGit2.tag_delete(repo, "v$ver") - rethrow() - end - end - return -end - -function check_metadata(pkgs::Set{ByteString} = Set{ByteString}()) - avail = Read.available() - deps, conflicts = Query.dependencies(avail) - - for (dp,dv) in deps, (v,a) in dv, p in keys(a.requires) - haskey(deps, p) || throw(PkgError("package $dp v$v requires a non-registered package: $p")) - end - - problematic = Resolve.sanity_check(deps, pkgs) - if !isempty(problematic) - msg = "packages with unsatisfiable requirements found:\n" - for (p, vn, rp) in problematic - msg *= " $p v$vn – no valid versions exist for package $rp\n" - end - throw(PkgError(msg)) - end - return -end - function warnbanner(msg...; label="[ WARNING ]", prefix="") cols = Base.tty_size()[2] warn(prefix="", Base.cpad(label,cols,"=")) diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl deleted file mode 100644 index b3b1aee469580..0000000000000 --- a/base/pkg/generate.jl +++ /dev/null @@ -1,536 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -module Generate - -import ...LibGit2, ..Read, ...Pkg.PkgError -importall ...LibGit2 - -copyright_year() = string(Dates.year(Dates.today())) -copyright_name(repo::GitRepo) = LibGit2.getconfig(repo, "user.name", "") -github_user() = LibGit2.getconfig("github.user", "") - -function git_contributors(repo::GitRepo, n::Int=typemax(Int)) - contrib = Dict() - for sig in LibGit2.authors(repo) - if haskey(contrib, sig.email) - contrib[sig.email][1] += 1 - else - contrib[sig.email] = [1, sig.name] - end - end - - names = Dict() - for (commits,name) in values(contrib) - names[name] = get(names,name,0) + commits - end - names = sort!(collect(keys(names)),by=name->names[name],rev=true) - length(names) <= n ? names : [names[1:n]; "et al."] -end - -function package( - pkg::AbstractString, - license::AbstractString; - force::Bool = false, - authors::Union{AbstractString,Array} = "", - years::Union{Int,AbstractString} = copyright_year(), - user::AbstractString = github_user(), - config::Dict = Dict(), -) - isnew = !ispath(pkg) - try - repo = if isnew - url = isempty(user) ? "" : "https://github.com/$user/$pkg.jl.git" - Generate.init(pkg,url,config=config) - else - repo = GitRepo(pkg) - if LibGit2.isdirty(repo) - finalize(repo) - throw(PkgError("$pkg is dirty – commit or stash your changes")) - end - repo - end - - LibGit2.transact(repo) do repo - if isempty(authors) - authors = isnew ? copyright_name(repo) : git_contributors(repo,5) - end - - files = [Generate.license(pkg,license,years,authors,force=force), - Generate.readme(pkg,user,force=force), - Generate.entrypoint(pkg,force=force), - Generate.tests(pkg,force=force), - Generate.require(pkg,force=force), - Generate.travis(pkg,force=force), - Generate.appveyor(pkg,force=force), - Generate.gitignore(pkg,force=force) ] - - msg = """ - $pkg.jl $(isnew ? "generated" : "regenerated") files. - - license: $license - authors: $(join(vcat(authors),", ")) - years: $years - user: $user - - Julia Version $VERSION [$(Base.GIT_VERSION_INFO.commit_short)] - """ - LibGit2.add!(repo, files..., flags = LibGit2.Consts.INDEX_ADD_FORCE) - if isnew - info("Committing $pkg generated files") - LibGit2.commit(repo, msg) - elseif LibGit2.isdirty(repo) - LibGit2.remove!(repo, files...) - info("Regenerated files left unstaged, use `git add -p` to select") - open(io->print(io,msg), joinpath(LibGit2.gitdir(repo),"MERGE_MSG"), "w") - else - info("Regenerated files are unchanged") - end - end - catch - isnew && rm(pkg, recursive=true) - rethrow() - end - return -end - -function init(pkg::AbstractString, url::AbstractString=""; config::Dict=Dict()) - if !ispath(pkg) - info("Initializing $pkg repo: $(abspath(pkg))") - repo = LibGit2.init(pkg) - try - with(GitConfig, repo) do cfg - for (key,val) in config - LibGit2.set!(cfg, key, val) - end - end - LibGit2.commit(repo, "initial empty commit") - catch err - throw(PkgError("Unable to initialize $pkg package: $err")) - end - else - repo = GitRepo(pkg) - end - try - if !isempty(url) - info("Origin: $url") - with(LibGit2.GitRemote, repo, "origin", url) do rmt - LibGit2.save(rmt) - end - LibGit2.set_remote_url(repo, url) - end - end - return repo -end - -function genfile(f::Function, pkg::AbstractString, file::AbstractString, force::Bool=false) - path = joinpath(pkg,file) - if force || !ispath(path) - info("Generating $file") - mkpath(dirname(path)) - open(f, path, "w") - return file - end - return "" -end - -function license(pkg::AbstractString, - license::AbstractString, - years::Union{Int,AbstractString}, - authors::Union{AbstractString,Array}; - force::Bool=false) - file = genfile(pkg,"LICENSE.md",force) do io - if !haskey(LICENSES,license) - licenses = join(sort!(collect(keys(LICENSES)), by=lowercase), ", ") - throw(PkgError("$license is not a known license choice, choose one of: $licenses.")) - end - print(io, LICENSES[license](pkg, string(years), authors)) - end - !isempty(file) || info("License file exists, leaving unmodified; use `force=true` to overwrite") - file -end - -function readme(pkg::AbstractString, user::AbstractString=""; force::Bool=false) - genfile(pkg,"README.md",force) do io - println(io, "# $pkg") - isempty(user) && return - url = "https://travis-ci.org/$user/$pkg.jl" - println(io, "\n[![Build Status]($url.svg?branch=master)]($url)") - end -end - -function tests(pkg::AbstractString; force::Bool=false) - genfile(pkg,"test/runtests.jl",force) do io - print(io, """ - using $pkg - using Base.Test - - # write your own tests here - @test 1 == 1 - """) - end -end - -function versionfloor(ver::VersionNumber) - # return "major.minor" for the most recent release version relative to ver - # for prereleases with ver.minor == ver.patch == 0, return "major-" since we - # don't know what the most recent minor version is for the previous major - if isempty(ver.prerelease) || ver.patch > 0 - return string(ver.major, '.', ver.minor) - elseif ver.minor > 0 - return string(ver.major, '.', ver.minor - 1) - else - return string(ver.major, '-') - end -end - -function require(pkg::AbstractString; force::Bool=false) - genfile(pkg,"REQUIRE",force) do io - print(io, """ - julia $(versionfloor(VERSION)) - """) - end -end - -function travis(pkg::AbstractString; force::Bool=false) - genfile(pkg,".travis.yml",force) do io - print(io, """ - # Documentation: http://docs.travis-ci.com/user/languages/julia/ - language: julia - os: - - linux - - osx - julia: - - release - - nightly - notifications: - email: false - # uncomment the following lines to override the default test script - #script: - # - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi - # - julia -e 'Pkg.clone(pwd()); Pkg.build("$pkg"); Pkg.test("$pkg"; coverage=true)' - """) - end -end - -function appveyor(pkg::AbstractString; force::Bool=false) - vf = versionfloor(VERSION) - if vf[end] == '-' # don't know what previous release was - vf = string(VERSION.major, '.', VERSION.minor) - rel32 = "# - JULIAVERSION: \"julialang/bin/winnt/x86/$vf/julia-$vf-latest-win32.exe\"" - rel64 = "# - JULIAVERSION: \"julialang/bin/winnt/x64/$vf/julia-$vf-latest-win64.exe\"" - else - rel32 = " - JULIAVERSION: \"julialang/bin/winnt/x86/$vf/julia-$vf-latest-win32.exe\"" - rel64 = " - JULIAVERSION: \"julialang/bin/winnt/x64/$vf/julia-$vf-latest-win64.exe\"" - end - genfile(pkg,"appveyor.yml",force) do io - print(io, """ - environment: - matrix: - $rel32 - $rel64 - - JULIAVERSION: "julianightlies/bin/winnt/x86/julia-latest-win32.exe" - - JULIAVERSION: "julianightlies/bin/winnt/x64/julia-latest-win64.exe" - - branches: - only: - - master - - /release-.*/ - - notifications: - - provider: Email - on_build_success: false - on_build_failure: false - on_build_status_changed: false - - install: - # Download most recent Julia Windows binary - - ps: (new-object net.webclient).DownloadFile( - \$("http://s3.amazonaws.com/"+\$env:JULIAVERSION), - "C:\\projects\\julia-binary.exe") - # Run installer silently, output to C:\\projects\\julia - - C:\\projects\\julia-binary.exe /S /D=C:\\projects\\julia - - build_script: - # Need to convert from shallow to complete for Pkg.clone to work - - IF EXIST .git\\shallow (git fetch --unshallow) - - C:\\projects\\julia\\bin\\julia -e "versioninfo(); - Pkg.clone(pwd(), \\"$pkg\\"); Pkg.build(\\"$pkg\\")" - - test_script: - - C:\\projects\\julia\\bin\\julia --check-bounds=yes -e "Pkg.test(\\"$pkg\\")" - """) - end -end - -function gitignore(pkg::AbstractString; force::Bool=false) - genfile(pkg,".gitignore",force) do io - print(io, """ - *.jl.cov - *.jl.*.cov - *.jl.mem - """) - end -end - -function entrypoint(pkg::AbstractString; force::Bool=false) - genfile(pkg,"src/$pkg.jl",force) do io - print(io, """ - module $pkg - - # package code goes here - - end # module - """) - end -end - -copyright(years::AbstractString, authors::AbstractString) = "> Copyright (c) $years: $authors." - -function copyright(years::AbstractString, authors::Array) - text = "> Copyright (c) $years:" - for author in authors - text *= "\n> * $author" - end - return text -end - -mit(pkg::AbstractString, years::AbstractString, authors::Union{AbstractString,Array}) = -""" -The $pkg.jl package is licensed under the MIT "Expat" License: - -$(copyright(years,authors)) -> -> Permission is hereby granted, free of charge, to any person obtaining -> a copy of this software and associated documentation files (the -> "Software"), to deal in the Software without restriction, including -> without limitation the rights to use, copy, modify, merge, publish, -> distribute, sublicense, and/or sell copies of the Software, and to -> permit persons to whom the Software is furnished to do so, subject to -> the following conditions: -> -> The above copyright notice and this permission notice shall be -> included in all copies or substantial portions of the Software. -> -> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -> IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -> CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -> TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -> SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -""" - -bsd(pkg::AbstractString, years::AbstractString, authors::Union{AbstractString,Array}) = -""" -The $pkg.jl package is licensed under the Simplified "2-clause" BSD License: - -$(copyright(years,authors)) -> -> Redistribution and use in source and binary forms, with or without -> modification, are permitted provided that the following conditions are -> met: -> -> 1. Redistributions of source code must retain the above copyright -> notice, this list of conditions and the following disclaimer. -> 2. Redistributions in binary form must reproduce the above copyright -> notice, this list of conditions and the following disclaimer in the -> documentation and/or other materials provided with the distribution. -> -> THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -> "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -> LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -> A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -> OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -> SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -> LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -> DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -> THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -> (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -> OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" - -asl(pkg::AbstractString, years::AbstractString, authors::Union{AbstractString,Array}) = -""" -The $pkg.jl package is licensed under version 2.0 of the Apache License: - -$(copyright(years,authors)) -> -> Apache License -> Version 2.0, January 2004 -> http://www.apache.org/licenses/ -> -> TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -> -> 1. Definitions. -> -> "License" shall mean the terms and conditions for use, reproduction, -> and distribution as defined by Sections 1 through 9 of this document. -> -> "Licensor" shall mean the copyright owner or entity authorized by -> the copyright owner that is granting the License. -> -> "Legal Entity" shall mean the union of the acting entity and all -> other entities that control, are controlled by, or are under common -> control with that entity. For the purposes of this definition, -> "control" means (i) the power, direct or indirect, to cause the -> direction or management of such entity, whether by contract or -> otherwise, or (ii) ownership of fifty percent (50%) or more of the -> outstanding shares, or (iii) beneficial ownership of such entity. -> -> "You" (or "Your") shall mean an individual or Legal Entity -> exercising permissions granted by this License. -> -> "Source" form shall mean the preferred form for making modifications, -> including but not limited to software source code, documentation -> source, and configuration files. -> -> "Object" form shall mean any form resulting from mechanical -> transformation or translation of a Source form, including but -> not limited to compiled object code, generated documentation, -> and conversions to other media types. -> -> "Work" shall mean the work of authorship, whether in Source or -> Object form, made available under the License, as indicated by a -> copyright notice that is included in or attached to the work -> (an example is provided in the Appendix below). -> -> "Derivative Works" shall mean any work, whether in Source or Object -> form, that is based on (or derived from) the Work and for which the -> editorial revisions, annotations, elaborations, or other modifications -> represent, as a whole, an original work of authorship. For the purposes -> of this License, Derivative Works shall not include works that remain -> separable from, or merely link (or bind by name) to the interfaces of, -> the Work and Derivative Works thereof. -> -> "Contribution" shall mean any work of authorship, including -> the original version of the Work and any modifications or additions -> to that Work or Derivative Works thereof, that is intentionally -> submitted to Licensor for inclusion in the Work by the copyright owner -> or by an individual or Legal Entity authorized to submit on behalf of -> the copyright owner. For the purposes of this definition, "submitted" -> means any form of electronic, verbal, or written communication sent -> to the Licensor or its representatives, including but not limited to -> communication on electronic mailing lists, source code control systems, -> and issue tracking systems that are managed by, or on behalf of, the -> Licensor for the purpose of discussing and improving the Work, but -> excluding communication that is conspicuously marked or otherwise -> designated in writing by the copyright owner as "Not a Contribution." -> -> "Contributor" shall mean Licensor and any individual or Legal Entity -> on behalf of whom a Contribution has been received by Licensor and -> subsequently incorporated within the Work. -> -> 2. Grant of Copyright License. Subject to the terms and conditions of -> this License, each Contributor hereby grants to You a perpetual, -> worldwide, non-exclusive, no-charge, royalty-free, irrevocable -> copyright license to reproduce, prepare Derivative Works of, -> publicly display, publicly perform, sublicense, and distribute the -> Work and such Derivative Works in Source or Object form. -> -> 3. Grant of Patent License. Subject to the terms and conditions of -> this License, each Contributor hereby grants to You a perpetual, -> worldwide, non-exclusive, no-charge, royalty-free, irrevocable -> (except as stated in this section) patent license to make, have made, -> use, offer to sell, sell, import, and otherwise transfer the Work, -> where such license applies only to those patent claims licensable -> by such Contributor that are necessarily infringed by their -> Contribution(s) alone or by combination of their Contribution(s) -> with the Work to which such Contribution(s) was submitted. If You -> institute patent litigation against any entity (including a -> cross-claim or counterclaim in a lawsuit) alleging that the Work -> or a Contribution incorporated within the Work constitutes direct -> or contributory patent infringement, then any patent licenses -> granted to You under this License for that Work shall terminate -> as of the date such litigation is filed. -> -> 4. Redistribution. You may reproduce and distribute copies of the -> Work or Derivative Works thereof in any medium, with or without -> modifications, and in Source or Object form, provided that You -> meet the following conditions: -> -> (a) You must give any other recipients of the Work or -> Derivative Works a copy of this License; and -> -> (b) You must cause any modified files to carry prominent notices -> stating that You changed the files; and -> -> (c) You must retain, in the Source form of any Derivative Works -> that You distribute, all copyright, patent, trademark, and -> attribution notices from the Source form of the Work, -> excluding those notices that do not pertain to any part of -> the Derivative Works; and -> -> (d) If the Work includes a "NOTICE" text file as part of its -> distribution, then any Derivative Works that You distribute must -> include a readable copy of the attribution notices contained -> within such NOTICE file, excluding those notices that do not -> pertain to any part of the Derivative Works, in at least one -> of the following places: within a NOTICE text file distributed -> as part of the Derivative Works; within the Source form or -> documentation, if provided along with the Derivative Works; or, -> within a display generated by the Derivative Works, if and -> wherever such third-party notices normally appear. The contents -> of the NOTICE file are for informational purposes only and -> do not modify the License. You may add Your own attribution -> notices within Derivative Works that You distribute, alongside -> or as an addendum to the NOTICE text from the Work, provided -> that such additional attribution notices cannot be construed -> as modifying the License. -> -> You may add Your own copyright statement to Your modifications and -> may provide additional or different license terms and conditions -> for use, reproduction, or distribution of Your modifications, or -> for any such Derivative Works as a whole, provided Your use, -> reproduction, and distribution of the Work otherwise complies with -> the conditions stated in this License. -> -> 5. Submission of Contributions. Unless You explicitly state otherwise, -> any Contribution intentionally submitted for inclusion in the Work -> by You to the Licensor shall be under the terms and conditions of -> this License, without any additional terms or conditions. -> Notwithstanding the above, nothing herein shall supersede or modify -> the terms of any separate license agreement you may have executed -> with Licensor regarding such Contributions. -> -> 6. Trademarks. This License does not grant permission to use the trade -> names, trademarks, service marks, or product names of the Licensor, -> except as required for reasonable and customary use in describing the -> origin of the Work and reproducing the content of the NOTICE file. -> -> 7. Disclaimer of Warranty. Unless required by applicable law or -> agreed to in writing, Licensor provides the Work (and each -> Contributor provides its Contributions) on an "AS IS" BASIS, -> WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -> implied, including, without limitation, any warranties or conditions -> of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A -> PARTICULAR PURPOSE. You are solely responsible for determining the -> appropriateness of using or redistributing the Work and assume any -> risks associated with Your exercise of permissions under this License. -> -> 8. Limitation of Liability. In no event and under no legal theory, -> whether in tort (including negligence), contract, or otherwise, -> unless required by applicable law (such as deliberate and grossly -> negligent acts) or agreed to in writing, shall any Contributor be -> liable to You for damages, including any direct, indirect, special, -> incidental, or consequential damages of any character arising as a -> result of this License or out of the use or inability to use the -> Work (including but not limited to damages for loss of goodwill, -> work stoppage, computer failure or malfunction, or any and all -> other commercial damages or losses), even if such Contributor -> has been advised of the possibility of such damages. -> -> 9. Accepting Warranty or Additional Liability. While redistributing -> the Work or Derivative Works thereof, You may choose to offer, -> and charge a fee for, acceptance of support, warranty, indemnity, -> or other liability obligations and/or rights consistent with this -> License. However, in accepting such obligations, You may act only -> on Your own behalf and on Your sole responsibility, not on behalf -> of any other Contributor, and only if You agree to indemnify, -> defend, and hold each Contributor harmless for any liability -> incurred by, or claims asserted against, such Contributor by reason -> of your accepting any such warranty or additional liability. -""" - -const LICENSES = Dict("MIT" => mit, "BSD" => bsd, "ASL" => asl) - -end # module From 698fa612f68bb22e2370b267d9b9b5a472cf1550 Mon Sep 17 00:00:00 2001 From: wildart Date: Wed, 30 Sep 2015 15:41:03 -0400 Subject: [PATCH 0486/1938] removed dev related tests --- test/pkg.jl | 121 ---------------------------------------------------- 1 file changed, 121 deletions(-) diff --git a/test/pkg.jl b/test/pkg.jl index f0c56efc82d41..c27cda05b7318 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -115,124 +115,3 @@ temp_pkg_dir() do Pkg.rm("Example") @test isempty(Pkg.installed()) end - -# testing a package with test dependencies causes them to be installed for the duration of the test -temp_pkg_dir() do - Pkg.generate("PackageWithTestDependencies", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) - @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] - @test readall(Pkg.dir("PackageWithTestDependencies","REQUIRE")) == "julia $(Pkg.Generate.versionfloor(VERSION))\n" - - isdir(Pkg.dir("PackageWithTestDependencies","test")) || mkdir(Pkg.dir("PackageWithTestDependencies","test")) - open(Pkg.dir("PackageWithTestDependencies","test","REQUIRE"),"w") do f - println(f,"Example") - end - - open(Pkg.dir("PackageWithTestDependencies","test","runtests.jl"),"w") do f - println(f,"using Base.Test") - println(f,"@test haskey(Pkg.installed(), \"Example\")") - end - - Pkg.resolve() - @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] - - Pkg.test("PackageWithTestDependencies") - - @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] - - # trying to pin an unregistered package errors - try - Pkg.pin("PackageWithTestDependencies", v"1.0.0") - error("unexpected") - catch err - @test isa(err, PkgError) - @test err.msg == "PackageWithTestDependencies cannot be pinned – not a registered package" - end -end - -# testing a package with no runtests.jl errors -temp_pkg_dir() do - Pkg.generate("PackageWithNoTests", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) - - if isfile(Pkg.dir("PackageWithNoTests", "test", "runtests.jl")) - rm(Pkg.dir("PackageWithNoTests", "test", "runtests.jl")) - end - - try - Pkg.test("PackageWithNoTests") - error("unexpected") - catch err - @test err.msg == "PackageWithNoTests did not provide a test/runtests.jl file" - end -end - -# testing a package with failing tests errors -temp_pkg_dir() do - Pkg.generate("PackageWithFailingTests", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) - - isdir(Pkg.dir("PackageWithFailingTests","test")) || mkdir(Pkg.dir("PackageWithFailingTests","test")) - open(Pkg.dir("PackageWithFailingTests", "test", "runtests.jl"),"w") do f - println(f,"using Base.Test") - println(f,"@test false") - end - - try - Pkg.test("PackageWithFailingTests") - error("unexpected") - catch err - @test err.msg == "PackageWithFailingTests had test errors" - end -end - -# Testing with code-coverage -temp_pkg_dir() do - Pkg.generate("PackageWithCodeCoverage", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) - - src = """ -module PackageWithCodeCoverage - -export f1, f2, f3, untested - -f1(x) = 2x -f2(x) = f1(x) -function f3(x) - 3x -end -untested(x) = 7 - -end""" - linetested = [false, false, false, false, true, true, false, true, false, false] - open(Pkg.dir("PackageWithCodeCoverage", "src", "PackageWithCodeCoverage.jl"), "w") do f - println(f, src) - end - isdir(Pkg.dir("PackageWithCodeCoverage","test")) || mkdir(Pkg.dir("PackageWithCodeCoverage","test")) - open(Pkg.dir("PackageWithCodeCoverage", "test", "runtests.jl"),"w") do f - println(f,"using PackageWithCodeCoverage, Base.Test") - println(f,"@test f2(2) == 4") - println(f,"@test f3(5) == 15") - end - - Pkg.test("PackageWithCodeCoverage") - covdir = Pkg.dir("PackageWithCodeCoverage","src") - covfiles = filter!(x -> contains(x, "PackageWithCodeCoverage.jl") && contains(x,".cov"), readdir(covdir)) - @test isempty(covfiles) - Pkg.test("PackageWithCodeCoverage", coverage=true) - covfiles = filter!(x -> contains(x, "PackageWithCodeCoverage.jl") && contains(x,".cov"), readdir(covdir)) - @test !isempty(covfiles) - for file in covfiles - @test isfile(joinpath(covdir,file)) - covstr = readall(joinpath(covdir,file)) - srclines = split(src, '\n') - covlines = split(covstr, '\n') - for i = 1:length(linetested) - covline = (linetested[i] ? " 1 " : " - ")*srclines[i] - @test covlines[i] == covline - end - end -end - -# issue #13374 -temp_pkg_dir() do - Pkg.generate("Foo", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) - Pkg.tag("Foo") - Pkg.tag("Foo") -end From fcfac39e7d29b34dd8cd71158d74c9de673009bb Mon Sep 17 00:00:00 2001 From: wildart Date: Mon, 5 Oct 2015 01:10:53 -0400 Subject: [PATCH 0487/1938] fix test helper to keep generated data --- test/pkg.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pkg.jl b/test/pkg.jl index c27cda05b7318..24f344735d23f 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -2,7 +2,7 @@ import Base.Pkg.PkgError -function temp_pkg_dir(fn::Function) +function temp_pkg_dir(fn::Function, remove_tmp_dir::Bool=true) # Used in tests below to setup and teardown a sandboxed package directory const tmpdir = ENV["JULIA_PKGDIR"] = joinpath(tempdir(),randstring()) @test !isdir(Pkg.dir()) @@ -13,7 +13,7 @@ function temp_pkg_dir(fn::Function) fn() finally - rm(tmpdir, recursive=true) + remove_tmp_dir && rm(tmpdir, recursive=true) end end From c07de056afc6ae6b33837279b74005d01bfb5754 Mon Sep 17 00:00:00 2001 From: wildart Date: Wed, 7 Oct 2015 22:48:39 -0400 Subject: [PATCH 0488/1938] removed github module --- base/pkg.jl | 4 +- base/pkg/entry.jl | 2 +- base/pkg/github.jl | 144 --------------------------------------------- 3 files changed, 3 insertions(+), 147 deletions(-) delete mode 100644 base/pkg/github.jl diff --git a/base/pkg.jl b/base/pkg.jl index 73596f6779315..fb7d3fbde2a35 100644 --- a/base/pkg.jl +++ b/base/pkg.jl @@ -2,7 +2,7 @@ module Pkg -export Git, Dir, GitHub, Types, Reqs, Cache, Read, Query, Resolve, Write, Entry, Git +export Git, Dir, Types, Reqs, Cache, Read, Query, Resolve, Write, Entry, Git export dir, init, rm, add, available, installed, status, clone, checkout, update, resolve, test, build, free, pin, PkgError @@ -13,7 +13,7 @@ type PkgError <: Exception msg::AbstractString end -for file in split("dir github types reqs cache read query resolve write entry git") +for file in split("dir types reqs cache read query resolve write entry git") include("pkg/$file.jl") end const cd = Dir.cd diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index c9dec1a77e572..a487d732e3f91 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -3,7 +3,7 @@ module Entry import Base: thispatch, nextpatch, nextminor, nextmajor, check_new_version -import ..Reqs, ..Read, ..Query, ..Resolve, ..Cache, ..Write, ..GitHub, ..Dir +import ..Reqs, ..Read, ..Query, ..Resolve, ..Cache, ..Write, ..Dir import ...LibGit2 importall ...LibGit2 import ...Pkg.PkgError diff --git a/base/pkg/github.jl b/base/pkg/github.jl deleted file mode 100644 index 1eced0e9aa99c..0000000000000 --- a/base/pkg/github.jl +++ /dev/null @@ -1,144 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -module GitHub - -import Main, ...LibGit2, ..Dir, ...Pkg.PkgError - -const AUTH_NOTE = "Julia Package Manager" -const AUTH_DATA = Dict{Any,Any}( - "scopes" => ["repo"], - "note" => AUTH_NOTE, - "note_url" => "http://docs.julialang.org/en/latest/manual/packages/", -) - -function user() - usr = LibGit2.getconfig("github.user", "") - if isempty(usr) #TODO: add `config` command to Git REPL and change below info - throw(PkgError(""" - no GitHub user name configured; please configure it with: - - git config --global github.user USERNAME - - where USERNAME is replaced with your GitHub user name. - """)) - end - return usr -end - -function json() - isdefined(:JSON) || try eval(Main, :(import JSON)) - catch err - warn(err) - throw(PkgError("using the GitHub API requires having the JSON package installed ")) - end - Main.JSON -end - -function curl(url::AbstractString, opts::Cmd=``) - success(`curl --version`) || throw(PkgError("using the GitHub API requires having `curl` installed")) - out, proc = open(`curl -i -s -S $opts $url`,"r") - head = readline(out) - status = parse(Int,split(head,r"\s+";limit=3)[2]) - header = Dict{AbstractString,AbstractString}() - for line in eachline(out) - if !ismatch(r"^\s*$",line) - (k,v) = split(line, r":\s*"; limit=2) - header[k] = v - continue - end - wait(proc); return status, header, readall(out) - end - throw(PkgError("strangely formatted HTTP response")) -end -curl(url::AbstractString, data::Void, opts::Cmd=``) = curl(url,opts) -curl(url::AbstractString, data, opts::Cmd=``) = - curl(url,`--data $(sprint(io->json().print(io,data))) $opts`) - -function delete_token() - tokfile = Dir.path(".github","token") - Base.rm(tokfile) - info("Could not authenticate with existing token. Deleting token and trying again.") -end - -readtoken(tokfile=Dir.path(".github","token")) = isfile(tokfile) ? strip(readchomp(tokfile)) : "" - -function token(user::AbstractString=user()) - tokfile = Dir.path(".github","token") - tok = readtoken(tokfile) - !isempty(tok) && return tok - - params = merge(AUTH_DATA, Dict("fingerprint" => randstring(40))) - status, header, content = curl("https://api.github.com/authorizations",params,`-u $user`) - tfa = false - - # Check for two-factor authentication - if status == 401 && get(header, "X-GitHub-OTP", "") |> x->startswith(x, "required") && isinteractive() - tfa = true - info("Two-factor authentication in use. Enter auth code. (You may have to re-enter your password.)") - print(STDERR, "Authentication code: ") - code = readline(STDIN) |> chomp - status, header, content = curl("https://api.github.com/authorizations",params,`-H "X-GitHub-OTP: $code" -u $user`) - end - - if status == 422 - error_code = json().parse(content)["errors"][1]["code"] - throw(PkgError("GitHub returned validation error (422): $error_code: $(json().parse(content)["message"])")) - else - (status != 401 && status != 403) || throw(PkgError("$status: $(json().parse(content)["message"])")) - tok = json().parse(content)["token"] - end - - mkpath(dirname(tokfile)) - open(io->println(io,tok),tokfile,"w") - return tok -end - -function req(resource::AbstractString, data, opts::Cmd=``) - url = "https://api.github.com/$resource" - status, header, content = curl(url,data,`-u $(token()):x-oauth-basic $opts`) - response = json().parse(content) - status, response -end - -GET(resource::AbstractString, data, opts::Cmd=``) = req(resource,data,opts) -HEAD(resource::AbstractString, data, opts::Cmd=``) = req(resource,data,`-I $opts`) -PUT(resource::AbstractString, data, opts::Cmd=``) = req(resource,data,`-X PUT $opts`) -POST(resource::AbstractString, data, opts::Cmd=``) = req(resource,data,`-X POST $opts`) -PATCH(resource::AbstractString, data, opts::Cmd=``) = req(resource,data,`-X PATCH $opts`) -DELETE(resource::AbstractString, data, opts::Cmd=``) = req(resource,data,`-X DELETE $opts`) - -for m in (:GET,:HEAD,:PUT,:POST,:PATCH,:DELETE) - @eval $m(resource::AbstractString, opts::Cmd=``) = $m(resource,nothing,opts) -end - -function pushable(owner::AbstractString, repo::AbstractString, user::AbstractString=user()) - status, response = HEAD("repos/$owner/$repo") - status == 404 && throw(PkgError("repo $owner/$repo does not exist")) - status, response = GET("repos/$owner/$repo/collaborators/$user") - status == 204 && return true - status == 404 && return false - throw(PkgError("unexpected API status code: $status – $(response["message"])")) -end - -function fork(owner::AbstractString, repo::AbstractString) - status, response = POST("repos/$owner/$repo/forks") - if status == 401 - delete_token() - status, response = POST("repos/$owner/$repo/forks") - end - status == 202 || throw(PkgError("forking $owner/$repo failed: $(response["message"])")) - return response -end - -function normalize_url(url::AbstractString) - m = match(LibGit2.GITHUB_REGEX,url) - m === nothing ? url : "https://github.com/$(m.captures[1]).git" -end - -function credentials() - username = try user() catch "" end - password = readtoken() - return Nullable(LibGit2.UserPasswordCredentials(username, password)) -end - -end # module From 18c90296f1c302637e5efa7881be3b35680935f7 Mon Sep 17 00:00:00 2001 From: Katie Hyatt Date: Wed, 7 Oct 2015 14:30:29 -0700 Subject: [PATCH 0489/1938] Fix bugs & add tests for triangular matrices --- base/linalg/triangular.jl | 4 ++-- test/linalg/triangular.jl | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/base/linalg/triangular.jl b/base/linalg/triangular.jl index 2996dbcbdcaf8..92e12f8039b73 100644 --- a/base/linalg/triangular.jl +++ b/base/linalg/triangular.jl @@ -362,7 +362,7 @@ scale!(c::Number, A::Union{UpperTriangular,LowerTriangular}) = scale!(A,c) A_mul_B!(A::Tridiagonal, B::AbstractTriangular) = A*full!(B) A_mul_B!(C::AbstractVecOrMat, A::AbstractTriangular, B::AbstractVecOrMat) = A_mul_B!(A, copy!(C, B)) -A_mul_Bc!(C::AbstractVecOrMat, A::AbstractTriangular, B::AbstractVecOrMat) = A_mul_Bc!(A, copy!(C, B)) +A_mul_Bc!(C::AbstractVecOrMat, A::AbstractTriangular, B::AbstractVecOrMat) = A_mul_B!(A, ctranspose!(C, B)) for (t, uploc, isunitc) in ((:LowerTriangular, 'L', 'N'), (:UnitLowerTriangular, 'L', 'U'), @@ -1022,7 +1022,7 @@ end ### Right division with triangle to the right hence lhs cannot be transposed. Quotients. for (f, g) in ((:/, :A_rdiv_B!), (:A_rdiv_Bc, :A_rdiv_Bc!), (:A_rdiv_Bt, :A_rdiv_Bt!)) @eval begin - function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::Union{UnitUpperTriangular{TB,S},UnitLowerTriangular{TB,S}}) + function ($f){TA,TB,S}(A::StridedVecOrMat{TA}, B::Union{UpperTriangular{TB,S},LowerTriangular{TB,S}}) TAB = typeof((zero(TA)*zero(TB) + zero(TA)*zero(TB))/one(TA)) ($g)(copy_oftype(A, TAB), convert(AbstractArray{TAB}, B)) end diff --git a/test/linalg/triangular.jl b/test/linalg/triangular.jl index f7b7af9a73a05..cea002bd2eda0 100644 --- a/test/linalg/triangular.jl +++ b/test/linalg/triangular.jl @@ -134,6 +134,11 @@ for elty1 in (Float32, Float64, Complex64, Complex128, BigFloat, Int) copy!(B, A1.') @test B == A1.' + #expm/logm + if (elty1 == Float64 || elty1 == Complex128) && (t1 == UpperTriangular || t1 == LowerTriangular) + @test expm(full(logm(A1))) ≈ full(A1) + end + # scale if (t1 == UpperTriangular || t1 == LowerTriangular) unitt = istriu(A1) ? UnitUpperTriangular : UnitLowerTriangular @@ -282,6 +287,10 @@ for elty1 in (Float32, Float64, Complex64, Complex128, BigFloat, Int) @test_approx_eq B[:,1]'A1' B[:,1]'full(A1)' @test_approx_eq B'A1' B'full(A1)' + if eltyB == elty1 + @test_approx_eq A_mul_B!(zeros(B),A1,B) A1*B + @test_approx_eq A_mul_Bc!(zeros(B),A1,B) A1*B' + end #error handling @test_throws DimensionMismatch Base.LinAlg.A_mul_B!(A1, ones(eltyB,n+1)) @test_throws DimensionMismatch Base.LinAlg.A_mul_B!(ones(eltyB,n+1,n+1), A1) @@ -305,6 +314,7 @@ for elty1 in (Float32, Float64, Complex64, Complex128, BigFloat, Int) # Error bounds elty1 != BigFloat && errorbounds(A1, A1\B, B) + end end end From ecb25bc6eed97094351da7a9d6b5325cc8085123 Mon Sep 17 00:00:00 2001 From: Felipe Date: Thu, 8 Oct 2015 01:16:09 -0300 Subject: [PATCH 0490/1938] minor review of README.windows.md --- README.windows.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.windows.md b/README.windows.md index a4acb7b594820..5a4576cd8fc29 100644 --- a/README.windows.md +++ b/README.windows.md @@ -237,7 +237,7 @@ a system compiler, and some downloaders. On Ubuntu (on other linux systems, the dependency names are likely to be similar): apt-add-repository ppa:ubuntu-wine/ppa - apt-get upate + apt-get update apt-get install wine1.7 subversion cvs gcc wget p7zip-full On Mac: Install XCode, XCode command line tools, X11 (now [XQuartz](http://xquartz.macosforge.org/)), @@ -252,7 +252,7 @@ So first we need to get a cross-compile version of gcc. Most binary packages appear to not include gfortran, so we will need to compile it from source. This is typically quite a bit of work, so we will use [this script](http://sourceforge.net/projects/mingw-w64-dgn/) o make it easy. -1. `svn checkout svn checkout svn://svn.code.sf.net/p/mingw-w64-dgn/code/trunk mingw-w64-dgn-code` +1. `svn checkout svn://svn.code.sf.net/p/mingw-w64-dgn/code/trunk mingw-w64-dgn` 2. `cd mingw-w64-dgn` 3. edit `rebuild_cross.sh` and make the following two changes: a. uncomment `export MAKE_OPT="-j 2"`, if appropriate for your machine @@ -260,7 +260,7 @@ This is typically quite a bit of work, so we will use [this script](http://sourc 5. `bash update_source.sh` 4. `bash rebuild_cross.sh` 5. `mv cross ~/cross-w64` -6. `export PATH=$HOME/cross-w64/bin:$PATH` # NOTE: it is important that you remember to always do this before using make in the following steps!, you can put this line in your .profile to make it easy +6. `export PATH=$HOME/cross-w64/bin:$PATH` # NOTE: it is important that you remember to always do this before using make in the following steps! You can put this line in your .profile to make it easy. Then we can essentially just repeat these steps for the 32-bit compiler, reusing some of the work: @@ -270,7 +270,7 @@ Then we can essentially just repeat these steps for the 32-bit compiler, reusing 10. `rm -r cross build` 11. `bash rebuild_cross.sh 32r` 12. `mv cross ~/cross-w32` -13. `export PATH=$HOME/cross-w32/bin:$PATH` # NOTE: it is important that you remember to always do this before using make in the following steps!, you can put this line in your .profile to make it easy +13. `export PATH=$HOME/cross-w32/bin:$PATH` # NOTE: it is important that you remember to always do this before using make in the following steps! You can put this line in your .profile to make it easy. Note: for systems that support rpm-based package managers, the necessary dependencies can be downloaded from the OpenSUSE build service (see the Vagrant script above). @@ -284,7 +284,7 @@ Then run the build: 1. `git clone https://github.com/JuliaLang/julia.git julia-win32` 2. `echo override XC_HOST = i686-w64-mingw32 >> Make.user` 3. `make` -4. `make win-extras` (Necessary before running `make binary-dist`p) +4. `make win-extras` (Necessary before running `make binary-dist`) 5. `make binary-dist` 6. move the julia-*.exe installer to the target machine From e917b71b06ad2bb353376a8316c9ad47363fbd26 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Thu, 8 Oct 2015 11:11:19 -0400 Subject: [PATCH 0491/1938] use color in precompilation output if parent process is using color --- base/loading.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/loading.jl b/base/loading.jl index b5b05e23b9ef5..0aeda4c3cac88 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -336,6 +336,7 @@ function create_expr_cache(input::AbstractString, output::AbstractString) io, pobj = open(detach(`$(julia_cmd()) --output-ji $output --output-incremental=yes --startup-file=no --history-file=no + --color=$(have_color ? "yes" : "no") --eval $code_object`), "w", STDOUT) try serialize(io, quote From c661ef6f9ff7ed7ec1dc6696ff3ffee14cc7f58c Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Mon, 5 Oct 2015 23:23:55 -0400 Subject: [PATCH 0492/1938] Make cov and cor similar to mean and var by removing keyword arguments. This fixes #13081 and the type instability that motivated that issue. --- NEWS.md | 2 + base/deprecated.jl | 12 ++- base/docs/helpdb.jl | 26 ------- base/statistics.jl | 173 ++++++++++++++++++++++---------------------- doc/stdlib/math.rst | 46 +++++++++--- test/docs.jl | 2 +- test/statistics.jl | 48 +++++++----- 7 files changed, 165 insertions(+), 144 deletions(-) diff --git a/NEWS.md b/NEWS.md index cf410b47bf7b6..02f04c1d1f525 100644 --- a/NEWS.md +++ b/NEWS.md @@ -26,6 +26,8 @@ Library improvements * The functions `remotecall`, `remotecall_fetch`, and `remotecall_wait` now have the function argument as the first argument to allow for do-block syntax ([#13338]). + * `cov` and `cor` don't use keyword arguments anymore and are therefore now type stable ([#13465]). + Deprecated or removed --------------------- diff --git a/base/deprecated.jl b/base/deprecated.jl index df4a5a144f831..19c38044cb80c 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -841,4 +841,14 @@ for f in (:remotecall, :remotecall_fetch, :remotecall_wait) @deprecate ($f)(w::Worker, f::Function, args...) ($f)(f, w::Worker, args...) @deprecate ($f)(id::Integer, f::Function, args...) ($f)(f, id::Integer, args...) end -end \ No newline at end of file +end + +@deprecate cov(x::AbstractVector; corrected=true, mean=nothing) covm(x, mean, corrected) +@deprecate cov(X::AbstractMatrix; vardim=1, corrected=true, mean=nothing) covm(X, mean, vardim, corrected) +@deprecate cov(x::AbstractVector, y::AbstractVector; corrected=true, mean=nothing) covm(x, mean[1], y, mean[2], corrected) +@deprecate cov(X::AbstractVecOrMat, Y::AbstractVecOrMat; vardim=1, corrected=true, mean=nothing) covm(X, mean[1], Y, mean[2], vardim, corrected) + +@deprecate cor(x::AbstractVector; mean=nothing) corm(x, mean) +@deprecate cor(X::AbstractMatrix; vardim=1, mean=nothing) corm(X, mean, vardim) +@deprecate cor(x::AbstractVector, y::AbstractVector; mean=nothing) corm(x, mean[1], y, mean[2]) +@deprecate cor(X::AbstractVecOrMat, Y::AbstractVecOrMat; vardim=1, mean=nothing) corm(X, mean[1], Y, mean[2], vardim) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index aecf883f77206..036d8d7d34fb0 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -1140,15 +1140,6 @@ Get the precision of a floating point number, as defined by the effective number """ precision -doc""" - cor(v1[, v2][, vardim=1, mean=nothing]) - -Compute the Pearson correlation between the vector(s) in `v1` and `v2`. - -Users can use the keyword argument `vardim` to specify the variable dimension, and `mean` to supply pre-computed mean values. -""" -cor - doc""" partitions(n) @@ -9003,23 +8994,6 @@ The process was stopped by a terminal interrupt (CTRL+C). """ InterruptException -doc""" - cov(v1[, v2][, vardim=1, corrected=true, mean=nothing]) - -Compute the Pearson covariance between the vector(s) in `v1` and `v2`. Here, `v1` and `v2` can be either vectors or matrices. - -This function accepts three keyword arguments: - -- `vardim`: the dimension of variables. When `vardim = 1`, variables are considered in columns while observations in rows; when `vardim = 2`, variables are in rows while observations in columns. By default, it is set to `1`. -- `corrected`: whether to apply Bessel's correction (divide by `n-1` instead of `n`). By default, it is set to `true`. -- `mean`: allow users to supply mean values that are known. By default, it is set to `nothing`, which indicates that the mean(s) are unknown, and the function will compute the mean. Users can use `mean=0` to indicate that the input data are centered, and hence there's no need to subtract the mean. - -The size of the result depends on the size of `v1` and `v2`. When both `v1` and `v2` are vectors, it returns the covariance between them as a scalar. When either one is a matrix, it returns a covariance matrix of size `(n1, n2)`, where `n1` and `n2` are the numbers of slices in `v1` and `v2`, which depend on the setting of `vardim`. - -Note: `v2` can be omitted, which indicates `v2 = v1`. -""" -cov - doc""" den(x) diff --git a/base/statistics.jl b/base/statistics.jl index 3f031075fb7c7..0f18007746416 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -261,66 +261,67 @@ unscaled_covzm(x::AbstractMatrix, y::AbstractMatrix, vardim::Int) = # covzm (with centered data) -covzm(x::AbstractVector; corrected::Bool=true) = unscaled_covzm(x) / (length(x) - Int(corrected)) - -covzm(x::AbstractMatrix; vardim::Int=1, corrected::Bool=true) = +covzm(x::AbstractVector, corrected::Bool=true) = unscaled_covzm(x) / (length(x) - Int(corrected)) +covzm(x::AbstractMatrix, vardim::Int=1, corrected::Bool=true) = scale!(unscaled_covzm(x, vardim), inv(size(x,vardim) - Int(corrected))) - -covzm(x::AbstractVector, y::AbstractVector; corrected::Bool=true) = +covzm(x::AbstractVector, y::AbstractVector, corrected::Bool=true) = unscaled_covzm(x, y) / (length(x) - Int(corrected)) - -covzm(x::AbstractVecOrMat, y::AbstractVecOrMat; vardim::Int=1, corrected::Bool=true) = +covzm(x::AbstractVecOrMat, y::AbstractVecOrMat, vardim::Int=1, corrected::Bool=true) = scale!(unscaled_covzm(x, y, vardim), inv(_getnobs(x, y, vardim) - Int(corrected))) # covm (with provided mean) -covm(x::AbstractVector, xmean; corrected::Bool=true) = - covzm(x .- xmean; corrected=corrected) +covm(x::AbstractVector, xmean, corrected::Bool=true) = + covzm(x .- xmean, corrected) +covm(x::AbstractMatrix, xmean, vardim::Int=1, corrected::Bool=true) = + covzm(x .- xmean, vardim, corrected) +covm(x::AbstractVector, xmean, y::AbstractVector, ymean, corrected::Bool=true) = + covzm(x .- xmean, y .- ymean, corrected) +covm(x::AbstractVecOrMat, xmean, y::AbstractVecOrMat, ymean, vardim::Int=1, corrected::Bool=true) = + covzm(x .- xmean, y .- ymean, vardim, corrected) -covm(x::AbstractMatrix, xmean; vardim::Int=1, corrected::Bool=true) = - covzm(x .- xmean; vardim=vardim, corrected=corrected) +# cov (API) +doc""" + cov(x[, corrected=true]) -covm(x::AbstractVector, xmean, y::AbstractVector, ymean; corrected::Bool=true) = - covzm(x .- xmean, y .- ymean; corrected=corrected) +Compute the variance of the vector `x`. If `corrected` is `true` (the default) then the sum is scaled with `n-1` wheares the sum is scaled with `n` if `corrected` is `false` where `n = length(x)`. +""" +cov(x::AbstractVector, corrected::Bool) = covm(x, Base.mean(x), corrected) +# This ugly hack is necessary to make the method below considered more specific than the deprecated method. When the old keyword version has been completely deprecated, these two methods can be merged +cov{T<:AbstractVector}(x::T) = covm(x, Base.mean(x), true) -covm(x::AbstractVecOrMat, xmean, y::AbstractVecOrMat, ymean; vardim::Int=1, corrected::Bool=true) = - covzm(x .- xmean, y .- ymean; vardim=vardim, corrected=corrected) +doc""" + cov(X[, vardim=1, corrected=true]) -# cov (API) +Compute the covariance matrix of the matrix `X` along the dimension `vardim`. If `corrected` is `true` (the default) then the sum is scaled with `n-1` wheares the sum is scaled with `n` if `corrected` is `false` where `n = size(X, vardim)`. +""" +cov(X::AbstractMatrix, vardim::Int, corrected::Bool=true) = + covm(X, _vmean(X, vardim), vardim, corrected) +# This ugly hack is necessary to make the method below considered more specific than the deprecated method. When the old keyword version has been completely deprecated, these two methods can be merged +cov{T<:AbstractMatrix}(X::T) = cov(X, 1, true) -function cov(x::AbstractVector; corrected::Bool=true, mean=nothing) - mean == 0 ? covzm(x; corrected=corrected) : - mean === nothing ? covm(x, Base.mean(x); corrected=corrected) : - isa(mean, Number) ? covm(x, mean; corrected=corrected) : - throw(ArgumentError("invalid value of mean, $(mean)::$(typeof(mean))")) -end +doc""" + cov(x, y[, corrected=true]) -function cov(x::AbstractMatrix; vardim::Int=1, corrected::Bool=true, mean=nothing) - mean == 0 ? covzm(x; vardim=vardim, corrected=corrected) : - mean === nothing ? covm(x, _vmean(x, vardim); vardim=vardim, corrected=corrected) : - isa(mean, AbstractArray) ? covm(x, mean; vardim=vardim, corrected=corrected) : - throw(ArgumentError("invalid value of mean, $(mean)::$(typeof(mean))")) -end +Compute the covariance between the vectors `x` and `y`. If `corrected` is `true` (the default) then the sum is scaled with `n-1` wheares the sum is scaled with `n` if `corrected` is `false` where `n = length(x) = length(y)`. +""" +cov(x::AbstractVector, y::AbstractVector, corrected::Bool) = + covm(x, Base.mean(x), y, Base.mean(y), corrected) +# This ugly hack is necessary to make the method below considered more specific than the deprecated method. When the old keyword version has been completely deprecated, these two methods can be merged +cov{T<:AbstractVector,S<:AbstractVector}(x::T, y::S) = + covm(x, Base.mean(x), y, Base.mean(y), true) -function cov(x::AbstractVector, y::AbstractVector; corrected::Bool=true, mean=nothing) - mean == 0 ? covzm(x, y; corrected=corrected) : - mean === nothing ? covm(x, Base.mean(x), y, Base.mean(y); corrected=corrected) : - isa(mean, (Number,Number)) ? covm(x, mean[1], y, mean[2]; corrected=corrected) : - throw(ArgumentError("invalid value of mean, $(mean)::$(typeof(mean))")) -end +doc""" + cov(X, Y[, vardim=1, corrected=true]) -function cov(x::AbstractVecOrMat, y::AbstractVecOrMat; vardim::Int=1, corrected::Bool=true, mean=nothing) - if mean == 0 - covzm(x, y; vardim=vardim, corrected=corrected) - elseif mean === nothing - covm(x, _vmean(x, vardim), y, _vmean(y, vardim); vardim=vardim, corrected=corrected) - elseif isa(mean, (Any,Any)) - covm(x, mean[1], y, mean[2]; vardim=vardim, corrected=corrected) - else - throw(ArgumentError("invalid value of mean, $(mean)::$(typeof(mean))")) - end -end +Compute the covariance between the vectors or matrices `X` and `Y` along the dimension `vardim`. If `corrected` is `true` (the default) then the sum is scaled with `n-1` wheares the sum is scaled with `n` if `corrected` is `false` where `n = size(X, vardim) = size(Y, vardim)`. +""" +cov(X::AbstractVecOrMat, Y::AbstractVecOrMat, vardim::Int, corrected::Bool=true) = + covm(X, _vmean(X, vardim), Y, _vmean(Y, vardim), vardim, corrected) +# This ugly hack is necessary to make the method below considered more specific than the deprecated method. When the old keyword version has been completely deprecated, these two methods can be merged +cov{T<:AbstractVecOrMat,S<:AbstractVecOrMat}(X::T, Y::S) = + covm(X, _vmean(X, vardim), Y, _vmean(Y, vardim), 1, true) ##### correlation ##### @@ -340,7 +341,6 @@ function cov2cor!{T}(C::AbstractMatrix{T}, xsd::AbstractArray) end return C end - function cov2cor!(C::AbstractMatrix, xsd::Number, ysd::AbstractArray) nx, ny = size(C) length(ysd) == ny || throw(DimensionMismatch("inconsistent dimensions")) @@ -351,7 +351,6 @@ function cov2cor!(C::AbstractMatrix, xsd::Number, ysd::AbstractArray) end return C end - function cov2cor!(C::AbstractMatrix, xsd::AbstractArray, ysd::Number) nx, ny = size(C) length(xsd) == nx || throw(DimensionMismatch("inconsistent dimensions")) @@ -362,7 +361,6 @@ function cov2cor!(C::AbstractMatrix, xsd::AbstractArray, ysd::Number) end return C end - function cov2cor!(C::AbstractMatrix, xsd::AbstractArray, ysd::AbstractArray) nx, ny = size(C) (length(xsd) == nx && length(ysd) == ny) || @@ -378,10 +376,10 @@ end # corzm (non-exported, with centered data) corzm{T}(x::AbstractVector{T}) = one(real(T)) - -corzm(x::AbstractMatrix; vardim::Int=1) = - (c = unscaled_covzm(x, vardim); cov2cor!(c, sqrt!(diag(c)))) - +function corzm(x::AbstractMatrix, vardim::Int=1) + c = unscaled_covzm(x, vardim) + return cov2cor!(c, sqrt!(diag(c))) +end function corzm(x::AbstractVector, y::AbstractVector) n = length(x) length(y) == n || throw(DimensionMismatch("inconsistent lengths")) @@ -401,57 +399,58 @@ function corzm(x::AbstractVector, y::AbstractVector) end return xy / (sqrt(xx) * sqrt(yy)) end - -corzm(x::AbstractVector, y::AbstractMatrix; vardim::Int=1) = +corzm(x::AbstractVector, y::AbstractMatrix, vardim::Int=1) = cov2cor!(unscaled_covzm(x, y, vardim), sqrt(sumabs2(x)), sqrt!(sumabs2(y, vardim))) - -corzm(x::AbstractMatrix, y::AbstractVector; vardim::Int=1) = +corzm(x::AbstractMatrix, y::AbstractVector, vardim::Int=1) = cov2cor!(unscaled_covzm(x, y, vardim), sqrt!(sumabs2(x, vardim)), sqrt(sumabs2(y))) - -corzm(x::AbstractMatrix, y::AbstractMatrix; vardim::Int=1) = +corzm(x::AbstractMatrix, y::AbstractMatrix, vardim::Int=1) = cov2cor!(unscaled_covzm(x, y, vardim), sqrt!(sumabs2(x, vardim)), sqrt!(sumabs2(y, vardim))) # corm corm{T}(x::AbstractVector{T}, xmean) = one(real(T)) +corm(x::AbstractMatrix, xmean, vardim::Int=1) = corzm(x .- xmean, vardim) +corm(x::AbstractVector, xmean, y::AbstractVector, ymean) = corzm(x .- xmean, y .- ymean) +corm(x::AbstractVecOrMat, xmean, y::AbstractVecOrMat, ymean, vardim::Int=1) = + corzm(x .- xmean, y .- ymean, vardim) -corm(x::AbstractMatrix, xmean; vardim::Int=1) = corzm(x .- xmean; vardim=vardim) +# cor +doc""" + cor(x) -corm(x::AbstractVector, xmean, y::AbstractVector, ymean) = corzm(x .- xmean, y .- ymean) +Return the number one. +""" +cor{T<:AbstractVector}(x::T) = one(real(eltype(x))) +# This ugly hack is necessary to make the method below considered more specific than the deprecated method. When the old keyword version has been completely deprecated, these two methods can be merged -corm(x::AbstractVecOrMat, xmean, y::AbstractVecOrMat, ymean; vardim::Int=1) = - corzm(x .- xmean, y .- ymean; vardim=vardim) +doc""" + cor(X[, vardim=1]) -# cor +Compute the Pearson correlation matrix of the matrix `X` along the dimension `vardim`. +""" +cor(X::AbstractMatrix, vardim::Int) = corm(X, _vmean(X, vardim), vardim) +# This ugly hack is necessary to make the method below considered more specific than the deprecated method. When the old keyword version has been completely deprecated, these two methods can be merged +cor{T<:AbstractMatrix}(X::T) = corm(X, _vmean(X, vardim), 1) -cor{T}(x::AbstractVector{T}; mean=nothing) = one(real(T)) +doc""" + cor(x, y) -function cor(x::AbstractMatrix; vardim::Int=1, mean=nothing) - mean == 0 ? corzm(x; vardim=vardim) : - mean === nothing ? corm(x, _vmean(x, vardim); vardim=vardim) : - isa(mean, AbstractArray) ? corm(x, mean; vardim=vardim) : - throw(ArgumentError("invalid value of mean, $(mean)::$(typeof(mean))")) -end +Compute the Pearson correlation between the vectors `x` and `y`. +""" +cor{T<:AbstractVector,S<:AbstractVector}(x::T, y::S) = corm(x, Base.mean(x), y, Base.mean(y)) +# This ugly hack is necessary to make the method below considered more specific than the deprecated method. When the old keyword version has been completely deprecated, these two methods can be merged -function cor(x::AbstractVector, y::AbstractVector; mean=nothing) - mean == 0 ? corzm(x, y) : - mean === nothing ? corm(x, Base.mean(x), y, Base.mean(y)) : - isa(mean, (Number,Number)) ? corm(x, mean[1], y, mean[2]) : - throw(ArgumentError("invalid value of mean, $(mean)::$(typeof(mean))")) -end +doc""" + cor(X, Y[, vardim=1]) -function cor(x::AbstractVecOrMat, y::AbstractVecOrMat; vardim::Int=1, mean=nothing) - if mean == 0 - corzm(x, y; vardim=vardim) - elseif mean === nothing - corm(x, _vmean(x, vardim), y, _vmean(y, vardim); vardim=vardim) - elseif isa(mean, (Any,Any)) - corm(x, mean[1], y, mean[2]; vardim=vardim) - else - throw(ArgumentError("invalid value of mean, $(mean)::$(typeof(mean))")) - end -end +Compute the Pearson correlation between the vectors or matrices `X` and `Y` along the dimension `vardim`. +""" +cor(x::AbstractVecOrMat, y::AbstractVecOrMat, vardim::Int) = + corm(x, _vmean(x, vardim), y, _vmean(y, vardim), vardim) +# This ugly hack is necessary to make the method below considered more specific than the deprecated method. When the old keyword version has been completely deprecated, these two methods can be merged +cor(x::AbstractVecOrMat, y::AbstractVecOrMat) = + corm(x, _vmean(x, vardim), y, _vmean(y, vardim), 1) ##### median & quantiles ##### diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index c3fa039b1d8c3..3b141f037fcc4 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -1691,29 +1691,53 @@ Statistics Like ``quantile``\ , but overwrites the input vector. -.. function:: cov(v1[, v2][, vardim=1, corrected=true, mean=nothing]) +.. function:: cov(x[, corrected=true]) .. Docstring generated from Julia source - Compute the Pearson covariance between the vector(s) in ``v1`` and ``v2``\ . Here, ``v1`` and ``v2`` can be either vectors or matrices. + Compute the variance of the vector ``x``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1`` wheares the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x)``\ . - This function accepts three keyword arguments: +.. function:: cov(X[, vardim=1, corrected=true]) - * ``vardim``\ : the dimension of variables. When ``vardim = 1``\ , variables are considered in columns while observations in rows; when ``vardim = 2``\ , variables are in rows while observations in columns. By default, it is set to ``1``\ . - * ``corrected``\ : whether to apply Bessel's correction (divide by ``n-1`` instead of ``n``\ ). By default, it is set to ``true``\ . - * ``mean``\ : allow users to supply mean values that are known. By default, it is set to ``nothing``\ , which indicates that the mean(s) are unknown, and the function will compute the mean. Users can use ``mean=0`` to indicate that the input data are centered, and hence there's no need to subtract the mean. + .. Docstring generated from Julia source + + Compute the covariance matrix of the matrix ``X`` along the dimension ``vardim``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1`` wheares the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = size(X, vardim)``\ . + +.. function:: cov(x, y[, corrected=true]) + + .. Docstring generated from Julia source - The size of the result depends on the size of ``v1`` and ``v2``\ . When both ``v1`` and ``v2`` are vectors, it returns the covariance between them as a scalar. When either one is a matrix, it returns a covariance matrix of size ``(n1, n2)``\ , where ``n1`` and ``n2`` are the numbers of slices in ``v1`` and ``v2``\ , which depend on the setting of ``vardim``\ . + Compute the covariance between the vectors ``x`` and ``y``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1`` wheares the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x) = length(y)``\ . - Note: ``v2`` can be omitted, which indicates ``v2 = v1``\ . +.. function:: cov(X, Y[, vardim=1, corrected=true]) -.. function:: cor(v1[, v2][, vardim=1, mean=nothing]) + .. Docstring generated from Julia source + + Compute the covariance between the vectors or matrices ``X`` and ``Y`` along the dimension ``vardim``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1`` wheares the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = size(X, vardim) = size(Y, vardim)``\ . + +.. function:: cor(x) .. Docstring generated from Julia source - Compute the Pearson correlation between the vector(s) in ``v1`` and ``v2``\ . + Return the number one. + +.. function:: cor(X[, vardim=1]) + + .. Docstring generated from Julia source + + Compute the Pearson correlation matrix of the matrix ``X`` along the dimension ``vardim``\ . + +.. function:: cor(x, y) + + .. Docstring generated from Julia source + + Compute the Pearson correlation between the vectors ``x`` and ``y``\ . + +.. function:: cor(X, Y[, vardim=1]) + + .. Docstring generated from Julia source - Users can use the keyword argument ``vardim`` to specify the variable dimension, and ``mean`` to supply pre-computed mean values. + Compute the Pearson correlation between the vectors or matrices ``X`` and ``Y`` along the dimension ``vardim``\ . Signal Processing ----------------- diff --git a/test/docs.jl b/test/docs.jl index 5abb6b059b5c3..306cbfbb9411a 100644 --- a/test/docs.jl +++ b/test/docs.jl @@ -415,7 +415,7 @@ f12593_2() = 1 @test (Docs.@repl @r_str) !== nothing # Simple tests for apropos: -@test contains(sprint(apropos, "pearson"), "cov") +@test contains(sprint(apropos, "pearson"), "cor") @test contains(sprint(apropos, r"ind(exes|ices)"), "eachindex") @test contains(sprint(apropos, "print"), "Profile.print") diff --git a/test/statistics.jl b/test/statistics.jl index 122dfd15d0061..56638933a72db 100644 --- a/test/statistics.jl +++ b/test/statistics.jl @@ -178,35 +178,41 @@ for vd in [1, 2], zm in [true, false], cr in [true, false] y1 = vec(Y[1,:]) end - c = zm ? cov(x1; mean=0, corrected=cr) : - cov(x1; corrected=cr) + c = zm ? Base.covm(x1, 0, cr) : + cov(x1, cr) @test isa(c, Float64) @test_approx_eq c Cxx[1,1] + @inferred cov(x1, cr) - C = zm ? cov(X; vardim=vd, mean=0, corrected=cr) : - cov(X; vardim=vd, corrected=cr) + C = zm ? Base.covm(X, 0, vd, cr) : + cov(X, vd, cr) @test size(C) == (k, k) @test_approx_eq C Cxx + @inferred cov(X, vd, cr) - c = zm ? cov(x1, y1; mean=0, corrected=cr) : - cov(x1, y1; corrected=cr) + c = zm ? Base.covm(x1, 0, y1, 0, cr) : + cov(x1, y1, cr) @test isa(c, Float64) @test_approx_eq c Cxy[1,1] + @inferred cov(x1, y1, cr) - C = zm ? cov(x1, Y; vardim=vd, mean=0, corrected=cr) : - cov(x1, Y; vardim=vd, corrected=cr) + C = zm ? Base.covm(x1, 0, Y, 0, vd, cr) : + cov(x1, Y, vd, cr) @test size(C) == (1, k) @test_approx_eq C Cxy[1,:] + @inferred cov(x1, Y, vd, cr) - C = zm ? cov(X, y1; vardim=vd, mean=0, corrected=cr) : - cov(X, y1; vardim=vd, corrected=cr) + C = zm ? Base.covm(X, 0, y1, 0, vd, cr) : + cov(X, y1, vd, cr) @test size(C) == (k, 1) @test_approx_eq C Cxy[:,1] + @inferred cov(X, y1, vd, cr) - C = zm ? cov(X, Y; vardim=vd, mean=0, corrected=cr) : - cov(X, Y; vardim=vd, corrected=cr) + C = zm ? Base.covm(X, 0, Y, 0, vd, cr) : + cov(X, Y, vd, cr) @test size(C) == (k, k) @test_approx_eq C Cxy + @inferred cov(X, Y, vd, cr) end # test correlation @@ -245,29 +251,35 @@ for vd in [1, 2], zm in [true, false] y1 = vec(Y[1,:]) end - c = zm ? cor(x1; mean=0) : cor(x1) + c = zm ? Base.corm(x1, 0) : cor(x1) @test isa(c, Float64) @test_approx_eq c Cxx[1,1] + @inferred cor(x1) - C = zm ? cor(X; vardim=vd, mean=0) : cor(X; vardim=vd) + C = zm ? Base.corm(X, 0, vd) : cor(X, vd) @test size(C) == (k, k) @test_approx_eq C Cxx + @inferred cor(X, vd) - c = zm ? cor(x1, y1; mean=0) : cor(x1, y1) + c = zm ? Base.corm(x1, 0, y1, 0) : cor(x1, y1) @test isa(c, Float64) @test_approx_eq c Cxy[1,1] + @inferred cor(x1, y1) - C = zm ? cor(x1, Y; vardim=vd, mean=0) : cor(x1, Y; vardim=vd) + C = zm ? Base.corm(x1, 0, Y, 0, vd) : cor(x1, Y, vd) @test size(C) == (1, k) @test_approx_eq C Cxy[1,:] + @inferred cor(x1, Y, vd) - C = zm ? cor(X, y1; vardim=vd, mean=0) : cor(X, y1; vardim=vd) + C = zm ? Base.corm(X, 0, y1, 0, vd) : cor(X, y1, vd) @test size(C) == (k, 1) @test_approx_eq C Cxy[:,1] + @inferred cor(X, y1, vd) - C = zm ? cor(X, Y; vardim=vd, mean=0) : cor(X, Y; vardim=vd) + C = zm ? Base.corm(X, 0, Y, 0, vd) : cor(X, Y, vd) @test size(C) == (k, k) @test_approx_eq C Cxy + @inferred cor(X, Y, vd) end From 2acdb837de026fd205f6ef6cc5f52e55b9cdeae0 Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Thu, 8 Oct 2015 13:48:35 -0400 Subject: [PATCH 0493/1938] fixes at-testloop documentation in stdlib docs --- doc/stdlib/test.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/stdlib/test.rst b/doc/stdlib/test.rst index bb5e4a471a38b..debd745ce7c2d 100644 --- a/doc/stdlib/test.rst +++ b/doc/stdlib/test.rst @@ -125,13 +125,17 @@ test set will then throw a ``TestSetException``. By default the ``@testset`` macro will return the testset object itself, though this behavior can be customized in other testset types. -.. function:: @testloop [CustomTestSet] [option=val ...] ["description \$v"] for v in (...) ... end - @testloop [CustomTestSet] [option=val ...] ["description \$v, \$w"] for v in (...), w in (...) ... end +.. function:: @testloop [CustomTestSet] [option=val ...] ["description $v"] for v in (...) ... end + @testloop [CustomTestSet] [option=val ...] ["description $v, $w"] for v in (...), w in (...) ... end .. Docstring generated from Julia source Starts a new test set for each iteration of the loop. The description string accepts interpolation from the loop indices. If no description is provided, one is constructed based on the variables. + Any custom testset type (subtype of ``AbstractTestSet``\ ) can be given and it will also be used for any nested ``@testset`` or ``@testloop`` invocations. The given options are only applied to the test sets where they are given. The default test set type does not take any options. + + The ``@testloop`` macro collects and returns a list of the return values of the ``finish`` method, which by default will return a list of the testset objects used in each iteration. + We can put our tests for the ``foo(x)`` function in a test set:: julia> @testset "Foo Tests" begin From 7a107715c7440e6bc64d0ee9f33fff97bc383474 Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Tue, 6 Oct 2015 23:35:59 -0400 Subject: [PATCH 0494/1938] Add generic pivoted QR and a bit of restructuring of the qrfacts --- NEWS.md | 2 ++ base/linalg/dense.jl | 2 +- base/linalg/qr.jl | 72 +++++++++++++++++++++++++++++++++++++------ test/linalg/lapack.jl | 3 +- test/linalg/qr.jl | 34 ++++++++------------ 5 files changed, 80 insertions(+), 33 deletions(-) diff --git a/NEWS.md b/NEWS.md index 02f04c1d1f525..7db7662e707e6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -28,6 +28,8 @@ Library improvements * `cov` and `cor` don't use keyword arguments anymore and are therefore now type stable ([#13465]). + * New method for generic QR with column pivoting ([#13480]). + Deprecated or removed --------------------- diff --git a/base/linalg/dense.jl b/base/linalg/dense.jl index 52471aeaa6087..2b8bf10afe9dd 100644 --- a/base/linalg/dense.jl +++ b/base/linalg/dense.jl @@ -433,7 +433,7 @@ function factorize{T}(A::Matrix{T}) end return lufact(A) end - qrfact(A,typeof(zero(T)/sqrt(zero(T) + zero(T)))<:BlasFloat?Val{true}:Val{false}) # Generic pivoted QR not implemented yet + qrfact(A, Val{true}) end (\)(a::Vector, B::StridedVecOrMat) = (\)(reshape(a, length(a), 1), B) diff --git a/base/linalg/qr.jl b/base/linalg/qr.jl index 5d2d70a0c5418..1f39077391839 100644 --- a/base/linalg/qr.jl +++ b/base/linalg/qr.jl @@ -24,8 +24,7 @@ immutable QRPivoted{T,S<:AbstractMatrix} <: Factorization{T} end QRPivoted{T}(factors::AbstractMatrix{T}, τ::Vector{T}, jpvt::Vector{BlasInt}) = QRPivoted{T,typeof(factors)}(factors, τ, jpvt) -function qrfact!{T}(A::AbstractMatrix{T}, pivot::Union{Type{Val{false}}, Type{Val{true}}}=Val{false}) - pivot==Val{true} && warn("pivoting only implemented for Float32, Float64, Complex64 and Complex128") +function qrfactUnblocked!{T}(A::AbstractMatrix{T}) m, n = size(A) τ = zeros(T, min(m,n)) for k = 1:min(m - 1 + !(T<:Real), n) @@ -36,12 +35,66 @@ function qrfact!{T}(A::AbstractMatrix{T}, pivot::Union{Type{Val{false}}, Type{Va end QR(A, τ) end -qrfact!{T<:BlasFloat}(A::StridedMatrix{T}, pivot::Type{Val{false}} = Val{false}) = QRCompactWY(LAPACK.geqrt!(A, min(minimum(size(A)), 36))...) -qrfact!{T<:BlasFloat}(A::StridedMatrix{T}, pivot::Type{Val{true}}) = QRPivoted(LAPACK.geqp3!(A)...) -qrfact{T<:BlasFloat}(A::StridedMatrix{T}, pivot::Union{Type{Val{false}}, Type{Val{true}}}=Val{false}) = qrfact!(copy(A), pivot) -copy_oftype{T}(A::StridedMatrix{T}, ::Type{T}) = copy(A) -copy_oftype{T,S}(A::StridedMatrix{T}, ::Type{S}) = convert(AbstractMatrix{S}, A) -qrfact{T}(A::StridedMatrix{T}, pivot::Union{Type{Val{false}}, Type{Val{true}}}=Val{false}) = qrfact!(copy_oftype(A, typeof(one(T)/norm(one(T)))), pivot) + +# Find index for columns with largest two norm +function indmaxcolumn(A::StridedMatrix) + mm = norm(slice(A, :, 1)) + ii = 1 + for i = 2:size(A, 2) + mi = norm(slice(A, :, i)) + if abs(mi) > mm + mm = mi + ii = i + end + end + return ii +end + +function qrfactPivotedUnblocked!(A::StridedMatrix) + m, n = size(A) + piv = collect(UnitRange{BlasInt}(1,n)) + τ = Array(eltype(A), min(m,n)) + for j = 1:min(m,n) + + # Find column with maximum norm in trailing submatrix + jm = indmaxcolumn(slice(A, j:m, j:n)) + j - 1 + + if jm != j + # Flip elements in pivoting vector + tmpp = piv[jm] + piv[jm] = piv[j] + piv[j] = tmpp + + # Update matrix with + for i = 1:m + tmp = A[i,jm] + A[i,jm] = A[i,j] + A[i,j] = tmp + end + end + + # Compute reflector of columns j + x = slice(A, j:m, j) + τj = LinAlg.reflector!(x) + τ[j] = τj + + # Update trailing submatrix with reflector + LinAlg.reflectorApply!(x, τj, sub(A, j:m, j+1:n)) + end + return LinAlg.QRPivoted{eltype(A), typeof(A)}(A, τ, piv) +end + +# LAPACK version +qrfact!{T<:BlasFloat}(A::StridedMatrix{T}, ::Type{Val{false}}) = QRCompactWY(LAPACK.geqrt!(A, min(minimum(size(A)), 36))...) +qrfact!{T<:BlasFloat}(A::StridedMatrix{T}, ::Type{Val{true}}) = QRPivoted(LAPACK.geqp3!(A)...) +qrfact!{T<:BlasFloat}(A::StridedMatrix{T}) = qrfact!(A, Val{false}) +qrfact{T<:BlasFloat}(A::StridedMatrix{T}, args...) = qrfact!(copy(A), args...) + +# Generic fallbacks +qrfact!(A::StridedMatrix, ::Type{Val{false}}) = qrfactUnblocked!(A) +qrfact!(A::StridedMatrix, ::Type{Val{true}}) = qrfactPivotedUnblocked!(A) +qrfact!(A::StridedMatrix) = qrfact!(A, Val{false}) +qrfact{T}(A::StridedMatrix{T}, args...) = qrfact!(copy_oftype(A, typeof(zero(T)/norm(one(T)))), args...) qrfact(x::Number) = qrfact(fill(x,1,1)) qr(A::Union{Number, AbstractMatrix}, pivot::Union{Type{Val{false}}, Type{Val{true}}}=Val{false}; thin::Bool=true) = @@ -336,7 +389,7 @@ function A_ldiv_B!{T}(A::QR{T}, B::StridedMatrix{T}) m, n = size(A) minmn = min(m,n) mB, nB = size(B) - Ac_mul_B!(A[:Q], sub(B, 1:m, 1:nB)) + Ac_mul_B!(A[:Q], sub(B, 1:m, :)) R = A[:R] @inbounds begin if n > m # minimum norm solution @@ -389,6 +442,7 @@ function A_ldiv_B!(A::QRPivoted, B::StridedMatrix) B[1:size(A.factors, 2),:] = sub(B, 1:size(A.factors, 2), :)[invperm(A.jpvt),:] B end + function \{TA,Tb}(A::Union{QR{TA},QRCompactWY{TA},QRPivoted{TA}}, b::StridedVector{Tb}) S = promote_type(TA,Tb) m,n = size(A) diff --git a/test/linalg/lapack.jl b/test/linalg/lapack.jl index 6dfdb5b4be240..6a315f3b695b2 100644 --- a/test/linalg/lapack.jl +++ b/test/linalg/lapack.jl @@ -523,8 +523,7 @@ for elty in (Float32, Float64, Complex{Float32}, Complex{Float64}) end ## QR - FJulia = invoke(qrfact!, Tuple{AbstractMatrix, Type{Val{false}}}, - copy(A), Val{false}) + FJulia = LinAlg.qrfactUnblocked!(copy(A)) FLAPACK = Base.LinAlg.LAPACK.geqrf!(copy(A)) @test_approx_eq FJulia.factors FLAPACK[1] @test_approx_eq FJulia.τ FLAPACK[2] diff --git a/test/linalg/qr.jl b/test/linalg/qr.jl index cc77f34f4b072..aaefd9b5f17c5 100644 --- a/test/linalg/qr.jl +++ b/test/linalg/qr.jl @@ -73,21 +73,20 @@ debug && println("Thin QR decomposition (without pivoting)") @test eye(eltyb,n)*q ≈ convert(AbstractMatrix{tab},q) end -debug && println("(Automatic) Fat (pivoted) QR decomposition") # Pivoting is only implemented for BlasFloats - if eltya <: BlasFloat - @inferred qrfact(a, Val{true}) - @inferred qr(a, Val{true}) - end +debug && println("(Automatic) Fat (pivoted) QR decomposition") + @inferred qrfact(a, Val{true}) + @inferred qr(a, Val{true}) + qrpa = factorize(a[1:n1,:]) q,r = qrpa[:Q], qrpa[:R] @test_throws KeyError qrpa[:Z] - if isa(qrpa,QRPivoted) p = qrpa[:p] end # Reconsider if pivoted QR gets implemented in julia + p = qrpa[:p] @test_approx_eq q'*full(q, thin=false) eye(n1) @test_approx_eq q*full(q, thin=false)' eye(n1) @test_approx_eq (UpperTriangular(eye(eltya,size(q,2)))*q')*full(q, thin=false) eye(n1) @test_approx_eq q*r isa(qrpa,QRPivoted) ? a[1:n1,p] : a[1:n1,:] - @test_approx_eq isa(qrpa, QRPivoted) ? q*r[:,invperm(p)] : q*r a[1:n1,:] - @test_approx_eq isa(qrpa, QRPivoted) ? q*r*qrpa[:P].' : q*r a[1:n1,:] + @test_approx_eq q*r[:,invperm(p)] a[1:n1,:] + @test_approx_eq q*r*qrpa[:P].' a[1:n1,:] @test_approx_eq_eps a[1:n1,:]*(qrpa\b[1:n1]) b[1:n1] 5000ε @test_approx_eq full(qrpa) a[1:5,:] @test_throws DimensionMismatch q*b[1:n1 + 1] @@ -96,15 +95,15 @@ debug && println("(Automatic) Fat (pivoted) QR decomposition") # Pivoting is onl @test eye(eltyb,n1)*q ≈ convert(AbstractMatrix{tab},q) end -debug && println("(Automatic) Thin (pivoted) QR decomposition") # Pivoting is only implemented for BlasFloats +debug && println("(Automatic) Thin (pivoted) QR decomposition") qrpa = factorize(a[:,1:n1]) q,r = qrpa[:Q], qrpa[:R] @test_throws KeyError qrpa[:Z] - if isa(qrpa, QRPivoted) p = qrpa[:p] end # Reconsider if pivoted QR gets implemented in julia + p = qrpa[:p] @test_approx_eq q'*full(q, thin=false) eye(n) @test_approx_eq q*full(q, thin=false)' eye(n) - @test_approx_eq q*r isa(qrpa, QRPivoted) ? a[:,p] : a[:,1:n1] - @test_approx_eq isa(qrpa, QRPivoted) ? q*r[:,invperm(p)] : q*r a[:,1:n1] + @test_approx_eq q*r a[:,p] + @test_approx_eq q*r[:,invperm(p)] a[:,1:n1] @test_approx_eq full(qrpa) a[:,1:5] @test_throws DimensionMismatch q*b[1:n1 + 1] @test_throws DimensionMismatch b[1:n1 + 1]*q' @@ -124,10 +123,8 @@ debug && println("Matmul with QR factorizations") @test_approx_eq A_mul_Bc!(full(q,thin=false),q) eye(n) @test_throws DimensionMismatch A_mul_Bc!(eye(eltya,n+1),q) @test_throws BoundsError size(q,-1) - if eltya == BigFloat - @test_throws DimensionMismatch Base.LinAlg.A_mul_B!(q,zeros(eltya,n1+1)) - @test_throws DimensionMismatch Base.LinAlg.Ac_mul_B!(q,zeros(eltya,n1+1)) - end + @test_throws DimensionMismatch Base.LinAlg.A_mul_B!(q,zeros(eltya,n1+1)) + @test_throws DimensionMismatch Base.LinAlg.Ac_mul_B!(q,zeros(eltya,n1+1)) qra = qrfact(a[:,1:n1], Val{false}) q,r = qra[:Q], qra[:R] @@ -165,8 +162,3 @@ let Q=full(qrfact(A)[:Q]) @test vecnorm(A-Q) < eps() end - -qrpa = qrfact(areal,Val{true}) -qrpa = convert(QRPivoted{BigFloat},qrpa) -@test qrpa \ zeros(BigFloat,n) ≈ zeros(BigFloat,n) -@test qrpa \ zeros(BigFloat,n,n) ≈ zeros(BigFloat,n,n) From e0fb40418ddcf1f834363a9cb53a42f181aabd43 Mon Sep 17 00:00:00 2001 From: Felipe Noronha Date: Thu, 8 Oct 2015 01:31:15 -0300 Subject: [PATCH 0495/1938] [master] bump version of embedded git for windows --- Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 8890e06a23f08..963a13a0b9186 100644 --- a/Makefile +++ b/Makefile @@ -594,7 +594,8 @@ ifneq (,$(filter $(ARCH), i386 i486 i586 i686)) 7z x -y 7z920.exe 7z.exe 7z.dll && \ ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win32/openSUSE_13.1 \ "mingw32-libgfortran3 mingw32-libquadmath0 mingw32-libstdc++6 mingw32-libgcc_s_sjlj1 mingw32-libssp0 mingw32-libexpat1 mingw32-zlib1" && \ - cp usr/i686-w64-mingw32/sys-root/mingw/bin/*.dll . + cp usr/i686-w64-mingw32/sys-root/mingw/bin/*.dll . && \ + $(JLDOWNLOAD) PortableGit.7z https://github.com/git-for-windows/git/releases/download/v2.6.1.windows.1/PortableGit-2.6.1-32-bit.7z.exe else ifeq ($(ARCH),x86_64) cd $(JULIAHOME)/dist-extras && \ $(JLDOWNLOAD) 7z920-x64.msi http://downloads.sourceforge.net/sevenzip/7z920-x64.msi && \ @@ -603,7 +604,8 @@ else ifeq ($(ARCH),x86_64) mv _7z.exe 7z.exe && \ ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win64/openSUSE_13.1 \ "mingw64-libgfortran3 mingw64-libquadmath0 mingw64-libstdc++6 mingw64-libgcc_s_seh1 mingw64-libssp0 mingw64-libexpat1 mingw64-zlib1" && \ - cp usr/x86_64-w64-mingw32/sys-root/mingw/bin/*.dll . + cp usr/x86_64-w64-mingw32/sys-root/mingw/bin/*.dll . && \ + $(JLDOWNLOAD) PortableGit.7z https://github.com/git-for-windows/git/releases/download/v2.6.1.windows.1/PortableGit-2.6.1-64-bit.7z.exe else $(error no win-extras target for ARCH=$(ARCH)) endif @@ -615,5 +617,4 @@ endif chmod a+x 7z.dll && \ $(call spawn,./7z.exe) x -y -onsis nsis-2.46.5-Unicode-setup.exe && \ chmod a+x ./nsis/makensis.exe && \ - chmod a+x busybox.exe && \ - $(JLDOWNLOAD) PortableGit.7z https://github.com/msysgit/msysgit/releases/download/Git-1.9.5-preview20141217/PortableGit-1.9.5-preview20141217.7z + chmod a+x busybox.exe From 978c588a68e3aacf85a19203aa74315330c4c71c Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Thu, 8 Oct 2015 21:14:21 -0400 Subject: [PATCH 0496/1938] A few fixes of the deprecations of the keyword versions of cov and cor. Also make sure that all methods are tested. --- base/deprecated.jl | 18 +++++++++--------- base/statistics.jl | 16 +++++++++------- test/statistics.jl | 18 ++++++++++++++++++ 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 19c38044cb80c..1219c1107bffe 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -843,12 +843,12 @@ for f in (:remotecall, :remotecall_fetch, :remotecall_wait) end end -@deprecate cov(x::AbstractVector; corrected=true, mean=nothing) covm(x, mean, corrected) -@deprecate cov(X::AbstractMatrix; vardim=1, corrected=true, mean=nothing) covm(X, mean, vardim, corrected) -@deprecate cov(x::AbstractVector, y::AbstractVector; corrected=true, mean=nothing) covm(x, mean[1], y, mean[2], corrected) -@deprecate cov(X::AbstractVecOrMat, Y::AbstractVecOrMat; vardim=1, corrected=true, mean=nothing) covm(X, mean[1], Y, mean[2], vardim, corrected) - -@deprecate cor(x::AbstractVector; mean=nothing) corm(x, mean) -@deprecate cor(X::AbstractMatrix; vardim=1, mean=nothing) corm(X, mean, vardim) -@deprecate cor(x::AbstractVector, y::AbstractVector; mean=nothing) corm(x, mean[1], y, mean[2]) -@deprecate cor(X::AbstractVecOrMat, Y::AbstractVecOrMat; vardim=1, mean=nothing) corm(X, mean[1], Y, mean[2], vardim) +@deprecate cov(x::AbstractVector; corrected=true, mean=Base.mean(x)) covm(x, mean, corrected) +@deprecate cov(X::AbstractMatrix; vardim=1, corrected=true, mean=Base.mean(X, vardim)) covm(X, mean, vardim, corrected) +@deprecate cov(x::AbstractVector, y::AbstractVector; corrected=true, mean=(Base.mean(x), Base.mean(y))) covm(x, mean[1], y, mean[2], corrected) +@deprecate cov(X::AbstractVecOrMat, Y::AbstractVecOrMat; vardim=1, corrected=true, mean=(Base.mean(X, vardim), Base.mean(Y, vardim))) covm(X, mean[1], Y, mean[2], vardim, corrected) + +@deprecate cor(x::AbstractVector; mean=Base.mean(x)) corm(x, mean) +@deprecate cor(X::AbstractMatrix; vardim=1, mean=Base.mean(X, vardim)) corm(X, mean, vardim) +@deprecate cor(x::AbstractVector, y::AbstractVector; mean=(Base.mean(x), Base.mean(y))) corm(x, mean[1], y, mean[2]) +@deprecate cor(X::AbstractVecOrMat, Y::AbstractVecOrMat; vardim=1, mean=(Base.mean(X, vardim), Base.mean(Y, vardim))) corm(X, mean[1], Y, mean[2], vardim) diff --git a/base/statistics.jl b/base/statistics.jl index 0f18007746416..7ab0f7d17d08c 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -319,9 +319,10 @@ Compute the covariance between the vectors or matrices `X` and `Y` along the dim """ cov(X::AbstractVecOrMat, Y::AbstractVecOrMat, vardim::Int, corrected::Bool=true) = covm(X, _vmean(X, vardim), Y, _vmean(Y, vardim), vardim, corrected) -# This ugly hack is necessary to make the method below considered more specific than the deprecated method. When the old keyword version has been completely deprecated, these two methods can be merged -cov{T<:AbstractVecOrMat,S<:AbstractVecOrMat}(X::T, Y::S) = - covm(X, _vmean(X, vardim), Y, _vmean(Y, vardim), 1, true) +# This ugly hack is necessary to make the method below considered more specific than the deprecated method. When the old keyword version has been completely deprecated, these methods can be merged +cov(x::AbstractVector, Y::AbstractMatrix) = cov(x, Y, 1, true) +cov(X::AbstractMatrix, y::AbstractVector) = cov(X, y, 1, true) +cov(X::AbstractMatrix, Y::AbstractMatrix) = cov(X, Y, 1, true) ##### correlation ##### @@ -430,7 +431,7 @@ Compute the Pearson correlation matrix of the matrix `X` along the dimension `va """ cor(X::AbstractMatrix, vardim::Int) = corm(X, _vmean(X, vardim), vardim) # This ugly hack is necessary to make the method below considered more specific than the deprecated method. When the old keyword version has been completely deprecated, these two methods can be merged -cor{T<:AbstractMatrix}(X::T) = corm(X, _vmean(X, vardim), 1) +cor{T<:AbstractMatrix}(X::T) = cor(X, 1) doc""" cor(x, y) @@ -448,9 +449,10 @@ Compute the Pearson correlation between the vectors or matrices `X` and `Y` alon """ cor(x::AbstractVecOrMat, y::AbstractVecOrMat, vardim::Int) = corm(x, _vmean(x, vardim), y, _vmean(y, vardim), vardim) -# This ugly hack is necessary to make the method below considered more specific than the deprecated method. When the old keyword version has been completely deprecated, these two methods can be merged -cor(x::AbstractVecOrMat, y::AbstractVecOrMat) = - corm(x, _vmean(x, vardim), y, _vmean(y, vardim), 1) +# This ugly hack is necessary to make the method below considered more specific than the deprecated method. When the old keyword version has been completely deprecated, these methods can be merged +cor(x::AbstractVector, Y::AbstractMatrix) = cor(x, Y, 1) +cor(X::AbstractMatrix, y::AbstractVector) = cor(X, y, 1) +cor(X::AbstractMatrix, Y::AbstractMatrix) = cor(X, Y, 1) ##### median & quantiles ##### diff --git a/test/statistics.jl b/test/statistics.jl index 56638933a72db..c6de820f78790 100644 --- a/test/statistics.jl +++ b/test/statistics.jl @@ -184,30 +184,39 @@ for vd in [1, 2], zm in [true, false], cr in [true, false] @test_approx_eq c Cxx[1,1] @inferred cov(x1, cr) + @test cov(X) == Base.covm(X, mean(X, 1)) C = zm ? Base.covm(X, 0, vd, cr) : cov(X, vd, cr) @test size(C) == (k, k) @test_approx_eq C Cxx @inferred cov(X, vd, cr) + @test cov(x1, y1) == Base.covm(x1, mean(x1), y1, mean(y1)) c = zm ? Base.covm(x1, 0, y1, 0, cr) : cov(x1, y1, cr) @test isa(c, Float64) @test_approx_eq c Cxy[1,1] @inferred cov(x1, y1, cr) + if vd == 1 + @test cov(x1, Y) == Base.covm(x1, mean(x1), Y, mean(Y, 1)) + end C = zm ? Base.covm(x1, 0, Y, 0, vd, cr) : cov(x1, Y, vd, cr) @test size(C) == (1, k) @test_approx_eq C Cxy[1,:] @inferred cov(x1, Y, vd, cr) + if vd == 1 + @test cov(X, y1) == Base.covm(X, mean(X, 1), y1, mean(y1)) + end C = zm ? Base.covm(X, 0, y1, 0, vd, cr) : cov(X, y1, vd, cr) @test size(C) == (k, 1) @test_approx_eq C Cxy[:,1] @inferred cov(X, y1, vd, cr) + @test cov(X, Y) == Base.covm(X, mean(X, 1), Y, mean(Y, 1)) C = zm ? Base.covm(X, 0, Y, 0, vd, cr) : cov(X, Y, vd, cr) @test size(C) == (k, k) @@ -256,26 +265,35 @@ for vd in [1, 2], zm in [true, false] @test_approx_eq c Cxx[1,1] @inferred cor(x1) + @test cor(X) == Base.corm(X, mean(X, 1)) C = zm ? Base.corm(X, 0, vd) : cor(X, vd) @test size(C) == (k, k) @test_approx_eq C Cxx @inferred cor(X, vd) + @test cor(x1, y1) == Base.corm(x1, mean(x1), y1, mean(y1)) c = zm ? Base.corm(x1, 0, y1, 0) : cor(x1, y1) @test isa(c, Float64) @test_approx_eq c Cxy[1,1] @inferred cor(x1, y1) + if vd == 1 + @test cor(x1, Y) == Base.corm(x1, mean(x1), Y, mean(Y, 1)) + end C = zm ? Base.corm(x1, 0, Y, 0, vd) : cor(x1, Y, vd) @test size(C) == (1, k) @test_approx_eq C Cxy[1,:] @inferred cor(x1, Y, vd) + if vd == 1 + @test cor(X, y1) == Base.corm(X, mean(X, 1), y1, mean(y1)) + end C = zm ? Base.corm(X, 0, y1, 0, vd) : cor(X, y1, vd) @test size(C) == (k, 1) @test_approx_eq C Cxy[:,1] @inferred cor(X, y1, vd) + @test cor(X, Y) == Base.corm(X, mean(X, 1), Y, mean(Y, 1)) C = zm ? Base.corm(X, 0, Y, 0, vd) : cor(X, Y, vd) @test size(C) == (k, k) @test_approx_eq C Cxy From 22e86cb4b6bc403c895a2bbf337755fb91334f42 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Thu, 8 Oct 2015 23:34:56 -0400 Subject: [PATCH 0497/1938] eval build.jl files in a separate Julia process for Pkg.build (fixes #13458) --- base/pkg/entry.jl | 58 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 8e25da91524b1..9602588ea32ba 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -725,22 +725,66 @@ function warnbanner(msg...; label="[ WARNING ]", prefix="") warn(prefix="", "="^cols) end -function build!(pkgs::Vector, errs::Dict, seen::Set=Set()) +function build!(pkgs::Vector, buildstream::IO, seen::Set) for pkg in pkgs pkg == "julia" && continue pkg in seen && continue - build!(Read.requires_list(pkg),errs,push!(seen,pkg)) + build!(Read.requires_list(pkg),buildstream,push!(seen,pkg)) Read.isinstalled(pkg) || throw(PkgError("$pkg is not an installed package")) path = abspath(pkg,"deps","build.jl") isfile(path) || continue - info("Building $pkg") - cd(dirname(path)) do - try evalfile(path) - catch err - warnbanner(err, label="[ ERROR: $pkg ]") + println(buildstream, path) # send to build process for evalfile + flush(buildstream) + end +end + +function build!(pkgs::Vector, errs::Dict, seen::Set=Set()) + # To isolate the build from the running Julia process, we + # execute the build.jl files in a separate process that + # is sitting there waiting for paths to evaluate. Errors + # are serialized to errfile for later retrieval into errs[pkg] + errfile = tempname() + close(open(errfile, "w")) # create empty file + code = """ + open("$(escape_string(errfile))", "a") do f + for path_ in eachline(STDIN) + path = chomp(path_) + pkg = basename(dirname(dirname(path))) + try + info("Building \$pkg") + cd(dirname(path)) do + evalfile(path) + end + catch err + Base.Pkg.Entry.warnbanner(err, label="[ ERROR: \$pkg ]") + serialize(f, pkg) + serialize(f, err) + end + end + end + """ + io, pobj = open(detach(`$(Base.julia_cmd()) + --history-file=no + --color=$(Base.have_color ? "yes" : "no") + --eval $code`), "w", STDOUT) + try + build!(pkgs, io, seen) + close(io) + wait(pobj) + success(pobj) || error("Build process failed.") + open(errfile, "r") do f + while !eof(f) + pkg = deserialize(f) + err = deserialize(f) errs[pkg] = err end end + catch + kill(pobj) + close(io) + rethrow() + finally + isfile(errfile) && Base.rm(errfile) end end From b3b22e8bf9a4fc16dc14ff38be6bf903b86597ad Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Fri, 9 Oct 2015 01:19:37 -0700 Subject: [PATCH 0498/1938] Fixes for MSVC build --- src/gc.c | 2 +- src/ia_misc.h | 34 ++++++++++++++++++++++------------ src/threadgroup.c | 3 +++ src/threading.c | 4 +++- src/threading.h | 3 +++ 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/gc.c b/src/gc.c index 4eb9284e5274c..133b73403d27c 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1381,7 +1381,7 @@ static int gc_sweep_inc(int sweep_mask) page_done = 0; int finished = 1; - gcval_t **pfl[jl_n_threads * N_POOLS]; + gcval_t ***pfl = (gcval_t ***) alloca(jl_n_threads * N_POOLS * sizeof(gcval_t**)); // update metadata of pages that were pointed to by freelist or newpages from a pool // i.e. pages being the current allocation target diff --git a/src/ia_misc.h b/src/ia_misc.h index f94bea826a28d..a97d40d29a507 100644 --- a/src/ia_misc.h +++ b/src/ia_misc.h @@ -5,10 +5,11 @@ #include #include +#include "support/dtypes.h" #if defined(__i386__) -static __inline__ unsigned long long rdtsc(void) +STATIC_INLINE unsigned long long rdtsc(void) { unsigned long long int x; __asm__ volatile (".byte 0x0f, 0x31" : "=A" (x)); @@ -17,67 +18,76 @@ static __inline__ unsigned long long rdtsc(void) #elif defined(__x86_64__) -static inline uint64_t rdtsc() +STATIC_INLINE uint64_t rdtsc() { unsigned hi, lo; __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi)); return ((uint64_t)lo) | (((uint64_t)hi) << 32); } +#elif defined(_COMPILER_MICROSOFT_) + +#include + +STATIC_INLINE uint64_t rdtsc() +{ + return __rdtsc(); +} + #endif /* __i386__ */ #if (__MIC__) -static inline void cpu_pause() +STATIC_INLINE void cpu_pause() { _mm_delay_64(100); } -static inline void cpu_delay(int64_t cycles) +STATIC_INLINE void cpu_delay(int64_t cycles) { _mm_delay_64(cycles); } -static inline void cpu_mfence() +STATIC_INLINE void cpu_mfence() { __asm__ __volatile__ ("":::"memory"); } -static inline void cpu_sfence() +STATIC_INLINE void cpu_sfence() { __asm__ __volatile__ ("":::"memory"); } -static inline void cpu_lfence() +STATIC_INLINE void cpu_lfence() { __asm__ __volatile__ ("":::"memory"); } #else /* !__MIC__ */ -static inline void cpu_pause() +STATIC_INLINE void cpu_pause() { _mm_pause(); } -static inline void cpu_delay(int64_t cycles) +STATIC_INLINE void cpu_delay(int64_t cycles) { uint64_t s = rdtsc(); while ((rdtsc() - s) < cycles) _mm_pause(); } -static inline void cpu_mfence() +STATIC_INLINE void cpu_mfence() { _mm_mfence(); } -static inline void cpu_sfence() +STATIC_INLINE void cpu_sfence() { _mm_sfence(); } -static inline void cpu_lfence() +STATIC_INLINE void cpu_lfence() { _mm_lfence(); } diff --git a/src/threadgroup.c b/src/threadgroup.c index 37b7a9d0a85af..28ef9348e507c 100644 --- a/src/threadgroup.c +++ b/src/threadgroup.c @@ -212,3 +212,6 @@ int ti_threadgroup_destroy(ti_threadgroup_t *tg) return 0; } +#ifdef __cplusplus +} +#endif diff --git a/src/threading.c b/src/threading.c index 808a0ec95a04c..7017d3d18333a 100644 --- a/src/threading.c +++ b/src/threading.c @@ -18,8 +18,10 @@ #include #include #include +#ifndef _MSC_VER #include #include +#endif #include "julia.h" #include "julia_internal.h" @@ -447,7 +449,7 @@ void jl_init_threading(void) jl_all_task_states = &_jl_all_task_states; jl_max_threads = 1; jl_n_threads = 1; - jl_all_pgcstacks = malloc(jl_n_threads * sizeof(void*)); + jl_all_pgcstacks = (jl_gcframe_t ***) malloc(jl_n_threads * sizeof(void*)); ti_initthread(0); } diff --git a/src/threading.h b/src/threading.h index a91ffcd2c0e2f..c719b0b11392c 100644 --- a/src/threading.h +++ b/src/threading.h @@ -69,6 +69,9 @@ void ti_threadfun(void *arg); void ti_initthread(int16_t tid); jl_value_t *ti_runthread(jl_function_t *f, jl_svec_t *args, size_t nargs); +#ifdef __cplusplus +} +#endif #endif /* THREADING_H */ From c18d2716626efe4f599dce3ee8ba79369d006d1f Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Fri, 9 Oct 2015 06:03:51 -0700 Subject: [PATCH 0499/1938] Fix Windows Git path issues from upgrade to Git 2.x, causing failure of spawn test on appveyor also remove busybox since Git 2.x makes it unnecessary --- LICENSE.md | 3 +-- Makefile | 10 +++------- contrib/windows/juliarc.jl | 6 ++---- contrib/windows/msys_build.sh | 4 +--- 4 files changed, 7 insertions(+), 16 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index f3a3c3e63d3b2..5f7a455ef7b6c 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -83,10 +83,9 @@ Julia's build process uses the following external tools: Julia bundles the following external programs and libraries on some platforms: - [7-Zip](http://www.7-zip.org/license.txt) -- [BUSYBOX](https://github.com/rmyorston/busybox-w32/blob/master/LICENSE) - [GIT](http://git-scm.com/about/free-and-open-source) - [ZLIB](http://zlib.net/zlib_license.html) - [LIBEXPAT](http://expat.cvs.sourceforge.net/viewvc/expat/expat/README) - [OPENSSL](https://github.com/openssl/openssl/blob/master/LICENSE) -On some platforms, distributions of Julia contain SSL certificate authority certificates, released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License). \ No newline at end of file +On some platforms, distributions of Julia contain SSL certificate authority certificates, released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License). diff --git a/Makefile b/Makefile index 963a13a0b9186..87cf862daa515 100644 --- a/Makefile +++ b/Makefile @@ -465,10 +465,8 @@ ifeq ($(OS), WINNT) cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(bindir) && \ mkdir $(DESTDIR)$(prefix)/Git && \ 7z x PortableGit.7z -o"$(DESTDIR)$(prefix)/Git" && \ - echo "[core] eol = lf" >> "$(DESTDIR)$(prefix)/Git/etc/gitconfig" && \ - sed -i "s/\bautocrlf = true$$/autocrlf = input/" "$(DESTDIR)$(prefix)/Git/etc/gitconfig" && \ - cp busybox.exe $(DESTDIR)$(prefix)/Git/bin/echo.exe && \ - cp busybox.exe $(DESTDIR)$(prefix)/Git/bin/printf.exe ) + echo "[core] eol = lf" >> "$(DESTDIR)$(prefix)"/Git/mingw*/etc/gitconfig && \ + sed -i "s/\bautocrlf = true$$/autocrlf = input/" "$(DESTDIR)$(prefix)"/Git/mingw*/etc/gitconfig ) cd $(DESTDIR)$(bindir) && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe # create file listing for uninstall. note: must have Windows path separators and line endings. @@ -612,9 +610,7 @@ endif cd $(JULIAHOME)/dist-extras && \ $(JLDOWNLOAD) http://downloads.sourceforge.net/sevenzip/7z920_extra.7z && \ $(JLDOWNLOAD) https://unsis.googlecode.com/files/nsis-2.46.5-Unicode-setup.exe && \ - $(JLDOWNLOAD) busybox.exe http://frippery.org/files/busybox/busybox-w32-FRP-1-g9eb16cb.exe && \ chmod a+x 7z.exe && \ chmod a+x 7z.dll && \ $(call spawn,./7z.exe) x -y -onsis nsis-2.46.5-Unicode-setup.exe && \ - chmod a+x ./nsis/makensis.exe && \ - chmod a+x busybox.exe + chmod a+x ./nsis/makensis.exe diff --git a/contrib/windows/juliarc.jl b/contrib/windows/juliarc.jl index 533dc06c5fd34..db0e0ea61a36f 100644 --- a/contrib/windows/juliarc.jl +++ b/contrib/windows/juliarc.jl @@ -1,6 +1,4 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -let user_data_dir - ENV["PATH"] = JULIA_HOME*";"*joinpath(JULIA_HOME,"..","Git","bin")*";"*ENV["PATH"] - #haskey(ENV,"JULIA_EDITOR") || (ENV["JULIA_EDITOR"] = "start") #start is not a program, so this doesn't work -end +ENV["PATH"] = JULIA_HOME*";"*joinpath(JULIA_HOME,"..","Git","bin")*";"* + joinpath(JULIA_HOME,"..","Git","usr","bin")*";"*ENV["PATH"] diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index 87cc53c4c22e3..4a6f504b5e19d 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -106,7 +106,7 @@ if ! [ -e julia-installer.exe ]; then echo "Extracting $f" $SEVENZIP x -y $f >> get-deps.log fi -for i in bin/*.dll Git/bin/msys-1.0.dll Git/bin/msys-perl5_8.dll Git/bin/*.exe; do +for i in bin/*.dll Git/usr/bin/*.dll Git/usr/bin/*.exe; do $SEVENZIP e -y julia-installer.exe "\$_OUTDIR/$i" \ -ousr\\`dirname $i | sed -e 's|/julia||' -e 's|/|\\\\|g'` >> get-deps.log done @@ -179,8 +179,6 @@ if [ -z "`which make 2>/dev/null`" ]; then fi $SEVENZIP x -y `basename $f.lzma` >> get-deps.log tar -xf `basename $f` - # msysgit has an ancient version of touch that fails with `touch -c nonexistent` - cp usr/Git/bin/echo.exe bin/touch.exe export PATH=$PWD/bin:$PATH fi From a2aabfa9c0adc39aade0908808f7aa4be1e5b2c6 Mon Sep 17 00:00:00 2001 From: polarke Date: Fri, 9 Oct 2015 17:36:35 +0200 Subject: [PATCH 0500/1938] add same constructors as array --- base/sharedarray.jl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/base/sharedarray.jl b/base/sharedarray.jl index 24b1679bbe855..b4a1d2b7b1b48 100644 --- a/base/sharedarray.jl +++ b/base/sharedarray.jl @@ -24,6 +24,17 @@ type SharedArray{T,N} <: DenseArray{T,N} SharedArray(d,p,r,sn) = new(d,p,r,sn) end +call{T,N}(::Type{SharedArray{T}}, d::NTuple{N,Int}; kwargs...) = + SharedArray(T, d; kwargs...) +call{T}(::Type{SharedArray{T}}, d::Integer...; kwargs...) = + SharedArray(T, d; kwargs...) +call{T}(::Type{SharedArray{T}}, m::Integer; kwargs...) = + SharedArray(T, m; kwargs...) +call{T}(::Type{SharedArray{T}}, m::Integer, n::Integer; kwargs...) = + SharedArray(T, m, n; kwargs...) +call{T}(::Type{SharedArray{T}}, m::Integer, n::Integer, o::Integer; kwargs...) = + SharedArray(T, m, n, o; kwargs...) + function SharedArray(T::Type, dims::NTuple; init=false, pids=Int[]) N = length(dims) From 7dcc01be36a854bf95915f1cdf157df7c722ea68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiahao=20Chen=20=28=E9=99=88=E5=AE=B6=E8=B1=AA=29?= Date: Fri, 9 Oct 2015 17:38:35 -0400 Subject: [PATCH 0501/1938] Update name of OpenBLAS library in microbenchmark Ref: #13407 Cross-reference release-0.4 f8ab4bdffa5daee40f3ae5653f95e86d5cda35b3 --- test/perf/micro/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/perf/micro/Makefile b/test/perf/micro/Makefile index ae7b240e8529a..188b477f94687 100644 --- a/test/perf/micro/Makefile +++ b/test/perf/micro/Makefile @@ -18,7 +18,7 @@ endif #Which BLAS library am I using? ifeq ($(USE_SYSTEM_BLAS), 0) BLASDIR=$(JULIAHOME)/deps/build/openblas/ -LIBBLAS=$(BLASDIR)libopenblas.a +LIBBLAS=$(BLASDIR)libopenblas64_.a endif FFLAGS=-fexternal-blas From 42e9e9d18a434dc4e08df51dcbffadee93206358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiahao=20Chen=20=28=E9=99=88=E5=AE=B6=E8=B1=AA=29?= Date: Fri, 9 Oct 2015 18:16:16 -0400 Subject: [PATCH 0502/1938] micro: change LIBBLAS to use LIBBLASNAME Also get rid of -L$(BLASDIR), which is not necessary when statically linking Ref: 7dcc01be36a854bf95915f1cdf157df7c722ea68 --- test/perf/micro/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/perf/micro/Makefile b/test/perf/micro/Makefile index 188b477f94687..bb4919427e36d 100644 --- a/test/perf/micro/Makefile +++ b/test/perf/micro/Makefile @@ -18,7 +18,7 @@ endif #Which BLAS library am I using? ifeq ($(USE_SYSTEM_BLAS), 0) BLASDIR=$(JULIAHOME)/deps/build/openblas/ -LIBBLAS=$(BLASDIR)libopenblas64_.a +LIBBLAS=$(BLASDIR)$(LIBBLASNAME).a endif FFLAGS=-fexternal-blas @@ -53,11 +53,11 @@ perf.h: $(JULIAHOME)/deps/Versions.make echo '#include "$(RMATHDIR)/src/randmtzig.c"' >> $@ bin/perf%: perf.c perf.h - $(CC) -std=c99 -O$* $< -o $@ -I$(DSFMTDIR) -L$(BLASDIR) $(LIBBLAS) -L$(LIBMDIR) $(LIBM) $(CFLAGS) -lpthread + $(CC) -std=c99 -O$* $< -o $@ -I$(DSFMTDIR) $(LIBBLAS) -L$(LIBMDIR) $(LIBM) $(CFLAGS) -lpthread bin/fperf%: perf.f90 mkdir -p mods/$@ #Modules for each binary go in separate directories - $(FC) $(FFLAGS) -Jmods/$@ -O$* $< -o $@ -L$(BLASDIR) $(LIBBLAS) -L$(LIBMDIR) $(LIBM) -lpthread + $(FC) $(FFLAGS) -Jmods/$@ -O$* $< -o $@ $(LIBBLAS) -L$(LIBMDIR) $(LIBM) -lpthread benchmarks/c.csv: \ benchmarks/c0.csv \ From 1fd50a098bc672329a8ea947343a058466e24315 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 9 Oct 2015 04:04:25 -0400 Subject: [PATCH 0503/1938] implement @pure annotation (recorded in the linfo) and support in inference such that pure functions might be run at that time --- base/expr.jl | 12 ++- base/inference.jl | 189 ++++++++++++++++++++++++++++++++++------------ src/alloc.c | 1 + src/dump.c | 2 + src/interpreter.c | 3 + src/jltypes.c | 9 ++- src/julia.h | 1 + 7 files changed, 156 insertions(+), 61 deletions(-) diff --git a/base/expr.jl b/base/expr.jl index 9a322edc9c573..8378e53e517ad 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -67,18 +67,16 @@ macro eval(x) end macro inline(ex) - esc(_inline(ex)) + esc(isa(ex, Expr) ? pushmeta!(ex, :inline) : ex) end -_inline(ex::Expr) = pushmeta!(ex, :inline) -_inline(arg) = arg - macro noinline(ex) - esc(_noinline(ex)) + esc(isa(ex, Expr) ? pushmeta!(ex, :noinline) : ex) end -_noinline(ex::Expr) = pushmeta!(ex, :noinline) -_noinline(arg) = arg +macro pure(ex) + esc(isa(ex, Expr) ? pushmeta!(ex, :pure) : ex) +end ## some macro utilities ## diff --git a/base/inference.jl b/base/inference.jl index e4e5fa17d4a67..49a1115751781 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -6,7 +6,7 @@ const MAX_TYPE_DEPTH = 5 const MAX_TUPLETYPE_LEN = 8 const MAX_TUPLE_DEPTH = 4 -type NotFound +immutable NotFound end const NF = NotFound() @@ -534,7 +534,7 @@ function builtin_tfunction(f::ANY, args::ANY, argtype::ANY, vtypes::ObjectIdDict return tf[3](argtypes...) end -function isconstantfunc(f::ANY, sv::StaticVarInfo) +function isconstantref(f::ANY, sv::StaticVarInfo) if isa(f,TopNode) m = _topmod() return isconst(m, f.name) && isdefined(m, f.name) && f @@ -543,39 +543,46 @@ function isconstantfunc(f::ANY, sv::StaticVarInfo) M = f.mod; s = f.name return isdefined(M,s) && isconst(M,s) && f end - if isa(f,Expr) && is(f.head,:call) - if length(f.args) == 3 && isa(f.args[1], TopNode) && - is(f.args[1].name,:getfield) && isa(f.args[3],QuoteNode) - s = f.args[3].value - if isa(f.args[2],Module) - M = f.args[2] - else - M = isconstantfunc(f.args[2], sv) - if M === false - return false - end - M = _ieval(M) - if !isa(M,Module) - return false + if isa(f,Expr) + if is(f.head,:call) + if length(f.args) == 3 && isa(f.args[1], TopNode) && + is(f.args[1].name,:getfield) && isa(f.args[3],QuoteNode) + s = f.args[3].value + if isa(f.args[2],Module) + M = f.args[2] + else + M = isconstantref(f.args[2], sv) + if M === false + return false + end + M = _ieval(M) + if !isa(M,Module) + return false + end end + return isdefined(M,s) && isconst(M,s) && f end - return isdefined(M,s) && isconst(M,s) && f + elseif is(f.head,:inert) + return f end + return false end - - if isa(f,QuoteNode) && (isa(f.value, Function) || isa(f.value, IntrinsicFunction)) - return f.value - end - if isa(f,Function) || isa(f,IntrinsicFunction) + if isa(f,QuoteNode) return f end if isa(f,SymbolNode) f = f.name end - return isa(f,Symbol) && is_global(sv, f) && _iisconst(f) && f + if isa(f,Symbol) + return is_global(sv, f) && _iisconst(f) && f + end + if isa(f,GenSym) || isa(f,LambdaStaticData) + return false + end + return f end -const isconstantref = isconstantfunc +const isconstantfunc = isconstantref const limit_tuple_depth = t->limit_tuple_depth_(t,0) @@ -650,25 +657,7 @@ function abstract_call_gf(f, fargs, argtype, e) end end if istopfunction(tm, f, :promote_type) || istopfunction(tm, f, :typejoin) - la = length(argtypes) - c = cell(la) - for i = 1:la - t = argtypes[i] - if isType(t) && !isa(t.parameters[1],TypeVar) - c[i] = t.parameters[1] - else - return Type - end - end - if istopfunction(tm, f, :promote_type) - try - RT = Type{f(c...)} - return RT - catch - end - else - return Type{f(c...)} - end + return Type end # don't consider more than N methods. this trades off between # compiler performance and generated code performance. @@ -821,7 +810,100 @@ function abstract_apply(af, fargs, aargtypes::Vector{Any}, vtypes, sv, e) return abstract_call(af, (), Any[Vararg{Any}], vtypes, sv, ()) end +function isconstantargs(args, argtypes::Vector{Any}, sv::StaticVarInfo) + if isempty(argtypes) + return true + end + if is(args,()) || isa(argtypes[end], Vararg) + return false + end + for i = 1:length(args) + arg = args[i] + t = argtypes[i] + if !effect_free(arg, sv, false) # TODO: handle !effect_free args + return false + end + if !isType(t) || has_typevars(t.parameters[1]) + if isconstantref(arg, sv) === false + return false + end + end + end + return true +end + +function _ieval_args(args, argtypes::Vector{Any}, sv::StaticVarInfo) + c = cell(length(args)) + for i = 1:length(argtypes) + t = argtypes[i] + if isType(t) && !has_typevars(t.parameters[1]) + c[i] = t.parameters[1] + else + c[i] = _ieval(isconstantref(args[i], sv)) + end + end + return c +end + +@pure function type_typeof(v::ANY) + if isa(v, Type) + return Type{v} + end + return typeof(v) +end + +function pure_eval_call(f, fargs, argtypes, sv, e) + if !isa(f, Function) # TODO: maybe replace with :call? + return false + end + if !isconstantargs(fargs, argtypes, sv) + return false + end + + fargs = _ieval_args(fargs, argtypes, sv) + tm = _topmod() + if isgeneric(f) + atype = Tuple{Any[type_typeof(a) for a in fargs]...} + meth = _methods(f, atype, 1) + if meth === false || length(meth) != 1 + return false + end + meth = meth[1]::SimpleVector + linfo = try + func_for_method(meth[3], atype, meth[2]) + catch + NF + end + if linfo === NF + return false + end + if !linfo.pure + typeinf(linfo, meth[1], meth[2], linfo) + if !linfo.pure + return false + end + end + elseif !isdefined(f, :code) || !f.code.pure + return false + end + + local v + try + v = f(fargs...) + catch + return false + end + if isa(e, Expr) + e.head = :inert # eval_annotate will turn this into a QuoteNode + e.args = Any[v] + end + return type_typeof(v) +end + + function abstract_call(f, fargs, argtypes::Vector{Any}, vtypes, sv::StaticVarInfo, e) + t = pure_eval_call(f, fargs, argtypes, sv, e) + t !== false && return t if is(f,_apply) && length(fargs)>1 af = isconstantfunc(fargs[2], sv) if !is(af,false) @@ -925,7 +1007,8 @@ function abstract_eval_call(e, vtypes, sv::StaticVarInfo) if !(Function <: ft) && _iisdefined(:call) call_func = _ieval(:call) if isa(call_func,Function) - return abstract_call(call_func, e.args, Any[ft,argtypes...], vtypes, sv, e) + unshift!(argtypes, ft) + return abstract_call(call_func, e.args, argtypes, vtypes, sv, e) end end return Any @@ -942,7 +1025,8 @@ end function abstract_eval(e::ANY, vtypes, sv::StaticVarInfo) if isa(e,QuoteNode) - return typeof((e::QuoteNode).value) + v = (e::QuoteNode).value + return type_typeof(v) elseif isa(e,TopNode) return abstract_eval_global(_topmod(), (e::TopNode).name) elseif isa(e,Symbol) @@ -1018,6 +1102,9 @@ function abstract_eval(e::ANY, vtypes, sv::StaticVarInfo) t = Function elseif is(e.head,:copyast) t = abstract_eval(e.args[1], vtypes, sv) + elseif is(e.head,:inert) + v = e.args[1] + return type_typeof(v) else t = Any end @@ -1712,6 +1799,9 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV getfield_elim_pass(fulltree.args[3], sv) end linfo.inferred = true + body = Expr(:block) + body.args = fulltree.args[3].args::Array{Any,1} + linfo.pure = popmeta!(body, :pure)[1] fulltree = ccall(:jl_compress_ast, Any, (Any,Any), def, fulltree) end @@ -1778,6 +1868,8 @@ function eval_annotate(e::ANY, vtypes::ANY, sv::StaticVarInfo, decls, clo, undef return e #elseif is(head,:gotoifnot) || is(head,:return) # e.typ = Any + elseif is(head,:inert) + return QuoteNode(e.args[1]) elseif is(head,:(=)) # e.typ = Any s = e.args[1] @@ -1967,10 +2059,7 @@ function exprtype(x::ANY, sv::StaticVarInfo) return abstract_eval(x::Symbol, emptydict, sv) elseif isa(x,QuoteNode) v = (x::QuoteNode).value - if isa(v,Type) - return Type{v} - end - return typeof(v) + return type_typeof(v) elseif isa(x,Type) return Type{x} elseif isa(x,LambdaStaticData) @@ -2032,7 +2121,7 @@ function effect_free(e::ANY, sv, allow_volatile::Bool) allow_volatile && return true return isconst(e.mod, e.name) end - if isconstantfunc(e, sv) !== false + if isconstantref(e, sv) !== false return true end if isa(e,Expr) diff --git a/src/alloc.c b/src/alloc.c index ab453d5a8e264..402b8b209108b 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -361,6 +361,7 @@ jl_lambda_info_t *jl_new_lambda_info(jl_value_t *ast, jl_svec_t *sparams, jl_mod li->specFunctionID = 0; li->specTypes = NULL; li->inferred = 0; + li->pure = 0; li->inInference = 0; li->inCompile = 0; li->unspecialized = NULL; diff --git a/src/dump.c b/src/dump.c index e87b27aac7184..20c1571027d69 100644 --- a/src/dump.c +++ b/src/dump.c @@ -822,6 +822,7 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) jl_serialize_value(s, (jl_value_t*)li->specTypes); jl_serialize_value(s, (jl_value_t*)li->specializations); write_int8(s, li->inferred); + write_int8(s, li->pure); jl_serialize_value(s, (jl_value_t*)li->file); write_int32(s, li->line); jl_serialize_value(s, (jl_value_t*)li->module); @@ -1366,6 +1367,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t li->specializations = (jl_array_t*)jl_deserialize_value(s, (jl_value_t**)&li->specializations); if (li->specializations) jl_gc_wb(li, li->specializations); li->inferred = read_int8(s); + li->pure = read_int8(s); li->file = (jl_sym_t*)jl_deserialize_value(s, NULL); jl_gc_wb(li, li->file); li->line = read_int32(s); diff --git a/src/interpreter.c b/src/interpreter.c index df6da9788cefe..509cd6e50b319 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -493,6 +493,9 @@ static jl_value_t *eval(jl_value_t *e, jl_value_t **locals, size_t nl, size_t ng else if (ex->head == meta_sym) { return (jl_value_t*)jl_nothing; } + else if (ex->head == inert_sym) { + return args[0]; + } jl_errorf("unsupported or misplaced expression %s", ex->head->name); return (jl_value_t*)jl_nothing; } diff --git a/src/jltypes.c b/src/jltypes.c index 9c76d07c20414..9b4c3491233e5 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3420,7 +3420,7 @@ void jl_init_types(void) jl_lambda_info_type = jl_new_datatype(jl_symbol("LambdaStaticData"), jl_any_type, jl_emptysvec, - jl_svec(14, jl_symbol("ast"), jl_symbol("sparams"), + jl_svec(15, jl_symbol("ast"), jl_symbol("sparams"), jl_symbol("tfunc"), jl_symbol("name"), jl_symbol("roots"), /* jl_symbol("specTypes"), @@ -3430,15 +3430,16 @@ void jl_init_types(void) jl_symbol("module"), jl_symbol("def"), jl_symbol("capt"), jl_symbol("file"), jl_symbol("line"), - jl_symbol("inferred")), - jl_svec(14, jl_any_type, jl_simplevector_type, + jl_symbol("inferred"), + jl_symbol("pure")), + jl_svec(15, jl_any_type, jl_simplevector_type, jl_any_type, jl_sym_type, jl_any_type, jl_any_type, jl_any_type, jl_array_any_type, jl_module_type, jl_any_type, jl_any_type, jl_sym_type, jl_int32_type, - jl_bool_type), + jl_bool_type, jl_bool_type), 0, 1, 4); jl_box_type = diff --git a/src/julia.h b/src/julia.h index 43da2db6963c7..39b894d08ed55 100644 --- a/src/julia.h +++ b/src/julia.h @@ -203,6 +203,7 @@ typedef struct _jl_lambda_info_t { jl_sym_t *file; int32_t line; int8_t inferred; + int8_t pure; // hidden fields: // flag telling if inference is running on this function From 0cd98aa7b51273ba140e38ce2ea6913f3546b97f Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 10 Oct 2015 01:52:26 -0400 Subject: [PATCH 0504/1938] detect @pure in src, without requiring inference to have been run on a function --- src/alloc.c | 9 ++++++--- src/ast.c | 16 ++++++++++++++++ src/jltypes.c | 1 + src/julia.h | 2 +- src/julia_internal.h | 1 + 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 402b8b209108b..40b970cb183de 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -100,7 +100,7 @@ jl_sym_t *compositetype_sym; jl_sym_t *type_goto_sym; jl_sym_t *global_sym; jl_sym_t *tuple_sym; jl_sym_t *dot_sym; jl_sym_t *newvar_sym; jl_sym_t *boundscheck_sym; jl_sym_t *copyast_sym; -jl_sym_t *fastmath_sym; +jl_sym_t *fastmath_sym; jl_sym_t *pure_sym; jl_sym_t *simdloop_sym; jl_sym_t *meta_sym; jl_sym_t *arrow_sym; jl_sym_t *inert_sym; jl_sym_t *vararg_sym; @@ -339,8 +339,12 @@ jl_lambda_info_t *jl_new_lambda_info(jl_value_t *ast, jl_svec_t *sparams, jl_mod li->ast = ast; li->file = null_sym; li->line = 0; + li->pure = 0; if (ast != NULL && jl_is_expr(ast)) { - jl_value_t *body1 = skip_meta(jl_lam_body((jl_expr_t*)ast)->args); + jl_array_t *body = jl_lam_body((jl_expr_t*)ast)->args; + if (has_meta(body, pure_sym)) + li->pure = 1; + jl_value_t *body1 = skip_meta(body); if (jl_is_linenode(body1)) { li->file = jl_linenode_file(body1); li->line = jl_linenode_line(body1); @@ -361,7 +365,6 @@ jl_lambda_info_t *jl_new_lambda_info(jl_value_t *ast, jl_svec_t *sparams, jl_mod li->specFunctionID = 0; li->specTypes = NULL; li->inferred = 0; - li->pure = 0; li->inInference = 0; li->inCompile = 0; li->unspecialized = NULL; diff --git a/src/ast.c b/src/ast.c index 8242bc3e17932..1051b503e07be 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1008,6 +1008,22 @@ jl_value_t *skip_meta(jl_array_t *body) return body1; } +int has_meta(jl_array_t *body, jl_sym_t *sym) +{ + size_t i, l = jl_array_len(body); + for (i = 0; i < l; i++) { + jl_expr_t *stmt = (jl_expr_t*)jl_cellref(body, i); + if (jl_is_expr((jl_value_t*)stmt) && stmt->head == meta_sym) { + size_t i, l = jl_array_len(stmt->args); + for (i = 0; i < l; i++) + if (jl_cellref(stmt->args, i) == (jl_value_t*)sym) + return 1; + } + } + return 0; +} + + int jl_in_vinfo_array(jl_array_t *a, jl_sym_t *v) { size_t i, l=jl_array_len(a); diff --git a/src/jltypes.c b/src/jltypes.c index 9b4c3491233e5..04f618b84867b 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3573,6 +3573,7 @@ void jl_init_types(void) newvar_sym = jl_symbol("newvar"); copyast_sym = jl_symbol("copyast"); simdloop_sym = jl_symbol("simdloop"); + pure_sym = jl_symbol("pure"); meta_sym = jl_symbol("meta"); arrow_sym = jl_symbol("->"); dots_sym = jl_symbol("..."); diff --git a/src/julia.h b/src/julia.h index 39b894d08ed55..6b8863c684622 100644 --- a/src/julia.h +++ b/src/julia.h @@ -493,7 +493,7 @@ extern jl_sym_t *abstracttype_sym; extern jl_sym_t *bitstype_sym; extern jl_sym_t *compositetype_sym; extern jl_sym_t *type_goto_sym; extern jl_sym_t *global_sym; extern jl_sym_t *tuple_sym; extern jl_sym_t *boundscheck_sym; extern jl_sym_t *copyast_sym; -extern jl_sym_t *fastmath_sym; +extern jl_sym_t *fastmath_sym; extern jl_sym_t *pure_sym; extern jl_sym_t *simdloop_sym; extern jl_sym_t *meta_sym; extern jl_sym_t *arrow_sym; extern jl_sym_t *inert_sym; diff --git a/src/julia_internal.h b/src/julia_internal.h index ea14b50079e83..b878383ad7617 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -155,6 +155,7 @@ void jl_generate_fptr(jl_function_t *f); void jl_fptr_to_llvm(void *fptr, jl_lambda_info_t *lam, int specsig); jl_value_t* skip_meta(jl_array_t *body); +int has_meta(jl_array_t *body, jl_sym_t *sym); // backtraces #ifdef _OS_WINDOWS_ From c416e9fb92d32f1ac926d113cb931b14dc5a17ce Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 10 Oct 2015 01:07:02 -0400 Subject: [PATCH 0505/1938] add @pure annotation to promotion functions --- base/essentials.jl | 9 +++++++-- base/promotion.jl | 47 +++++++++++++++++++++++++--------------------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/base/essentials.jl b/base/essentials.jl index a02585ad1d6b8..a43d5cf13e225 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -14,6 +14,9 @@ end macro _noinline_meta() Expr(:meta, :noinline) end +macro _pure_meta() + Expr(:meta, :pure) +end # constructors for Core types in boot.jl @@ -74,7 +77,8 @@ macro generated(f) end -@generated function tuple_type_head{T<:Tuple}(::Type{T}) +function tuple_type_head{T<:Tuple}(::Type{T}) + @_pure_meta T.parameters[1] end @@ -82,7 +86,8 @@ isvarargtype(t::ANY) = isa(t,DataType)&&is((t::DataType).name,Vararg.name) isvatuple(t::DataType) = (n = length(t.parameters); n > 0 && isvarargtype(t.parameters[n])) unwrapva(t::ANY) = isvarargtype(t) ? t.parameters[1] : t -@generated function tuple_type_tail{T<:Tuple}(::Type{T}) +function tuple_type_tail{T<:Tuple}(::Type{T}) + @_pure_meta if isvatuple(T) && length(T.parameters) == 1 return T end diff --git a/base/promotion.jl b/base/promotion.jl index 63e3b9a3d7606..92fe23d5a70ff 100644 --- a/base/promotion.jl +++ b/base/promotion.jl @@ -2,10 +2,11 @@ ## type join (closest common ancestor, or least upper bound) ## -typejoin() = Bottom -typejoin(t::ANY) = t -typejoin(t::ANY, ts...) = typejoin(t, typejoin(ts...)) +typejoin() = (@_pure_meta; Bottom) +typejoin(t::ANY) = (@_pure_meta; t) +typejoin(t::ANY, ts...) = (@_pure_meta; typejoin(t, typejoin(ts...))) function typejoin(a::ANY, b::ANY) + @_pure_meta if isa(a,TypeConstructor); a = a.body; end if isa(b,TypeConstructor); b = b.body; end if a <: b @@ -102,37 +103,39 @@ end ## promotion mechanism ## -promote_type() = Bottom -promote_type(T) = T -promote_type(T, S, U, V...) = promote_type(T, promote_type(S, U, V...)) +promote_type() = (@_pure_meta; Bottom) +promote_type(T) = (@_pure_meta; T) +promote_type(T, S, U, V...) = (@_pure_meta; promote_type(T, promote_type(S, U, V...))) -promote_type(::Type{Bottom}, ::Type{Bottom}) = Bottom -promote_type{T}(::Type{T}, ::Type{T}) = T -promote_type{T}(::Type{T}, ::Type{Bottom}) = T -promote_type{T}(::Type{Bottom}, ::Type{T}) = T +promote_type(::Type{Bottom}, ::Type{Bottom}) = (@_pure_meta; Bottom) +promote_type{T}(::Type{T}, ::Type{T}) = (@_pure_meta; T) +promote_type{T}(::Type{T}, ::Type{Bottom}) = (@_pure_meta; T) +promote_type{T}(::Type{Bottom}, ::Type{T}) = (@_pure_meta; T) # Try promote_rule in both orders. Typically only one is defined, # and there is a fallback returning Bottom below, so the common case is # promote_type(T, S) => # promote_result(T, S, result, Bottom) => # typejoin(result, Bottom) => result -promote_type{T,S}(::Type{T}, ::Type{S}) = +function promote_type{T,S}(::Type{T}, ::Type{S}) + @_pure_meta promote_result(T, S, promote_rule(T,S), promote_rule(S,T)) +end -promote_rule(T, S) = Bottom +promote_rule(T, S) = (@_pure_meta; Bottom) -promote_result(t,s,T,S) = promote_type(T,S) +promote_result(t,s,T,S) = (@_pure_meta; promote_type(T,S)) # If no promote_rule is defined, both directions give Bottom. In that # case use typejoin on the original types instead. -promote_result{T,S}(::Type{T},::Type{S},::Type{Bottom},::Type{Bottom}) = typejoin(T, S) +promote_result{T,S}(::Type{T},::Type{S},::Type{Bottom},::Type{Bottom}) = (@_pure_meta; typejoin(T, S)) promote() = () promote(x) = (x,) function promote{T,S}(x::T, y::S) (convert(promote_type(T,S),x), convert(promote_type(T,S),y)) end -promote_typeof(x) = typeof(x) -promote_typeof(x, xs...) = promote_type(typeof(x), promote_typeof(xs...)) +promote_typeof(x) = (@_pure_meta; typeof(x)) +promote_typeof(x, xs...) = (@_pure_meta; promote_type(typeof(x), promote_typeof(xs...))) function promote(x, y, z) (convert(promote_typeof(x,y,z), x), convert(promote_typeof(x,y,z), y), @@ -152,15 +155,17 @@ end # Otherwise, typejoin(T,S) is called (returning Number) so no conversion # happens, and +(promote(x,y)...) is called again, causing a stack # overflow. -promote_result{T<:Number,S<:Number}(::Type{T},::Type{S},::Type{Bottom},::Type{Bottom}) = +function promote_result{T<:Number,S<:Number}(::Type{T},::Type{S},::Type{Bottom},::Type{Bottom}) + @_pure_meta promote_to_super(T, S, typejoin(T,S)) +end # promote numeric types T and S to typejoin(T,S) if T<:S or S<:T # for example this makes promote_type(Integer,Real) == Real without # promoting arbitrary pairs of numeric types to Number. -promote_to_super{T<:Number }(::Type{T}, ::Type{T}, ::Type{T}) = T -promote_to_super{T<:Number,S<:Number}(::Type{T}, ::Type{S}, ::Type{T}) = T -promote_to_super{T<:Number,S<:Number}(::Type{T}, ::Type{S}, ::Type{S}) = S +promote_to_super{T<:Number }(::Type{T}, ::Type{T}, ::Type{T}) = (@_pure_meta; T) +promote_to_super{T<:Number,S<:Number}(::Type{T}, ::Type{S}, ::Type{T}) = (@_pure_meta; T) +promote_to_super{T<:Number,S<:Number}(::Type{T}, ::Type{S}, ::Type{S}) = (@_pure_meta; S) promote_to_super{T<:Number,S<:Number}(::Type{T}, ::Type{S}, ::Type) = error("no promotion exists for ", T, " and ", S) @@ -203,7 +208,7 @@ checked_mul(x::Integer, y::Integer) = checked_mul(promote(x,y)...) # as needed. For example, if you need to provide a custom result type # for the multiplication of two types, # promote_op{R<:MyType,S<:MyType}(::MulFun, ::Type{R}, ::Type{S}) = MyType{multype(R,S)} -promote_op{R,S}(::Any, ::Type{R}, ::Type{S}) = promote_type(R, S) +promote_op{R,S}(::Any, ::Type{R}, ::Type{S}) = (@_pure_meta; promote_type(R, S)) ## catch-alls to prevent infinite recursion when definitions are missing ## From 8ce7a02f6bed51a09600307f395d8c6887f0f104 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 10 Oct 2015 01:50:37 -0400 Subject: [PATCH 0506/1938] mark some more key functions are @pure --- base/abstractarray.jl | 9 --------- base/iterator.jl | 3 ++- base/reflection.jl | 27 ++++++++++++++------------- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 3fde6b72ed129..7d6eacaa640b8 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -117,15 +117,6 @@ linearindexing(A::AbstractArray, B::AbstractArray...) = linearindexing(linearind linearindexing(::LinearFast, ::LinearFast) = LinearFast() linearindexing(::LinearIndexing, ::LinearIndexing) = LinearSlow() -# The real @inline macro is not available this early in the bootstrap, so this -# internal macro splices the meta Expr directly into the function body. -macro _inline_meta() - Expr(:meta, :inline) -end -macro _noinline_meta() - Expr(:meta, :noinline) -end - ## Bounds checking ## @generated function trailingsize{T,N,n}(A::AbstractArray{T,N}, ::Type{Val{n}}) n > N && return 1 diff --git a/base/iterator.jl b/base/iterator.jl index d3f76c1e83f0c..a040728704ffc 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -45,7 +45,8 @@ immutable Zip{I, Z<:AbstractZipIterator} <: AbstractZipIterator end zip(a, b, c...) = Zip(a, zip(b, c...)) length(z::Zip) = min(length(z.a), length(z.z)) -@generated function tuple_type_cons{S,T<:Tuple}(::Type{S}, ::Type{T}) +function tuple_type_cons{S,T<:Tuple}(::Type{S}, ::Type{T}) + @_pure_meta Tuple{S, T.parameters...} end eltype{I,Z}(::Type{Zip{I,Z}}) = tuple_type_cons(eltype(I), eltype(Z)) diff --git a/base/reflection.jl b/base/reflection.jl index feda3d695bdf3..9d152edaafc53 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -71,16 +71,16 @@ isconst(m::Module, s::Symbol) = object_id(x::ANY) = ccall(:jl_object_id, UInt, (Any,), x) # type predicates -isimmutable(x::ANY) = (isa(x,Tuple) || !typeof(x).mutable) -isstructtype(t::DataType) = nfields(t) != 0 || (t.size==0 && !t.abstract) -isstructtype(x) = false -isbits(t::DataType) = !t.mutable & t.pointerfree & isleaftype(t) -isbits(t::Type) = false -isbits(x) = isbits(typeof(x)) -isleaftype(t::ANY) = ccall(:jl_is_leaf_type, Int32, (Any,), t) != 0 +isimmutable(x::ANY) = (@_pure_meta; (isa(x,Tuple) || !typeof(x).mutable)) +isstructtype(t::DataType) = (@_pure_meta; nfields(t) != 0 || (t.size==0 && !t.abstract)) +isstructtype(x) = (@_pure_meta; false) +isbits(t::DataType) = (@_pure_meta; !t.mutable & t.pointerfree & isleaftype(t)) +isbits(t::Type) = (@_pure_meta; false) +isbits(x) = (@_pure_meta; isbits(typeof(x))) +isleaftype(t::ANY) = (@_pure_meta; ccall(:jl_is_leaf_type, Int32, (Any,), t) != 0) -typeintersect(a::ANY,b::ANY) = ccall(:jl_type_intersection, Any, (Any,Any), a, b) -typeseq(a::ANY,b::ANY) = a<:b && b<:a +typeintersect(a::ANY,b::ANY) = (@_pure_meta; ccall(:jl_type_intersection, Any, (Any,Any), a, b)) +typeseq(a::ANY,b::ANY) = (@_pure_meta; a<:b && b<:a) function fieldoffsets(x::DataType) offsets = Array(Int, nfields(x)) @@ -88,8 +88,8 @@ function fieldoffsets(x::DataType) offsets end -type_alignment(x::DataType) = ccall(:jl_get_alignment,Csize_t,(Any,),x) -field_offset(x::DataType,idx) = ccall(:jl_get_field_offset,Csize_t,(Any,Int32),x,idx) +type_alignment(x::DataType) = (@_pure_meta; ccall(:jl_get_alignment,Csize_t,(Any,),x)) +field_offset(x::DataType,idx) = (@_pure_meta; ccall(:jl_get_field_offset,Csize_t,(Any,Int32),x,idx)) # return all instances, for types that can be enumerated function instances end @@ -114,11 +114,12 @@ subtypes(m::Module, x::DataType) = sort(collect(_subtypes(m, x)), by=string) subtypes(x::DataType) = subtypes(Main, x) # function reflection -isgeneric(f::ANY) = (isa(f,Function) && isa(f.env,MethodTable)) +isgeneric(f::ANY) = (@_pure_meta; (isa(f,Function) && isa(f.env,MethodTable))) function_name(f::Function) = isgeneric(f) ? f.env.name : (:anonymous) function to_tuple_type(t::ANY) + @_pure_meta if isa(t,Tuple) || isa(t,AbstractArray) || isa(t,SimpleVector) t = Tuple{t...} end @@ -132,7 +133,7 @@ function to_tuple_type(t::ANY) t end -tt_cons(t::ANY, tup::ANY) = Tuple{t, (isa(tup, Type) ? tup.parameters : tup)...} +tt_cons(t::ANY, tup::ANY) = (@_pure_meta; Tuple{t, (isa(tup, Type) ? tup.parameters : tup)...}) code_lowered(f, t::ANY) = map(m->uncompressed_ast(m.func.code), methods(f, t)) function methods(f::Function,t::ANY) From 4fed9e5f1c8e55530576d390971091dc9fff5e74 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sat, 10 Oct 2015 03:53:31 -0400 Subject: [PATCH 0507/1938] Improve apply_type inference for complicated types The motivating example here is: ``` julia> T = Cxx.CppValue{Cxx.CxxQualType{Cxx.CppTemplate{Cxx.CppBaseType{:A},Tuple{Cxx.CxxQualType{Cxx.CppBaseType{:B},(false,false,false)}}},(false,false,false)}} Cxx.CppValue{Cxx.CxxQualType{Cxx.CppTemplate{Cxx.CppBaseType{:A},Tuple{Cxx.CxxQualType{Cxx.CppBaseType{:B},(false,false,false)}}},(false,false,false)},N} julia> @eval baz() = ($T){4} ``` Without this patch it infers it as Type{<:Cxx.CppValue{Cxx.CxxQualType{Cxx.CppTemplate{Cxx.CppBaseType{:A},Tuple{Cxx.CxxQualType{Cxx.CppBaseType{:B},(false,false,false)}}},(false,false,false)},N}}, which seems pretty sad given how simple the expression is. --- base/inference.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/inference.jl b/base/inference.jl index e4e5fa17d4a67..ed5516179bd6f 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -451,10 +451,11 @@ const apply_type_tfunc = function (A, args...) appl = headtype uncertain = true end + !uncertain && return Type{appl} if type_too_complex(appl,0) return Type{TypeVar(:_,headtype)} end - uncertain && !isa(appl,TypeVar) ? Type{TypeVar(:_,appl)} : Type{appl} + !isa(appl,TypeVar) ? Type{TypeVar(:_,appl)} : Type{appl} end add_tfunc(apply_type, 1, IInf, apply_type_tfunc) From caab3108933a7233ad9406e5328250cf2e289dfd Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 10 Oct 2015 03:04:20 -0700 Subject: [PATCH 0508/1938] Use delayed expansion in CXXLD to fix #13337 generation of libjulia.dll.a on Windows --- src/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index f403327b5fc46..480e5674e224c 100644 --- a/src/Makefile +++ b/src/Makefile @@ -135,12 +135,12 @@ $(BUILDDIR)/julia_version.h: $(JULIAHOME)/VERSION mv $@.$(JULIA_BUILD_MODE).tmp $@ ifneq ($(USEMSVC), 1) -CXXLD := $(CXX) -shared +CXXLD = $(CXX) -shared ifeq ($(OS),WINNT) CXXLD += -Wl,--out-implib,$(build_libdir)/$(notdir $@).a endif else -CXXLD := $(LD) -dll -export:jl_setjmp -export:jl_longjmp +CXXLD = $(LD) -dll -export:jl_setjmp -export:jl_longjmp endif $(build_shlibdir)/libjulia-debug.$(SHLIB_EXT): $(SRCDIR)/julia.expmap $(DOBJS) $(BUILDDIR)/flisp/libflisp-debug.a $(BUILDDIR)/support/libsupport-debug.a $(LIBUV) From 09c9a195add174e1f7f141703caf8a74fed6f1d6 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 10 Oct 2015 03:09:35 -0700 Subject: [PATCH 0509/1938] Fix gitconfig modification with bundled git --- Makefile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 87cf862daa515..77477dfe1e4fa 100644 --- a/Makefile +++ b/Makefile @@ -426,6 +426,12 @@ distclean dist-clean: dist: @echo \'dist\' target is deprecated: use \'binary-dist\' instead. +ifeq ($(ARCH),x86_64) +GITCONFIG := $(DESTDIR)$(prefix)/Git/mingw64/etc/gitconfig +else +GITCONFIG := $(DESTDIR)$(prefix)/Git/mingw32/etc/gitconfig +endif + binary-dist: distclean ifeq ($(USE_SYSTEM_BLAS),0) ifeq ($(ISX86),1) @@ -465,8 +471,8 @@ ifeq ($(OS), WINNT) cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(bindir) && \ mkdir $(DESTDIR)$(prefix)/Git && \ 7z x PortableGit.7z -o"$(DESTDIR)$(prefix)/Git" && \ - echo "[core] eol = lf" >> "$(DESTDIR)$(prefix)"/Git/mingw*/etc/gitconfig && \ - sed -i "s/\bautocrlf = true$$/autocrlf = input/" "$(DESTDIR)$(prefix)"/Git/mingw*/etc/gitconfig ) + echo "[core] eol = lf" >> "$(GITCONFIG)" && \ + sed -i "s/\bautocrlf = true$$/autocrlf = input/" "$(GITCONFIG)" ) cd $(DESTDIR)$(bindir) && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe # create file listing for uninstall. note: must have Windows path separators and line endings. From ae154d076a6ae75bfdb9a0a377a6a5f9b0e1096f Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 10 Oct 2015 03:05:24 -0700 Subject: [PATCH 0510/1938] Fix make binary-dist on Windows for out of tree build --- Makefile | 20 ++++++++++---------- contrib/windows/build-installer.nsi | 3 --- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 77477dfe1e4fa..e58e3fc574e1f 100644 --- a/Makefile +++ b/Makefile @@ -427,9 +427,9 @@ dist: @echo \'dist\' target is deprecated: use \'binary-dist\' instead. ifeq ($(ARCH),x86_64) -GITCONFIG := $(DESTDIR)$(prefix)/Git/mingw64/etc/gitconfig +GITCONFIG := $(BUILDROOT)/julia-$(JULIA_COMMIT)/Git/mingw64/etc/gitconfig else -GITCONFIG := $(DESTDIR)$(prefix)/Git/mingw32/etc/gitconfig +GITCONFIG := $(BUILDROOT)/julia-$(JULIA_COMMIT)/Git/mingw32/etc/gitconfig endif binary-dist: distclean @@ -448,7 +448,7 @@ ifneq ($(DESTDIR),) $(error DESTDIR must not be set for make binary-dist) endif @$(MAKE) -C $(BUILDROOT) -f $(JULIAHOME)/Makefile install - cp $(JULIAHOME)/LICENSE.md $(prefix) + cp $(JULIAHOME)/LICENSE.md $(BUILDROOT)/julia-$(JULIA_COMMIT) ifneq ($(OS), WINNT) -$(JULIAHOME)/contrib/fixup-libgfortran.sh $(DESTDIR)$(private_libdir) endif @@ -468,18 +468,18 @@ endif ifeq ($(OS), WINNT) [ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \ - cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(bindir) && \ - mkdir $(DESTDIR)$(prefix)/Git && \ - 7z x PortableGit.7z -o"$(DESTDIR)$(prefix)/Git" && \ + cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin && \ + mkdir $(BUILDROOT)/julia-$(JULIA_COMMIT)/Git && \ + 7z x PortableGit.7z -o"$(BUILDROOT)/julia-$(JULIA_COMMIT)/Git" && \ echo "[core] eol = lf" >> "$(GITCONFIG)" && \ sed -i "s/\bautocrlf = true$$/autocrlf = input/" "$(GITCONFIG)" ) - cd $(DESTDIR)$(bindir) && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe + cd $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe # create file listing for uninstall. note: must have Windows path separators and line endings. - cd $(prefix) && find * | sed -e 's/\//\\/g' -e 's/$$/\r/g' > etc/uninstall.log + cd $(BUILDROOT)/julia-$(JULIA_COMMIT) && find * | sed -e 's/\//\\/g' -e 's/$$/\r/g' > etc/uninstall.log # build nsis package - $(call spawn,$(JULIAHOME)/dist-extras/nsis/makensis.exe) -NOCD -DVersion=$(JULIA_VERSION) -DArch=$(ARCH) -DCommit=$(JULIA_COMMIT) $(call cygpath_w,$(JULIAHOME)/contrib/windows/build-installer.nsi) + cd $(BUILDROOT) && $(call spawn,$(JULIAHOME)/dist-extras/nsis/makensis.exe) -NOCD -DVersion=$(JULIA_VERSION) -DArch=$(ARCH) -DCommit=$(JULIA_COMMIT) -DMUI_ICON="$(call cygpath_w,$(JULIAHOME)/contrib/windows/julia.ico)" $(call cygpath_w,$(JULIAHOME)/contrib/windows/build-installer.nsi) # compress nsis installer and combine with 7zip self-extracting header cd $(BUILDROOT) && $(JULIAHOME)/dist-extras/7z a -mx9 "julia-install-$(JULIA_COMMIT)-$(ARCH).7z" julia-installer.exe @@ -488,7 +488,7 @@ ifeq ($(OS), WINNT) else cd $(BUILDROOT) && $(TAR) zcvf $(JULIA_BINARYDIST_TARNAME).tar.gz julia-$(JULIA_COMMIT) endif - rm -fr $(prefix) + rm -fr $(BUILDROOT)/julia-$(JULIA_COMMIT) # this target does not accept BUILDROOT light-source-dist.tmp: $(JULIAHOME)/doc/_build/html diff --git a/contrib/windows/build-installer.nsi b/contrib/windows/build-installer.nsi index f4718688bd136..8896b0a0b1552 100644 --- a/contrib/windows/build-installer.nsi +++ b/contrib/windows/build-installer.nsi @@ -33,9 +33,6 @@ Function createDesktopLink ${EndIf} FunctionEnd -# Icon settings -!define MUI_ICON "contrib\windows\julia.ico" - # Variable definitions used in installer pages InstallDir "$LOCALAPPDATA\Julia-${Version}" !define JuliaStartMenuFolder "Julia ${Version}" From 446ddaab49e3923aa8ecd4ca6b81f91953cc5191 Mon Sep 17 00:00:00 2001 From: Michael Hatherly Date: Sat, 10 Oct 2015 22:25:13 +0200 Subject: [PATCH 0511/1938] Fix regression in `atrepl` Docs attached to numbers weren't being found. Finding docs for specific methods using the syntax help?> f(::Int) wasn't being recognised due to changes in how `gen_call_with_extracted_types` generated an error when it failed. (Previously the error was thrown by the function, but is now returned as an `:(error("..."))` expression.) --- base/docs/utils.jl | 9 +++++---- test/docs.jl | 7 +++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/base/docs/utils.jl b/base/docs/utils.jl index 3df0009b0564c..160280da0d85b 100644 --- a/base/docs/utils.jl +++ b/base/docs/utils.jl @@ -126,13 +126,14 @@ repl(ex::Expr) = isregex(ex) ? :(apropos($ex)) : _repl(ex) repl(str::AbstractString) = :(apropos($str)) -repl(other) = nothing +repl(other) = :(@doc $(esc(other))) function _repl(x) docs = :(@doc $(esc(x))) - try - # Handles function call syntax where each argument is a symbol. - isexpr(x, :call) && (docs = Base.gen_call_with_extracted_types(doc, x)) + if isexpr(x, :call) + # Handles function call syntax where each argument is an atom (symbol, number, etc.) + t = Base.gen_call_with_extracted_types(doc, x) + (isexpr(t, :call, 3) && t.args[1] == doc) && (docs = t) end if isfield(x) quote diff --git a/test/docs.jl b/test/docs.jl index 306cbfbb9411a..9d2a915afe906 100644 --- a/test/docs.jl +++ b/test/docs.jl @@ -411,6 +411,13 @@ f12593_2() = 1 # test that macro documentation works @test (Docs.@repl @assert) !== nothing +@test (Docs.@repl 0) !== nothing + +let t = @doc(DocsTest.t(::Int, ::Int)) + @test docstrings_equal(Docs.@repl(DocsTest.t(0, 0)), t) + @test docstrings_equal(Docs.@repl(DocsTest.t(::Int, ::Int)), t) +end + # Issue #13467. @test (Docs.@repl @r_str) !== nothing From acfe4703ddf23e0e90bdd23fc71fce62fd3751db Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Sat, 10 Oct 2015 15:24:38 -0600 Subject: [PATCH 0512/1938] Fix error message interpolation in dates/types.jl --- base/dates/types.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/dates/types.jl b/base/dates/types.jl index ce9c7cb933ee1..36b92c20808d3 100644 --- a/base/dates/types.jl +++ b/base/dates/types.jl @@ -80,7 +80,7 @@ daysinmonth(y,m) = DAYSINMONTH[m] + (m == 2 && isleapyear(y)) function DateTime(y::Int64,m::Int64=1,d::Int64=1, h::Int64=0,mi::Int64=0,s::Int64=0,ms::Int64=0) 0 < m < 13 || throw(ArgumentError("Month: $m out of range (1:12)")) - 0 < d < daysinmonth(y,m)+1 || throw(ArgumentError("Day: $d out of range (1:$daysinmonth(y,m))")) + 0 < d < daysinmonth(y,m)+1 || throw(ArgumentError("Day: $d out of range (1:$(daysinmonth(y,m)))")) -1 < h < 24 || throw(ArgumentError("Hour: $h out of range (1:23)")) -1 < mi < 60 || throw(ArgumentError("Minute: $mi out of range (1:59)")) -1 < s < 60 || throw(ArgumentError("Second: $s out of range (1:59)")) @@ -90,7 +90,7 @@ function DateTime(y::Int64,m::Int64=1,d::Int64=1, end function Date(y::Int64,m::Int64=1,d::Int64=1) 0 < m < 13 || throw(ArgumentError("Month: $m out of range (1:12)")) - 0 < d < daysinmonth(y,m)+1 || throw(ArgumentError("Day: $d out of range (1:$daysinmonth(y,m))")) + 0 < d < daysinmonth(y,m)+1 || throw(ArgumentError("Day: $d out of range (1:$(daysinmonth(y,m)))")) return Date(UTD(totaldays(y,m,d))) end From b7697e70c3824b81e6b79a591d7f673b345e538c Mon Sep 17 00:00:00 2001 From: Jay Weisskopf Date: Sat, 29 Aug 2015 02:08:34 -0400 Subject: [PATCH 0513/1938] Add `Libdl.dlext` to documentation fixes #12865 --- base/docs/helpdb.jl | 7 +++++++ doc/stdlib/libdl.rst | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index fc0b6c6616af7..3754cefea2703 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -364,6 +364,13 @@ Convert a number of seconds since the epoch to broken-down format, with fields ` """ Libc.TmStruct +doc""" + dlext + +File extension for dynamic libraries (e.g. dll, dylib, so) on the current platform. +""" +Libdl.dlext + doc""" time(t::TmStruct) diff --git a/doc/stdlib/libdl.rst b/doc/stdlib/libdl.rst index a7a5f24dd4a5f..fd7d80002ca46 100644 --- a/doc/stdlib/libdl.rst +++ b/doc/stdlib/libdl.rst @@ -68,6 +68,10 @@ Close shared library referenced by handle. +.. data:: dlext + + File extension for dynamic libraries (e.g. dll, dylib, so) on the current platform. + .. function:: find_library(names, locations) .. Docstring generated from Julia source From 78a4d25b70c1ad19831482bb09e844354efa3efc Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Sat, 10 Oct 2015 18:38:42 -0400 Subject: [PATCH 0514/1938] Minor doc change to getsockname --- base/docs/helpdb.jl | 5 ++--- doc/stdlib/io-network.rst | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index d4442c92309cc..7421b8a7d4ce3 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -11831,7 +11831,6 @@ Base.(:$)(x, y) doc""" getsockname(sock::Union{TCPServer, TCPSocket}) -> (IPAddr,UInt16) -Get the IP address and the port of the TCP server socket, to which it is bound, -or the peer connected to the TCP server socket. +Get the IP address and the port that the given TCP socket is connected to (or bound to, in the case of TCPServer). """ -getsockname \ No newline at end of file +getsockname diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index b8036b244f1de..b0d8d1c793e40 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -805,8 +805,7 @@ Network I/O .. function:: getsockname(sock::Union{TCPServer, TCPSocket}) -> (IPAddr,UInt16) - Get the IP address and the port of the TCP server socket, to which it is bound, - or the peer connected to the TCP server socket. + Get the IP address and the port that the given TCP socket is connected to (or bound to, in the case of TCPServer). .. function:: parseip(addr) From 6a1477f641c4519e4d3ba91ec0d815ab47497018 Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Sat, 10 Oct 2015 08:02:27 -0400 Subject: [PATCH 0515/1938] Remove importall from LinAlg --- base/linalg.jl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/base/linalg.jl b/base/linalg.jl index 5947cd6d59d6f..5c695f24b3ea3 100644 --- a/base/linalg.jl +++ b/base/linalg.jl @@ -2,11 +2,13 @@ module LinAlg -importall Base -importall ..Base.Operators -import Base: USE_BLAS64, size, copy, copy_transpose!, power_by_squaring, - print_matrix, transpose!, unsafe_getindex, unsafe_setindex!, - isapprox +import Base: \, /, *, ^, +, -, ==, ./, .* +import Base: A_mul_Bt, At_ldiv_Bt, A_rdiv_Bc, At_ldiv_B, Ac_mul_Bc, A_mul_Bc, Ac_mul_B, + Ac_ldiv_B, Ac_ldiv_Bc, At_mul_Bt, A_rdiv_Bt, At_mul_B +import Base: USE_BLAS64, abs, big, ceil, conj, convert, copy, copy!, copy_transpose!, + ctranspose, ctranspose!, eltype, eye, findmax, findmin, fill!, floor, full, getindex, + imag, inv, isapprox, kron, ndims, power_by_squaring, print_matrix, real, setindex!, + show, similar, size, transpose, transpose!, unsafe_getindex, unsafe_setindex! export # Modules From ec8ed0c5d1253f492bfb33db679d371c5695de0a Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Sat, 10 Oct 2015 20:51:31 -0400 Subject: [PATCH 0516/1938] Define push! and shift! for channels --- base/channels.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/base/channels.jl b/base/channels.jl index 4d1b1c881a9cb..409965540a770 100644 --- a/base/channels.jl +++ b/base/channels.jl @@ -72,6 +72,8 @@ function put!(c::Channel, v) v end +push!(c::Channel, v) = put!(c, v) + function fetch(c::Channel) wait(c) c.data[c.take_pos] @@ -86,6 +88,8 @@ function take!(c::Channel) v end +shift!(c::Channel) = take!(c) + isready(c::Channel) = (c.take_pos == c.put_pos ? false : true) function wait(c::Channel) From 0ac8d805461acdfa3a9b973dc34e1e29f5ee9145 Mon Sep 17 00:00:00 2001 From: Isaiah Date: Fri, 9 Oct 2015 14:49:24 -0400 Subject: [PATCH 0517/1938] Some updates to CONTRIBUTING Adds "learning Julia" and "before filing an issue" sections to provide brief guidance for the most common situations. Also some cosmetic and ordering changes. --- CONTRIBUTING.md | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fb3ac0c0168ec..4dd94057f669e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,20 @@ # Notes for Julia Contributors -Hi! Thanks for checking out Julia. If you have questions or problems, the [Julia dev list](https://groups.google.com/forum/?fromgroups#!forum/julia-dev) is a good place to post them, but you should check out the [online Julia docs](http://docs.Julialang.org/en/latest/) first. If you have changes you'd like to contribute, you'll need a [free GitHub account](https://github.com/signup/free). (If you're reading this on GitHub, you probably already have one.) You'll also want a working copy of Julia, built from source (you can still contribute with a binary install, but it's harder and not really recommended). In list form, here's what to do to become a contributor: +Hi! If you are new to the Julia community: welcome, and thanks for trying Julia. Please be sure to respect our [community standards](http://julialang.org/community/standards/) in all interactions. -* Join the [dev list](https://groups.google.com/forum/?fromgroups#!forum/julia-dev). +## Learning Julia + +[The learning page](http://julialang.org/learning/) has a great list of resources for new and experienced users alike. [This tutorial video](https://www.youtube.com/watch?v=vWkgEddb4-A) is one recommended starting point, as is the "[Invitation to Julia](https://www.youtube.com/watch?v=gQ1y5NUD_RI)" workshop video from JuliaCon 2015 ([slide materials here](https://github.com/dpsanders/invitation_to_julia)). The [Julia documentation](http://docs.Julialang.org/en/latest/) covers the language and core library features, and is [searchable](http://docs.Julialang.org/en/latest/search/). (note: Javascript required). + +## Before filing an issue + +- Reporting a potential bug? Please read the "[How to file a bug report](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#how-to-file-a-bug-report)" section to make sure that all necessary information is included. + +- Contributing code? Be sure to review the [contributor checklist](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#contributor-checklist) for helpful tips on the tools we use to build Julia. + +- Library feature requests are generally not accepted on this issue tracker. New libraries should be developed as [packages](http://docs.julialang.org/en/release-0.4/manual/packages/#package-development). Please email the [julia-users](https://groups.google.com/forum/#!forum/julia-users) mailing list to discuss library ideas. Doing so will often lead to pointers to existing projects and bring together collaborators with common interests. + +## Contributor Checklist * Create a [GitHub account](https://github.com/signup/free). @@ -10,13 +22,15 @@ Hi! Thanks for checking out Julia. If you have questions or problems, the [Julia * Build the software and libraries (the first time takes a while, but it's fast after that). Detailed build instructions are in the [README](https://github.com/JuliaLang/julia/tree/master/README.md). Julia depends on several external packages; most are automatically downloaded and installed, but are less frequently updated than Julia itself. -* Keep Julia current. Julia is a fast-moving target, and many details of the language are still settling out. Keep your repository up-to-date and rebase your work in progress frequently. +* Keep Julia current. Julia is a fast-moving target, and many details of the language are still settling out. Keep the repository up-to-date and rebase work-in-progress frequently to make merges simpler. * Learn to use [git](http://git-scm.com), the version control system used by GitHub and the Julia project. Try a tutorial such as the one [provided by GitHub](http://try.GitHub.io/levels/1/challenges/1). -* Respect our [community standards](http://julialang.org/community/standards/). +* Review discussions on the [dev mailing list](https://groups.google.com/forum/?fromgroups#!forum/julia-dev). + +* For more detailed tips, read the [submission guide](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#submitting-contributions) below. -* Relax and have fun. +* Relax and have fun! ## How to file a bug report @@ -34,11 +48,11 @@ A useful bug report filed as a GitHub issue provides information about how to re 3. When filing a bug report, provide where possible: - The full error message, including the backtrace. - A minimal working example, i.e. the smallest chunk of code that triggers the error. Ideally, this should be code that can be pasted into a REPL or run from a source file. If the code is larger than (say) 50 lines, consider putting it in a [gist](https://gist.github.com). - - The version of Julia you are using as provided by the `versioninfo()` command. Occasionally, the longer output produced by `versioninfo(true)` may be useful also, especially if the issue is related to a specific package. + - The version of Julia as provided by the `versioninfo()` command. Occasionally, the longer output produced by `versioninfo(true)` may be useful also, especially if the issue is related to a specific package. -4. When pasting code blocks or output, put triple backquotes (\`\`\`) around the text so GitHub will format it nicely. You can format code statements by surrounding it in single backquotes (\`). Be aware that the `@` sign tags users on GitHub, so references to macros should always be in single backquotes. See [GitHub's guide on Markdown](https://guides.github.com/features/mastering-markdown/) for more formatting tricks. +4. When pasting code blocks or output, put triple backquotes (\`\`\`) around the text so GitHub will format it nicely. Code statements should be surrounded by single backquotes (\`). Be aware that the `@` sign tags users on GitHub, so references to macros should always be in single backquotes. See [GitHub's guide on Markdown](https://guides.github.com/features/mastering-markdown/) for more formatting tricks. -## Submitting your contributions +## Submitting contributions ### Contributing a Julia package @@ -162,10 +176,6 @@ Make sure that [Travis](http://www.travis-ci.org) greenlights the pull request w - To remove whitespace relative to the `master` branch, run `git rebase --whitespace=fix master`. -## Getting help - -While getting familiar with Julia, remember to check out [the docs](http://docs.Julialang.org/en/latest/), keeping in mind that they are [searchable](http://docs.Julialang.org/en/latest/search/). (If you use a script blocker then you'll have to unblock that page.) The [source code](https://github.com/JuliaLang/julia) is an excellent source of examples (and it's mostly pretty approachable). If you're still stumped, post something on [the dev list](https://groups.google.com/forum/?fromgroups#!forum/julia-dev), but you may want to search the archives first to see if there's already been a discussion about what you're stuck on. - ## Resources * Julia From 382671052ad7adb11b2e8fe0c7c00da5d9522160 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sun, 11 Oct 2015 01:53:25 -0400 Subject: [PATCH 0518/1938] Fix potential heap overflow in debuginfo.cpp Thanks AddressSanitizer! --- src/debuginfo.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index c2d10b24396d0..07142b08e120d 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -524,7 +524,7 @@ void lookup_pointer(DIContext *context, char **name, size_t *line, done: // If this is a jlcall wrapper, set fromC to match JIT behavior - if (*name == NULL || memcmp(*name, "jlcall_",7) == 0) { + if (*name == NULL || ((strlen(*name) > 7) && (memcmp(*name, "jlcall_",7) == 0))) { *fromC = true; } } From af71b86edde26e607fc95fd8a8e73075ef675745 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sun, 11 Oct 2015 03:51:19 -0400 Subject: [PATCH 0519/1938] Do not pass RTLD_DEEPIND to dlopen in asan mode Passing RTLD_DEEPIND causes asan's interception of libc symbols to fail, causing all kinds of sadness (particularly mismatched allocators). --- src/dlload.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/dlload.c b/src/dlload.c index 17e6411596e7b..294e513821955 100644 --- a/src/dlload.c +++ b/src/dlload.c @@ -39,6 +39,11 @@ static char *extensions[] = { ".so", "" }; extern char *julia_home; #define JL_RTLD(flags, FLAG) (flags & JL_RTLD_ ## FLAG ? RTLD_ ## FLAG : 0) +#ifdef __has_feature +# if __has_feature(address_sanitizer) +# define SANITIZE_ADDRESS +# endif +#endif DLLEXPORT int jl_uv_dlopen(const char *filename, jl_uv_libhandle lib_, unsigned flags) { @@ -58,7 +63,7 @@ DLLEXPORT int jl_uv_dlopen(const char *filename, jl_uv_libhandle lib_, unsigned #ifdef RTLD_NOLOAD | JL_RTLD(flags, NOLOAD) #endif -#ifdef RTLD_DEEPBIND +#if defined(RTLD_DEEPBIND) && !defined(SANITIZE_ADDRESS) | JL_RTLD(flags, DEEPBIND) #endif #ifdef RTLD_FIRST From a1e9bbb143231a18aa16e23f76beb072dcc65b77 Mon Sep 17 00:00:00 2001 From: Matthieu Gomez Date: Sat, 10 Oct 2015 18:16:28 -0400 Subject: [PATCH 0520/1938] vecdot --- base/linalg/generic.jl | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/base/linalg/generic.jl b/base/linalg/generic.jl index c8948f30c3eff..9d15ce0f5bc92 100644 --- a/base/linalg/generic.jl +++ b/base/linalg/generic.jl @@ -242,17 +242,23 @@ end @inline norm(x::Number, p::Real=2) = vecnorm(x, p) -function vecdot(x::AbstractVector, y::AbstractVector) +function vecdot(x::AbstractArray, y::AbstractArray) lx = length(x) if lx != length(y) - throw(DimensionMismatch("vector x has length $lx, but vector y has length $(length(y))")) + throw(DimensionMismatch("first array has length $(lx) which does not match the length of the second, $(length(y)).")) end if lx == 0 return dot(zero(eltype(x)), zero(eltype(y))) end - s = dot(x[1], y[1]) - @inbounds for i = 2:lx - s += dot(x[i], y[i]) + s = zero(dot(x[1], y[1])) + if size(x) == size(y) + for I in eachindex(x, y) + @inbounds s += dot(x[I], y[I]) + end + else + for (Ix, Iy) in zip(eachindex(x), eachindex(y)) + @inbounds s += dot(x[Ix], y[Iy]) + end end s end From 8e98763048d56a769ff2a2a92bb65c2c23cc135b Mon Sep 17 00:00:00 2001 From: polarke Date: Sun, 11 Oct 2015 20:10:12 +0200 Subject: [PATCH 0521/1938] add test for sharedarray constructor --- test/parallel.jl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/parallel.jl b/test/parallel.jl index dc4a01a8e58c8..7c4e523b14fcf 100644 --- a/test/parallel.jl +++ b/test/parallel.jl @@ -179,6 +179,20 @@ end ### Utility functions +# construct PR #13514 +S = SharedArray{Int}((1,2,3)) +@test size(S) == (1,2,3) +@test typeof(S) <: SharedArray{Int} +S = SharedArray{Int}(2) +@test size(S) == (2,) +@test typeof(S) <: SharedArray{Int} +S = SharedArray{Int}(1,2) +@test size(S) == (1,2) +@test typeof(S) <: SharedArray{Int} +S = SharedArray{Int}(1,2,3) +@test size(S) == (1,2,3) +@test typeof(S) <: SharedArray{Int} + # reshape d = Base.shmem_fill(1.0, (10,10,10)) From 5206bcaf5cdedda465c15f6cce2a3aecaa1604c1 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sun, 11 Oct 2015 16:56:52 -0400 Subject: [PATCH 0522/1938] Ignore line number node when counting expressions for inlining. Fix #13551 --- base/inference.jl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index ed5516179bd6f..053235a6c6bf5 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1934,6 +1934,7 @@ end function occurs_more(e::ANY, pred, n) if isa(e,Expr) e = e::Expr + e.head === :line && return 0 c = 0 for a = e.args c += occurs_more(a, pred, n) @@ -2620,10 +2621,17 @@ function inline_worthy(body::Expr, cost::Integer=1000) # precondition: 0 < cost; return false end symlim = 1000 + 5_000_000 ÷ cost - if length(body.args) < (symlim + 500) ÷ 1000 + nargs = 0 + for arg in body.args + if (!isa(arg, LineNumberNode) && + !(isa(arg, Expr) && (arg::Expr).head === :line)) + nargs += 1 + end + end + if nargs < (symlim + 500) ÷ 1000 symlim *= 16 symlim ÷= 1000 - if occurs_more(body, e->true, symlim) < symlim + if occurs_more(body, e->(!isa(e, LineNumberNode)), symlim) < symlim return true end end @@ -2661,7 +2669,7 @@ end const corenumtype = Union{Int32,Int64,Float32,Float64} function inlining_pass(e::Expr, sv, ast) - if e.head == :method + if e.head === :method # avoid running the inlining pass on function definitions return (e,()) end From c6423bf7d95e3322ff139e3543648cfea991e770 Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Sat, 10 Oct 2015 19:28:36 -0400 Subject: [PATCH 0523/1938] Remove importall from sparse.jl --- base/sparse.jl | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/base/sparse.jl b/base/sparse.jl index b9e751abbe20b..92573e83a184d 100644 --- a/base/sparse.jl +++ b/base/sparse.jl @@ -4,14 +4,17 @@ module SparseMatrix using Base: Func, AddFun, OrFun, ConjFun, IdFun using Base.Sort: Forward -using Base.LinAlg: AbstractTriangular +using Base.LinAlg: AbstractTriangular, PosDefException -importall Base -importall ..Base.Operators -importall Base.LinAlg -import Base.promote_eltype -import Base.@get! -import Base.Broadcast.eltype_plus, Base.Broadcast.broadcast_shape +import Base: +, -, *, &, |, $, .+, .-, .*, ./, .\, .^, .<, .!=, == +import Base: A_mul_B!, Ac_mul_B, Ac_mul_B!, At_mul_B!, A_ldiv_B! +import Base: @get!, abs, abs2, broadcast, ceil, complex, cond, conj, convert, copy, + ctranspose, diagm, exp, expm1, factorize, find, findmax, findmin, findnz, float, + full, getindex, hcat, hvcat, imag, indmax, ishermitian, kron, length, log, log1p, + max, min, norm, one, promote_eltype, real, reinterpret, reshape, rot180, rotl90, + rotr90, round, scale, scale!, setindex!, similar, size, transpose, tril, triu, vcat, + vec +import Base.Broadcast: eltype_plus, broadcast_shape export AbstractSparseArray, AbstractSparseMatrix, AbstractSparseVector, SparseMatrixCSC, blkdiag, dense, droptol!, dropzeros!, etree, issparse, nnz, nonzeros, nzrange, From 59821c9ef7ec2dd3441c9c6479629a5e741b0262 Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Sat, 10 Oct 2015 19:55:22 -0400 Subject: [PATCH 0524/1938] Remove importall from datafmt.jl --- base/datafmt.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/base/datafmt.jl b/base/datafmt.jl index 61fce42fdc323..595ec572216c0 100644 --- a/base/datafmt.jl +++ b/base/datafmt.jl @@ -4,7 +4,6 @@ module DataFmt -importall Base import Base: _default_delims, tryparse_internal export countlines, readdlm, readcsv, writedlm, writecsv From b7278d1833f06dd64d491f61913b54b91b39e1f3 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 11 Oct 2015 23:03:53 -0400 Subject: [PATCH 0525/1938] add support for calling `@pure` functions with args that are potentially not effect free (some of the linalg code expects this) --- base/inference.jl | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 49a1115751781..04558cf4d443d 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -820,9 +820,6 @@ function isconstantargs(args, argtypes::Vector{Any}, sv::StaticVarInfo) for i = 1:length(args) arg = args[i] t = argtypes[i] - if !effect_free(arg, sv, false) # TODO: handle !effect_free args - return false - end if !isType(t) || has_typevars(t.parameters[1]) if isconstantref(arg, sv) === false return false @@ -860,10 +857,10 @@ function pure_eval_call(f, fargs, argtypes, sv, e) return false end - fargs = _ieval_args(fargs, argtypes, sv) + args = _ieval_args(fargs, argtypes, sv) tm = _topmod() if isgeneric(f) - atype = Tuple{Any[type_typeof(a) for a in fargs]...} + atype = Tuple{Any[type_typeof(a) for a in args]...} meth = _methods(f, atype, 1) if meth === false || length(meth) != 1 return false @@ -889,13 +886,25 @@ function pure_eval_call(f, fargs, argtypes, sv, e) local v try - v = f(fargs...) + v = f(args...) catch return false end - if isa(e, Expr) - e.head = :inert # eval_annotate will turn this into a QuoteNode - e.args = Any[v] + if isa(e, Expr) # replace Expr with a constant + stmts = Any[] # check if any arguments aren't effect_free and need to be kept around + for i = 1:length(fargs) + arg = fargs[i] + if !effect_free(arg, sv, false) + push!(stmts, arg) + end + end + if isempty(stmts) + e.head = :inert # eval_annotate will turn this into a QuoteNode + e.args = Any[v] + else + e.head = :call # should get cleaned up by tuple elimination + e.args = Any[top_getfield, Expr(:call, top_tuple, stmts..., v), length(stmts) + 1] + end end return type_typeof(v) end From 658a0a8b1533ccc5f7d82f5276c4217312d6f76a Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 12 Oct 2015 05:57:15 -0500 Subject: [PATCH 0526/1938] Bump MAX_TYPE_DEPTH again --- base/inference.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/inference.jl b/base/inference.jl index ed5516179bd6f..e160403565988 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2,7 +2,7 @@ # parameters limiting potentially-infinite types const MAX_TYPEUNION_LEN = 3 -const MAX_TYPE_DEPTH = 5 +const MAX_TYPE_DEPTH = 7 const MAX_TUPLETYPE_LEN = 8 const MAX_TUPLE_DEPTH = 4 From f393cd4512bb841b3045399251e184abe9747104 Mon Sep 17 00:00:00 2001 From: "Arch D. Robison" Date: Thu, 1 Oct 2015 11:21:35 -0500 Subject: [PATCH 0527/1938] Add developer documentation for calling conventions. --- doc/devdocs/callconv.rst | 60 ++++++++++++++++++++++++++++++++++++++++ doc/devdocs/julia.rst | 1 + 2 files changed, 61 insertions(+) create mode 100644 doc/devdocs/callconv.rst diff --git a/doc/devdocs/callconv.rst b/doc/devdocs/callconv.rst new file mode 100644 index 0000000000000..5f313a6db8e6e --- /dev/null +++ b/doc/devdocs/callconv.rst @@ -0,0 +1,60 @@ +.. _devdocs-callconv: + +******************* +Calling Conventions +******************* + +.. currentmodule:: Base + +Julia uses three calling conventions for four distinct purposes: + ++-------+--------------+----------------------------------+ +|Name |Prefix |Purpose | ++=======+==============+==================================+ +|Native |``julia_`` | Speed via specialized signatures | ++-------+--------------+----------------------------------+ +|JL Call|``jlcall_`` | Wrapper for generic calls | ++ +--------------+----------------------------------+ +| |``jl_`` | Builtins | ++-------+--------------+----------------------------------+ ++C ABI |``jlcapi_`` | Wrapper callable from C | ++-------+--------------+----------------------------------+ + +Julia Native Calling Convention +------------------------------- + +The native calling convention is designed for fast non-generic calls. +It usually uses a specialized signature. + +* LLVM ghosts (zero-length types) are omitted. +* LLVM scalars and vectors are passed by value. +* LLVM aggregates (arrays and structs) are passed by reference. + +A small return values is returned as LLVM return values. +A large return values is returned via the "structure return" (``sret``) +convention, where the caller provides a pointer to a return slot. + +An argument or return values thta is a homogeneous tuple is +sometimes represented as an LLVM vector instead of an LLVM array. + +JL Call Convention +------------------ + +The JL Call convention is for builtins and generic dispatch. +Hand-written functions using this convention are declared via the macro ``JL_CALLABLE``. +The convention uses exactly 3 parameters: + +* ``F`` - Julia representation of function that is being applied +* ``args`` - pointer to array of pointers to boxes +* ``nargs`` - length of the array + +The return value is a pointer to a box. + +C ABI +----- + +C ABI wrappers enable calling Julia from C. +The wrapper calls a function using the native calling convention. + +Tuples are always represented as C arrays. + diff --git a/doc/devdocs/julia.rst b/doc/devdocs/julia.rst index 7506b05f86b0c..538add920f2a5 100644 --- a/doc/devdocs/julia.rst +++ b/doc/devdocs/julia.rst @@ -14,6 +14,7 @@ ast types object + callconv cartesian meta subarrays From 3d132a92dbeedf14a0bdf600efd71af67640fb76 Mon Sep 17 00:00:00 2001 From: mbeltagy Date: Mon, 12 Oct 2015 19:32:34 +0200 Subject: [PATCH 0528/1938] hvcat doc example fixed hvcat doc example was not working I changed it to one that does. --- doc/stdlib/arrays.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index fd6e168492e3e..33b9c0a6ed2c9 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -396,7 +396,7 @@ Indexing, Assignment, and Concatenation .. Docstring generated from Julia source - Horizontal and vertical concatenation in one call. This function is called for block matrix syntax. The first argument specifies the number of arguments to concatenate in each block row. For example, ``[a b;c d e]`` calls ``hvcat((2,3),a,b,c,d,e)``\ . + Horizontal and vertical concatenation in one call. This function is called for block matrix syntax. The first argument specifies the number of arguments to concatenate in each block row. For example, ``[a b;c d]`` calls ``hvcat((2,2),a,b,c,d)``\ . If the first argument is a single integer ``n``\ , then all block rows are assumed to have ``n`` block columns. From 079258527e254f7e72343689a9695383d5bea415 Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Mon, 12 Oct 2015 13:48:02 -0400 Subject: [PATCH 0529/1938] Make complex sin and cos type stable --- base/complex.jl | 14 ++++++++++++++ test/complex.jl | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/base/complex.jl b/base/complex.jl index 99cc4d2cd29fd..f1cc868cf468a 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -551,6 +551,13 @@ function sin(z::Complex) if !isfinite(zr) && zi == 0 return Complex(oftype(zr, NaN), zi) end if !isfinite(zr) && isfinite(zi) return Complex(oftype(zr, NaN), oftype(zi, NaN)) end if !isfinite(zr) && !isfinite(zi) return Complex(zr, oftype(zi, NaN)) end + _sin(z) +end + +sin{T<:Integer}(z::Complex{T}) = _sin(z) + +function _sin(z::Complex) + zr, zi = reim(z) Complex(sin(zr)*cosh(zi), cos(zr)*sinh(zi)) end @@ -567,6 +574,13 @@ function cos(z::Complex) return Complex(oftype(zr, NaN), zi==0 ? -copysign(zi, zr) : oftype(zi, NaN)) end if isnan(zr) && zi==0 return Complex(zr, abs(zi)) end + _cos(z) +end + +cos{T<:Integer}(z::Complex{T}) = _cos(z) + +function _cos(z::Complex) + zr, zi = reim(z) Complex(cos(zr)*cosh(zi), -sin(zr)*sinh(zi)) end diff --git a/test/complex.jl b/test/complex.jl index fe3c1188001cb..de1aaeb506c80 100644 --- a/test/complex.jl +++ b/test/complex.jl @@ -908,3 +908,9 @@ end # issue #10926 @test typeof(π - 1im) == Complex{Float64} + +# issue #11839: type stability for Complex{Int64} +let x = 1+im + @inferred sin(x) + @inferred cos(x) +end From 12cc0d3d196f92a922d7a41838dd14d5eeb58988 Mon Sep 17 00:00:00 2001 From: Tracy Wadleigh Date: Sat, 22 Aug 2015 11:43:38 -0400 Subject: [PATCH 0530/1938] Make info & warn colors configurable via ENV --- base/client.jl | 12 ++++++++++++ base/util.jl | 4 ++-- doc/manual/interacting-with-julia.rst | 23 +++++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/base/client.jl b/base/client.jl index a83893ee32ccf..4e96efa26ae43 100644 --- a/base/client.jl +++ b/base/client.jl @@ -19,12 +19,24 @@ const text_colors = AnyDict( ) have_color = false +default_color_warn = :red +default_color_info = :blue @unix_only default_color_answer = text_colors[:bold] @unix_only default_color_input = text_colors[:bold] @windows_only default_color_answer = text_colors[:normal] @windows_only default_color_input = text_colors[:normal] color_normal = text_colors[:normal] +function warn_color() + c = symbol(get(ENV, "JULIA_WARN_COLOR", "")) + haskey(text_colors, c) ? c : default_color_warn +end + +function info_color() + c = symbol(get(ENV, "JULIA_INFO_COLOR", "")) + haskey(text_colors, c) ? c : default_color_info +end + function answer_color() c = symbol(get(ENV, "JULIA_ANSWER_COLOR", "")) return get(text_colors, c, default_color_answer) diff --git a/base/util.jl b/base/util.jl index 173d9044fa27c..3d3347d7f1cc4 100644 --- a/base/util.jl +++ b/base/util.jl @@ -329,7 +329,7 @@ println_with_color(color::Symbol, msg::AbstractString...) = ## warnings and messages ## function info(io::IO, msg...; prefix="INFO: ") - println_with_color(:blue, io, prefix, chomp(string(msg...))) + println_with_color(info_color(), io, prefix, chomp(string(msg...))) end info(msg...; prefix="INFO: ") = info(STDERR, msg..., prefix=prefix) @@ -351,7 +351,7 @@ function warn(io::IO, msg...; (key in have_warned) && return push!(have_warned, key) end - print_with_color(:red, io, prefix, str) + print_with_color(warn_color(), io, prefix, str) if bt !== nothing show_backtrace(io, bt) end diff --git a/doc/manual/interacting-with-julia.rst b/doc/manual/interacting-with-julia.rst index 95fd9fba82cc8..dc06060970fc2 100644 --- a/doc/manual/interacting-with-julia.rst +++ b/doc/manual/interacting-with-julia.rst @@ -229,3 +229,26 @@ and get a list of LaTeX matches as well:: \hbar \hermitconjmatrix \hkswarow \hookrightarrow \hspace A full list of tab-completions can be found in the :ref:`man-unicode-input` section of the manual. + + +Customizing Colors +~~~~~~~~~~~~~~~~~~ + +The colors used by Julia and the REPL can be customized, as well. To change the color of the Julia +prompt you can add something like the following to your ``juliarc.jl`` file:: + + Base.active_repl.prompt_color = Base.text_colors[:cyan] + +The available color keys in ``Base.text_colors`` are ``:black``, ``:red``, ``:green``, ``:yellow``, +``:blue``, ``:magenta``, ``:cyan``, ``:white``, ``:normal``, and ``:bold``. Similarly, you can +change the colors for the help and shell prompts and input and answer text by setting the +appropriate member of ``Base.active_repl`` (respectively, ``help_color``, ``shell_color``, +``input_color``, and ``answer_color``). For the latter two, be sure that the ``envcolors`` member +is also set to false. + +You can also customize the color used to render warning and informational messages by +setting the appropriate environment variable. For instance, to render warning messages in yellow and +informational messages in cyan you can add the following to your ``juliarc.jl`` file:: + + ENV["JULIA_WARN_COLOR"] = :yellow + ENV["JULIA_INFO_COLOR"] = :cyan From 0e498d32fa9d9c0829537250b72afd4303f68251 Mon Sep 17 00:00:00 2001 From: mbeltagy Date: Mon, 12 Oct 2015 23:01:03 +0200 Subject: [PATCH 0531/1938] Added doctest for hvcat Removed test for a non-working example and added clearer doctest example instead. --- base/docs/helpdb.jl | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 7421b8a7d4ce3..46d4e51dc95ff 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -3792,7 +3792,34 @@ utf8(s) doc""" hvcat(rows::Tuple{Vararg{Int}}, values...) -Horizontal and vertical concatenation in one call. This function is called for block matrix syntax. The first argument specifies the number of arguments to concatenate in each block row. For example, `[a b;c d e]` calls `hvcat((2,3),a,b,c,d,e)`. +Horizontal and vertical concatenation in one call. This function is called for block matrix syntax. The first argument specifies the number of arguments to concatenate in each block row. + +```jldoctest +julia> a, b, c, d, e, f = 1, 2, 3, 4, 5, 6 +(1,2,3,4,5,6) + +julia> [a b c; d e f] +2x3 Array{Int64,2}: + 1 2 3 + 4 5 6 + +julia> hvcat((3,3), a,b,c,d,e,f) +2x3 Array{Int64,2}: + 1 2 3 + 4 5 6 + +julia> [a b;c d; e f] +3x2 Array{Int64,2}: + 1 2 + 3 4 + 5 6 + +julia> hvcat((2,2,2), a,b,c,d,e,f) +3x2 Array{Int64,2}: + 1 2 + 3 4 + 5 6 +``` If the first argument is a single integer `n`, then all block rows are assumed to have `n` block columns. """ From d06e348399ea9b58edeae6f67d3a7bdf3fae0153 Mon Sep 17 00:00:00 2001 From: mbeltagy Date: Tue, 13 Oct 2015 02:39:34 +0200 Subject: [PATCH 0532/1938] fixing whitespaces and generating arrary.rst doc --- base/docs/helpdb.jl | 4 ++-- doc/stdlib/arrays.rst | 29 ++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 46d4e51dc95ff..8cccb985bd338 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -3802,12 +3802,12 @@ julia> [a b c; d e f] 2x3 Array{Int64,2}: 1 2 3 4 5 6 - + julia> hvcat((3,3), a,b,c,d,e,f) 2x3 Array{Int64,2}: 1 2 3 4 5 6 - + julia> [a b;c d; e f] 3x2 Array{Int64,2}: 1 2 diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 33b9c0a6ed2c9..af88e9342a287 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -396,7 +396,34 @@ Indexing, Assignment, and Concatenation .. Docstring generated from Julia source - Horizontal and vertical concatenation in one call. This function is called for block matrix syntax. The first argument specifies the number of arguments to concatenate in each block row. For example, ``[a b;c d]`` calls ``hvcat((2,2),a,b,c,d)``\ . + Horizontal and vertical concatenation in one call. This function is called for block matrix syntax. The first argument specifies the number of arguments to concatenate in each block row. + + .. doctest:: + + julia> a, b, c, d, e, f = 1, 2, 3, 4, 5, 6 + (1,2,3,4,5,6) + + julia> [a b c; d e f] + 2x3 Array{Int64,2}: + 1 2 3 + 4 5 6 + + julia> hvcat((3,3), a,b,c,d,e,f) + 2x3 Array{Int64,2}: + 1 2 3 + 4 5 6 + + julia> [a b;c d; e f] + 3x2 Array{Int64,2}: + 1 2 + 3 4 + 5 6 + + julia> hvcat((2,2,2), a,b,c,d,e,f) + 3x2 Array{Int64,2}: + 1 2 + 3 4 + 5 6 If the first argument is a single integer ``n``\ , then all block rows are assumed to have ``n`` block columns. From bdd23ea7939394f32a8b5c475ee48aaf1e9c6917 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Mon, 12 Oct 2015 20:19:17 -0400 Subject: [PATCH 0533/1938] Do not show the type of intrisics when it's `Any` Close #13568 --- base/show.jl | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/base/show.jl b/base/show.jl index 177ef7e7e43a0..68279acb7756d 100644 --- a/base/show.jl +++ b/base/show.jl @@ -342,6 +342,20 @@ is_quoted(ex::Expr) = is_expr(ex, :quote, 1) || is_expr(ex, :inert, 1) unquoted(ex::QuoteNode) = ex.value unquoted(ex::Expr) = ex.args[1] +function is_intrinsic_expr(x::ANY) + isa(x, IntrinsicFunction) && return true + if isa(x, GlobalRef) + x = x::GlobalRef + return (x.mod == Base && isdefined(Base, x.name) && + isa(getfield(Base, x.name), IntrinsicFunction)) + elseif isa(x, TopNode) + x = x::TopNode + return (isdefined(Base, x.name) && + isa(getfield(Base, x.name), IntrinsicFunction)) + end + return false +end + ## AST printing helpers ## const indent_width = 4 @@ -519,7 +533,7 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) head !== :row && print(io, cl) # function call - elseif head == :call && nargs >= 1 + elseif head === :call && nargs >= 1 func = args[1] fname = isa(func,GlobalRef) ? func.name : func func_prec = operator_precedence(fname) @@ -528,7 +542,9 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) end func_args = args[2:end] - if in(ex.args[1], (:box, TopNode(:box), :throw)) || ismodulecall(ex) + if (in(ex.args[1], (GlobalRef(Base, :box), TopNode(:box), :throw)) || + ismodulecall(ex) || + (ex.typ === Any && is_intrinsic_expr(ex.args[1]))) show_type = task_local_storage(:TYPEEMPHASIZE, false) end From 44280b37d945aad30e9620d4c070d8b7e2ac549d Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Mon, 12 Oct 2015 23:12:43 -0400 Subject: [PATCH 0534/1938] Fix a scoping bug in Pkg.Read Should make JuliaStats/StatsBase.jl#139 pass tests on Travis --- base/pkg/read.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/pkg/read.jl b/base/pkg/read.jl index 2cf17750d20a2..c24019cc4b928 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -77,8 +77,8 @@ function isfixed(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::Dict=availa else false end + res = true try - res = true for (ver,info) in avail if cache_has_head && LibGit2.iscommit(info.sha1, crepo) if LibGit2.is_ancestor_of(head, info.sha1, crepo) From b78f5608f49cc01d2ee37d1f73f17bd126068db8 Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Tue, 13 Oct 2015 00:06:36 -0400 Subject: [PATCH 0535/1938] Make require case-sensitive on all platforms --- base/loading.jl | 88 ++++++++++++++++++++++++++++++++++++++++++++++--- base/path.jl | 17 ++++++++++ test/loading.jl | 15 +++++++++ 3 files changed, 115 insertions(+), 5 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 0aeda4c3cac88..89584055f4a6e 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2,6 +2,84 @@ # Base.require is the implementation for the `import` statement +# Cross-platform case-sensitive path canonicalization + +if OS_NAME ∈ (:Linux, :FreeBSD) + # Case-sensitive filesystems, don't have to do anything + isfile_casesensitive(path) = isfile(path) +elseif OS_NAME == :Windows + # GetLongPathName Win32 function returns the case-preserved filename on NTFS. + function isfile_casesensitive(path) + isfile(path) || return false # Fail fast + longpath(path) == path + end +elseif OS_NAME == :Darwin + # HFS+ filesystem is case-preserving. The getattrlist API returns + # a case-preserved filename. In the rare event that HFS+ is operating + # in case-sensitive mode, this will still work but will be redundant. + + # Constants from + const ATRATTR_BIT_MAP_COUNT = 5 + const ATTR_CMN_NAME = 1 + const BITMAPCOUNT = 1 + const COMMONATTR = 5 + const FSOPT_NOFOLLOW = 1 # Don't follow symbolic links + + const attr_list = zeros(UInt8, 24) + attr_list[BITMAPCOUNT] = ATRATTR_BIT_MAP_COUNT + attr_list[COMMONATTR] = ATTR_CMN_NAME + + # This essentially corresponds to the following C code: + # attrlist attr_list; + # memset(&attr_list, 0, sizeof(attr_list)); + # attr_list.bitmapcount = ATTR_BIT_MAP_COUNT; + # attr_list.commonattr = ATTR_CMN_NAME; + # struct Buffer { + # u_int32_t total_length; + # u_int32_t filename_offset; + # u_int32_t filename_length; + # char filename[max_filename_length]; + # }; + # Buffer buf; + # getattrpath(path, &attr_list, &buf, sizeof(buf), FSOPT_NOFOLLOW); + function isfile_casesensitive(path) + isfile(path) || return false + path_basename = bytestring(basename(path)) + local casepreserved_basename + const header_size = 12 + buf = Array(UInt8, length(path_basename) + header_size + 1) + while true + ret = ccall(:getattrlist, Cint, + (Cstring, Ptr{Void}, Ptr{Void}, Csize_t, Culong), + path, attr_list, buf, sizeof(buf), FSOPT_NOFOLLOW) + systemerror(:getattrlist, ret ≠ 0) + filename_length = unsafe_load( + convert(Ptr{UInt32}, pointer(buf) + 8)) + if (filename_length + header_size) > length(buf) + resize!(buf, filename_length + header_size) + continue + end + casepreserved_basename = + sub(buf, (header_size+1):(header_size+filename_length-1)) + break + end + # Hack to compensate for inability to create a string from a subarray with no allocations. + path_basename.data == casepreserved_basename && return true + + # If there is no match, it's possible that the file does exist but HFS+ + # performed unicode normalization. See https://developer.apple.com/library/mac/qa/qa1235/_index.html. + isascii(path_basename) && return false + normalize_string(path_basename, :NFD).data == casepreserved_basename + end +else + # Generic fallback that performs a slow directory listing. + function isfile_casesensitive(path) + isfile(path) || return false + dir, filename = splitdir(path) + any(readdir(dir) .== filename) + end +end + # `wd` is a working directory to search. defaults to current working directory. # if `wd === nothing`, no extra path is searched. function find_in_path(name::AbstractString, wd = pwd()) @@ -13,15 +91,15 @@ function find_in_path(name::AbstractString, wd = pwd()) name = string(base,".jl") end if wd !== nothing - isfile(joinpath(wd,name)) && return joinpath(wd,name) + isfile_casesensitive(joinpath(wd,name)) && return joinpath(wd,name) end for prefix in [Pkg.dir(); LOAD_PATH] path = joinpath(prefix, name) - isfile(path) && return abspath(path) + isfile_casesensitive(path) && return abspath(path) path = joinpath(prefix, base, "src", name) - isfile(path) && return abspath(path) + isfile_casesensitive(path) && return abspath(path) path = joinpath(prefix, name, "src", name) - isfile(path) && return abspath(path) + isfile_casesensitive(path) && return abspath(path) end return nothing end @@ -47,7 +125,7 @@ function find_all_in_cache_path(mod::Symbol) paths = AbstractString[] for prefix in LOAD_CACHE_PATH path = joinpath(prefix, name*".ji") - if isfile(path) + if isfile_casesensitive(path) push!(paths, path) end end diff --git a/base/path.jl b/base/path.jl index 24f47a69be6e7..0c941fdc1c35d 100644 --- a/base/path.jl +++ b/base/path.jl @@ -125,6 +125,23 @@ abspath(a::AbstractString, b::AbstractString...) = abspath(joinpath(a,b...)) end end +@windows_only longpath(path::AbstractString) = longpath(utf16(path)) +@windows_only function longpath(path::UTF16String) + buf = Array(UInt16, length(path.data)) + while true + p = ccall((:GetLongPathNameW, "Kernel32"), stdcall, UInt32, + (Cwstring, Ptr{UInt16}, UInt32), + path, buf, length(buf)) + systemerror(:longpath, p == 0) + # Buffer wasn't big enough, in which case `p` is the necessary buffer size + if (p > length(buf)) + resize!(buf, p) + continue + end + return utf8(UTF16String(buf)) + end +end + @unix_only function realpath(path::AbstractString) p = ccall(:realpath, Ptr{UInt8}, (Cstring, Ptr{UInt8}), path, C_NULL) systemerror(:realpath, p == C_NULL) diff --git a/test/loading.jl b/test/loading.jl index 0e7f05aeeba76..773180084cf7b 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -10,3 +10,18 @@ thefname = "the fname!//\\&\0\1*" @test include_string("Base.source_path()", thefname) == Base.source_path() @test basename(@__FILE__) == "loading.jl" @test isabspath(@__FILE__) + +# Issue #5789 and PR #13542: +let true_filename = "cAsEtEsT.jl", lowered_filename="casetest.jl" + touch(true_filename) + @test Base.isfile_casesensitive(true_filename) + @test !Base.isfile_casesensitive(lowered_filename) + rm(true_filename) +end + +# Test Unicode normalization; pertinent for OS X +let nfc_name = "\U00F4.jl" + touch(nfc_name) + @test Base.isfile_casesensitive(nfc_name) + rm(nfc_name) +end From 7e10ef94e3542737b8486eab1dafc00f2801e8f5 Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Tue, 13 Oct 2015 00:51:30 -0400 Subject: [PATCH 0536/1938] Test version handling for untagged detached package heads --- test/pkg.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/pkg.jl b/test/pkg.jl index f0c56efc82d41..0850d4c14e8e8 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -112,6 +112,12 @@ temp_pkg_dir() do @test err.msg == "Example – 2147483647.0.0 is not a registered version" end + # PR #13572, handling of versions with untagged detached heads + LibGit2.with(LibGit2.GitRepo, Pkg.dir("Example")) do repo + LibGit2.checkout!(repo, "72f09c7d0099793378c645929a9961155faae6d2") + end + @test Pkg.installed()["Example"] > v"0.0.0" + Pkg.rm("Example") @test isempty(Pkg.installed()) end From 51b3dd5133a3a9f421dca7a6060815bfc8520826 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Tue, 13 Oct 2015 17:12:45 +0530 Subject: [PATCH 0537/1938] client.jl: refactor colors code to reduce duplication somewhat --- base/client.jl | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/base/client.jl b/base/client.jl index 4e96efa26ae43..8e1c626ebf0ff 100644 --- a/base/client.jl +++ b/base/client.jl @@ -21,31 +21,21 @@ const text_colors = AnyDict( have_color = false default_color_warn = :red default_color_info = :blue -@unix_only default_color_answer = text_colors[:bold] -@unix_only default_color_input = text_colors[:bold] -@windows_only default_color_answer = text_colors[:normal] -@windows_only default_color_input = text_colors[:normal] +@unix_only default_color_input = :bold +@unix_only default_color_answer = :bold +@windows_only default_color_input = :normal +@windows_only default_color_answer = :normal color_normal = text_colors[:normal] -function warn_color() - c = symbol(get(ENV, "JULIA_WARN_COLOR", "")) - haskey(text_colors, c) ? c : default_color_warn +function repl_color(key, default) + c = symbol(get(ENV, key, "")) + haskey(text_colors, c) ? c : default end -function info_color() - c = symbol(get(ENV, "JULIA_INFO_COLOR", "")) - haskey(text_colors, c) ? c : default_color_info -end - -function answer_color() - c = symbol(get(ENV, "JULIA_ANSWER_COLOR", "")) - return get(text_colors, c, default_color_answer) -end - -function input_color() - c = symbol(get(ENV, "JULIA_INPUT_COLOR", "")) - return get(text_colors, c, default_color_input) -end +warn_color() = repl_color("JULIA_WARN_COLOR", default_color_warn) +info_color() = repl_color("JULIA_INFO_COLOR", default_color_info) +input_color() = text_colors[repl_color("JULIA_INPUT_COLOR", default_color_input)] +answer_color() = text_colors[repl_color("JULIA_ANSWER_COLOR", default_color_answer)] exit(n) = ccall(:jl_exit, Void, (Int32,), n) exit() = exit(0) From 4e1a056265a90371aa1302343175f0f2c2c4f49f Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Tue, 13 Oct 2015 08:33:12 -0400 Subject: [PATCH 0538/1938] Clean up code_warntype tests, add test for not showing ::Any for intrinsics (#13571) --- test/reflection.jl | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/test/reflection.jl b/test/reflection.jl index f3a53e633025e..d37f0e16342f3 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -46,18 +46,19 @@ test_code_reflections(test_bin_reflection, code_native) module WarnType using Base.Test -iob = IOBuffer() +function warntype_hastag(f, types, tag) + iob = IOBuffer() + code_warntype(iob, f, types) + str = takebuf_string(iob) + !isempty(search(str, tag)) +end pos_stable(x) = x > 0 ? x : zero(x) pos_unstable(x) = x > 0 ? x : 0 tag = Base.have_color ? Base.text_colors[:red] : "UNION" -code_warntype(iob, pos_unstable, Tuple{Float64,}) -str = takebuf_string(iob) -@test !isempty(search(str, tag)) -code_warntype(iob, pos_stable, Tuple{Float64,}) -str = takebuf_string(iob) -@test isempty(search(str, tag)) +@test warntype_hastag(pos_unstable, Tuple{Float64}, tag) +@test !warntype_hastag(pos_stable, Tuple{Float64}, tag) type Stable{T,N} A::Array{T,N} @@ -69,15 +70,9 @@ Base.getindex(A::Stable, i) = A.A[i] Base.getindex(A::Unstable, i) = A.A[i] tag = Base.have_color ? Base.text_colors[:red] : "ARRAY{FLOAT64,N}" -code_warntype(iob, getindex, Tuple{Unstable{Float64},Int}) -str = takebuf_string(iob) -@test !isempty(search(str, tag)) -code_warntype(iob, getindex, Tuple{Stable{Float64,2},Int}) -str = takebuf_string(iob) -@test isempty(search(str, tag)) -code_warntype(iob, getindex, Tuple{Stable{Float64},Int}) -str = takebuf_string(iob) -@test !isempty(search(str, tag)) +@test warntype_hastag(getindex, Tuple{Unstable{Float64},Int}, tag) +@test !warntype_hastag(getindex, Tuple{Stable{Float64,2},Int}, tag) +@test warntype_hastag(getindex, Tuple{Stable{Float64},Int}, tag) function funfun(x) function internal(y) @@ -88,16 +83,21 @@ function funfun(x) end tag = Base.have_color ? string("2y",Base.text_colors[:red],"::Any") : "2y::ANY" -code_warntype(iob, funfun, Tuple{Float64}) -str = takebuf_string(iob) -@test !isempty(search(str, tag)) +@test warntype_hastag(funfun, Tuple{Float64}, tag) # Make sure emphasis is not used for other functions tag = Base.have_color ? Base.text_colors[:red] : "ANY" +iob = IOBuffer() show(iob, expand(:(x->x^2))) str = takebuf_string(iob) @test isempty(search(str, tag)) +# issue #13568 +@test !warntype_hastag(+, Tuple{Int,Int}, tag) +@test !warntype_hastag(-, Tuple{Int,Int}, tag) +@test !warntype_hastag(*, Tuple{Int,Int}, tag) +@test !warntype_hastag(/, Tuple{Int,Int}, tag) + end # isbits From 079eec89ded6b2a0288aaecf59e0d12e52828ee4 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 13 Oct 2015 08:47:14 -0500 Subject: [PATCH 0539/1938] Fix DateTime conversion issues with Int32 On 32-bit systems when using unix2datetime or julian2datetime and passing a Int32 will result in a DateTime in an extremely limited range: ``` julia> Dates.unix2datetime(div(typemin(Int32), Int32(1000))) 1969-12-07T03:28:37 julia> Dates.unix2datetime(div(typemax(Int32), Int32(1000))) 1970-01-25T20:31:23 ``` --- base/dates/conversions.jl | 4 ++-- test/dates/conversions.jl | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/base/dates/conversions.jl b/base/dates/conversions.jl index afc7a6e2e1d6c..c37e86837464b 100644 --- a/base/dates/conversions.jl +++ b/base/dates/conversions.jl @@ -14,7 +14,7 @@ Base.convert{R<:Real}(::Type{R},x::Date) = convert(R,value(x)) ### External Conversions const UNIXEPOCH = value(DateTime(1970)) #Rata Die milliseconds for 1970-01-01T00:00:00 function unix2datetime(x) - rata = UNIXEPOCH + round(Int64,1000*x) + rata = UNIXEPOCH + round(Int64, Int64(1000) * x) return DateTime(UTM(rata)) end # Returns unix seconds since 1970-01-01T00:00:00 @@ -32,7 +32,7 @@ datetime2rata(dt::DateTime) = days(dt) # Julian conversions const JULIANEPOCH = value(DateTime(-4713,11,24,12)) function julian2datetime(f) - rata = JULIANEPOCH + round(Int64,86400000*f) + rata = JULIANEPOCH + round(Int64, Int64(86400000) * f) return DateTime(UTM(rata)) end # Returns # of julian days since -4713-11-24T12:00:00 diff --git a/test/dates/conversions.jl b/test/dates/conversions.jl index 68136d9a1d8b4..9d749c8d94994 100644 --- a/test/dates/conversions.jl +++ b/test/dates/conversions.jl @@ -71,3 +71,12 @@ end @test Dates.Year(3) < Dates.Month(37) @test_throws InexactError convert(Dates.Year, Dates.Month(37)) @test_throws InexactError Dates.Month(Dates.Year(typemax(Int64))) + +# Ensure that conversion of 32-bit integers work +let dt = DateTime(1915,1,1,12) + unix = Int32(Dates.datetime2unix(dt)) + julian = Int32(Dates.datetime2julian(dt)) + + @test Dates.unix2datetime(unix) == dt + @test Dates.julian2datetime(julian) == dt +end From 980226393b35e7d4b1dbeef976195c8252c90a61 Mon Sep 17 00:00:00 2001 From: Dahua Lin Date: Fri, 2 Oct 2015 17:41:34 -0400 Subject: [PATCH 0540/1938] Import code from JuliaSparse/SparseVectors.jl --- base/sparse/sparsevector.jl | 1396 ++++++++++++++++++++++++++++++++ test/sparsedir/sparsevector.jl | 675 +++++++++++++++ 2 files changed, 2071 insertions(+) create mode 100644 base/sparse/sparsevector.jl create mode 100644 test/sparsedir/sparsevector.jl diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl new file mode 100644 index 0000000000000..b580271f605d6 --- /dev/null +++ b/base/sparse/sparsevector.jl @@ -0,0 +1,1396 @@ +### common.jl + +# not exported, used mainly for testing + +_copy_convert{T}(::Type{T}, x::Vector{T}) = copy(x) +_copy_convert{R,T}(::Type{R}, x::AbstractVector{T}) = convert(Vector{R}, x) + +import Base: Func, AddFun, MulFun, MaxFun, MinFun + +if isdefined(Base, :call) + import Base: call +else + call(f::Function, x) = f(x) + call(f::Function, x, y) = f(x, y) + call(f::Func{1}, x) = Base.evaluate(f, x) + call(f::Func{2}, x, y) = Base.evaluate(f, x, y) +end + +if isdefined(Base, :SubFun) + import Base: SubFun +else + immutable SubFun <: Func{2} end + call(::SubFun, x, y) = x - y +end + +immutable RealFun <: Func{1} end +call(::RealFun, x) = real(x) + +immutable ImagFun <: Func{1} end +call(::ImagFun, x) = imag(x) + +immutable ComplexFun <: Func{2} end +call(::ComplexFun, x::Real, y::Real) = complex(x, y) + +immutable DotFun <: Func{2} end +_dot(x::Number, y::Number) = conj(x) * y +_dot(x::Real, y::Real) = x * y +call(::DotFun, x::Number, y::Number) = _dot(x, y) + +typealias UnaryOp Union(Function, Func{1}) +typealias BinaryOp Union(Function, Func{2}) + + +### sparsevec.jl + +### Types + +immutable SparseVector{Tv,Ti<:Integer} <: AbstractSparseVector{Tv,Ti} + n::Int # the number of elements + nzind::Vector{Ti} # the indices of nonzeros + nzval::Vector{Tv} # the values of nonzeros + + function SparseVector(n::Integer, nzind::Vector{Ti}, nzval::Vector{Tv}) + n >= 0 || throw(ArgumentError("The number of elements must be non-negative.")) + length(nzind) == length(nzval) || + throw(DimensionMismatch("The lengths of nzind and nzval are inconsistent.")) + new(convert(Int, n), nzind, nzval) + end +end + +SparseVector{Tv,Ti}(n::Integer, nzind::Vector{Ti}, nzval::Vector{Tv}) = + SparseVector{Tv,Ti}(n, nzind, nzval) + +### Basic properties + +length(x::SparseVector) = x.n +size(x::SparseVector) = (x.n,) +nnz(x::SparseVector) = length(x.nzval) +countnz(x::SparseVector) = countnz(x.nzval) +nonzeros(x::SparseVector) = x.nzval +nonzeroinds(x::SparseVector) = x.nzind + +### Construct empty sparse vector + +sparsevector{T}(::Type{T}, len::Integer) = SparseVector(len, Int[], T[]) + +### Construction from lists of indices and values + +function _sparsevector!{Ti<:Integer}(I::Vector{Ti}, V::Vector, len::Integer) + # pre-condition: no duplicate indexes in I + if !isempty(I) + p = sortperm(I) + permute!(I, p) + permute!(V, p) + end + SparseVector(len, I, V) +end + +function _sparsevector!{Tv,Ti<:Integer}(I::Vector{Ti}, V::Vector{Tv}, len::Integer, combine::BinaryOp) + if !isempty(I) + p = sortperm(I) + permute!(I, p) + permute!(V, p) + m = length(I) + + # advances to the first non-zero element + r = 1 # index of last processed entry + while r <= m + if V[r] == zero(Tv) + r += 1 + else + break + end + end + r > m && SparseVector(len, Ti[], Tv[]) + + # move r-th to l-th + l = 1 # length of processed part + i = I[r] # row-index of current element + if r > 1 + I[l] = i; V[l] = V[r] + end + + # main loop + while r < m + r += 1 + i2 = I[r] + if i2 == i # accumulate r-th to the l-th entry + V[l] = call(combine, V[l], V[r]) + else # advance l, and move r-th to l-th + pv = V[l] + if pv != zero(Tv) + l += 1 + end + i = i2 + if l < r + I[l] = i; V[l] = V[r] + end + end + end + if V[l] == zero(Tv) + l -= 1 + end + if l < m + resize!(I, l) + resize!(V, l) + end + end + SparseVector(len, I, V) +end + +function sparsevector{Tv,Ti<:Integer}(I::AbstractVector{Ti}, V::AbstractVector{Tv}, combine::BinaryOp) + length(I) == length(V) || + throw(DimensionMismatch("The lengths of I and V are inconsistent.")) + len = 0 + for i in I + i >= 1 || error("Index must be positive.") + if i > len + len = i + end + end + _sparsevector!(_copy_convert(Ti, I), _copy_convert(Tv, V), len, combine) +end + +function sparsevector{Tv,Ti<:Integer}(I::AbstractVector{Ti}, V::AbstractVector{Tv}, len::Integer, combine::BinaryOp) + length(I) == length(V) || + throw(DimensionMismatch("The lengths of I and V are inconsistent.")) + maxi = convert(Ti, len) + for i in I + 1 <= i <= maxi || error("An index is out of bound.") + end + _sparsevector!(_copy_convert(Ti, I), _copy_convert(Tv, V), len, combine) +end + +sparsevector{Ti<:Integer}(I::AbstractVector{Ti}, V::AbstractVector) = + sparsevector(I, V, AddFun()) + +sparsevector{Ti<:Integer}(I::AbstractVector{Ti}, V::AbstractVector, len::Integer) = + sparsevector(I, V, len, AddFun()) + +sparsevector{Ti<:Integer}(I::AbstractVector{Ti}, v::Number, combine::BinaryOp) = + sparsevector(I, fill(v, length(I)), combine) + +sparsevector{Ti<:Integer}(I::AbstractVector{Ti}, v::Number, len::Integer, combine::BinaryOp) = + sparsevector(I, fill(v, length(I)), len, combine) + +sparsevector{Ti<:Integer}(I::AbstractVector{Ti}, v::Number) = + sparsevector(I, v, AddFun()) + +sparsevector{Ti<:Integer}(I::AbstractVector{Ti}, v::Number, len::Integer) = + sparsevector(I, v, len, AddFun()) + + +### Construction from dictionary + +function sparsevector{Tv,Ti<:Integer}(dict::Associative{Ti,Tv}) + m = length(dict) + nzind = Array(Ti, m) + nzval = Array(Tv, m) + + cnt = 0 + len = zero(Ti) + for (k, v) in dict + k >= 1 || error("Index must be positive.") + if k > len + len = k + end + if v != zero(v) + cnt += 1 + @inbounds nzind[cnt] = k + @inbounds nzval[cnt] = v + end + end + resize!(nzind, cnt) + resize!(nzval, cnt) + _sparsevector!(nzind, nzval, len) +end + +function sparsevector{Tv,Ti<:Integer}(dict::Associative{Ti,Tv}, len::Integer) + m = length(dict) + nzind = Array(Ti, m) + nzval = Array(Tv, m) + + cnt = 0 + maxk = convert(Ti, len) + for (k, v) in dict + 1 <= k <= maxk || error("An index (key) is out of bound.") + if v != zero(v) + cnt += 1 + @inbounds nzind[cnt] = k + @inbounds nzval[cnt] = v + end + end + resize!(nzind, cnt) + resize!(nzval, cnt) + _sparsevector!(nzind, nzval, len) +end + + +### Element access + +function setindex!{Tv,Ti<:Integer}(x::SparseVector{Tv,Ti}, v::Tv, i::Ti) + nzind = nonzeroinds(x) + nzval = nonzeros(x) + + m = length(nzind) + k = searchsortedfirst(nzind, i) + if 1 <= k <= m && nzind[k] == i # i found + if v == zero(v) + deleteat!(nzind, k) + deleteat!(nzval, k) + else + nzval[k] = v + end + else # i not found + if v != zero(v) + insert!(nzind, k, i) + insert!(nzval, k, v) + end + end + x +end + +setindex!{Tv, Ti<:Integer}(x::SparseVector{Tv,Ti}, v, i::Integer) = + setindex!(x, convert(Tv, v), convert(Ti, i)) + + +### Conversion + +# convert SparseMatrixCSC to SparseVector +function convert{Tv,Ti<:Integer}(::Type{SparseVector{Tv,Ti}}, s::SparseMatrixCSC{Tv,Ti}) + size(s, 2) == 1 || throw(ArgumentError("The input argument must have a single-column.")) + SparseVector(s.m, s.rowval, s.nzval) +end + +convert{Tv,Ti}(::Type{SparseVector{Tv}}, s::SparseMatrixCSC{Tv,Ti}) = + convert(SparseVector{Tv,Ti}, s) + +convert{Tv,Ti}(::Type{SparseVector}, s::SparseMatrixCSC{Tv,Ti}) = + convert(SparseVector{Tv,Ti}, s) + +# convert Vector to SparseVector + +function _dense2sparsevec{Tv}(s::Vector{Tv}, initcap::Int) + # pre-condition: initcap > 0 + n = length(s) + cap = initcap + nzind = Array(Int, cap) + nzval = Array(Tv, cap) + c = 0 + @inbounds for i = 1:n + v = s[i] + if v != zero(v) + if c >= cap + cap *= 2 + resize!(nzind, cap) + resize!(nzval, cap) + end + c += 1 + nzind[c] = i + nzval[c] = v + end + end + if c < cap + resize!(nzind, c) + resize!(nzval, c) + end + SparseVector(n, nzind, nzval) +end + +convert{Tv}(::Type{SparseVector{Tv,Int}}, s::Vector{Tv}) = + _dense2sparsevec(s, max(8, div(length(s), 8))) + +convert{Tv}(::Type{SparseVector{Tv}}, s::Vector{Tv}) = + convert(SparseVector{Tv,Int}, s) + +convert{Tv}(::Type{SparseVector}, s::Vector{Tv}) = + convert(SparseVector{Tv,Int}, s) + + +# convert between different types of SparseVector +convert{Tv,Ti,TvS,TiS}(::Type{SparseVector{Tv,Ti}}, s::SparseVector{TvS,TiS}) = + SparseVector{Tv,Ti}(s.n, convert(Vector{Ti}, s.nzind), convert(Vector{Tv}, s.nzval)) + +convert{Tv,TvS,TiS}(::Type{SparseVector{Tv}}, s::SparseVector{TvS,TiS}) = + SparseVector{Tv,TiS}(s.n, s.nzind, convert(Vector{Tv}, s.nzval)) + + +### Rand Construction + +function sprand{T}(n::Integer, p::FloatingPoint, rfn::Function, ::Type{T}) + I = randsubseq(1:convert(Int, n), p) + V = rfn(T, length(I)) + SparseVector(n, I, V) +end + +function sprand(n::Integer, p::FloatingPoint, rfn::Function) + I = randsubseq(1:convert(Int, n), p) + V = rfn(length(I)) + SparseVector(n, I, V) +end + +sprand{T}(n::Integer, p::FloatingPoint, ::Type{T}) = sprand(n, p, rand, T) +sprand(n::Integer, p::FloatingPoint) = sprand(n, p, rand) +sprandn(n::Integer, p::FloatingPoint) = sprand(n, p, randn) + + +### sparsevecview.jl + +using ArrayViews +import ArrayViews: view + +typealias CVecView{T} ContiguousView{T,1,Vector{T}} + +immutable SparseVectorView{Tv,Ti<:Integer} <: AbstractSparseVector{Tv,Ti} + n::Int # the number of elements + nzind::CVecView{Ti} # the indices of nonzeros + nzval::CVecView{Tv} # the values of nonzeros + + function SparseVectorView(n::Integer, nzind::CVecView{Ti}, nzval::CVecView{Tv}) + n >= 0 || throw(ArgumentError("The number of elements must be non-negative.")) + length(nzind) == length(nzval) || + throw(DimensionMismatch("The lengths of nzind and nzval are inconsistent.")) + new(convert(Int, n), nzind, nzval) + end +end + +### Construction + +SparseVectorView{Tv,Ti}(n::Integer, nzind::CVecView{Ti}, nzval::CVecView{Tv}) = + SparseVectorView{Tv,Ti}(n, nzind, nzval) + +view(x::AbstractSparseVector) = + SparseVectorView(length(x), view(nonzeroinds(x)), view(nonzeros(x))) + +### Basic properties + +length(x::SparseVectorView) = x.n +size(x::SparseVectorView) = (x.n,) +nnz(x::SparseVectorView) = length(x.nzval) +countnz(x::SparseVectorView) = countnz(x.nzval) +nonzeros(x::SparseVectorView) = x.nzval +nonzeroinds(x::SparseVectorView) = x.nzind + +### sparsematview.jl + + +### Views + +jvec_rgn(x::Vector, first::Int, n::Int) = pointer_to_array(pointer(x, first), n, false) + +as_jvec{T}(x::ContiguousView{T,1,Vector{T}}) = pointer_to_array(pointer(x), length(x), false) + +function view(x::SparseMatrixCSC, ::Colon, j::Integer) + 1 <= j <= x.n || throw(BoundsError()) + r1 = convert(Int, x.colptr[j]) + r2 = convert(Int, x.colptr[j+1]) - 1 + rgn = r1:r2 + SparseVectorView(x.m, view(x.rowval, rgn), view(x.nzval, rgn)) +end + +function getcol(x::SparseMatrixCSC, j::Integer) + 1 <= j <= x.n || throw(BoundsError()) + r1 = convert(Int, x.colptr[j]) + r2 = convert(Int, x.colptr[j+1]) - 1 + SparseVector(x.m, x.rowval[r1:r2], x.nzval[r1:r2]) +end + +function unsafe_colrange{Tv,Ti}(x::SparseMatrixCSC{Tv,Ti}, J::UnitRange) + jfirst = first(J) + jlast = last(J) + (1 <= jfirst <= x.n && jlast <= x.n) || throw(BoundsError()) + r1 = x.colptr[jfirst] + r2 = x.colptr[jlast+1] - one(r1) + newcolptr = view(x.colptr, jfirst:jlast+1) - (r1 - one(r1)) + + fi = convert(Int, r1) + nc = convert(Int, r2 - r1) + 1 + SparseMatrixCSC{Tv, Ti}(x.m, length(J), newcolptr, + jvec_rgn(x.rowval, fi, nc), jvec_rgn(x.nzval, fi, nc)) +end + +### generics.jl +# Generic functions operating on AbstractSparseVector + +### getindex + +function _spgetindex{Tv,Ti}(m::Int, nzind::AbstractVector{Ti}, nzval::AbstractVector{Tv}, i::Integer) + ii = searchsortedfirst(nzind, convert(Ti, i)) + (ii <= m && nzind[ii] == i) ? nzval[ii] : zero(Tv) +end + +getindex{Tv}(x::AbstractSparseVector{Tv}, i::Integer) = + _spgetindex(nnz(x), nonzeroinds(x), nonzeros(x), i) + +function getindex{Tv,Ti}(x::AbstractSparseVector{Tv,Ti}, I::UnitRange) + xlen = length(x) + i0 = first(I) + i1 = last(I) + (i0 >= 1 && i1 <= xlen) || throw(BoundsError()) + + xnzind = nonzeroinds(x) + xnzval = nonzeros(x) + m = length(xnzind) + + # locate the first j0, s.t. xnzind[j0] >= i0 + j0 = 1 + while j0 <= m && xnzind[j0] < i0 + j0 += 1 + end + # locate the last j1, s.t. xnzind[j1] <= i1 + j1 = j0 - 1 + while j1 < m && xnzind[j1+1] <= i1 + j1 += 1 + end + + # compute the number of non-zeros + jrgn = j0:j1 + mr = length(jrgn) + rind = Array(Ti, mr) + rval = Array(Tv, mr) + if mr > 0 + c = 0 + for j in jrgn + c += 1 + rind[c] = convert(Ti, xnzind[j] - i0 + 1) + rval[c] = xnzval[j] + end + end + SparseVector(length(I), rind, rval) +end + +function getindex{Tv,Ti}(x::AbstractSparseVector{Tv,Ti}, I::AbstractArray) + xnzind = nonzeroinds(x) + xnzval = nonzeros(x) + m = length(xnzind) + n = length(I) + nzind = Array(Ti, n) + nzval = Array(Tv, n) + c = 0 + for j = 1:n + v = _spgetindex(m, xnzind, xnzval, I[j]) + if v != zero(v) + c += 1 + nzind[c] = convert(Ti, j) + nzval[c] = v + end + end + if c < n + resize!(nzind, c) + resize!(nzval, c) + end + SparseVector(n, nzind, nzval) +end + +### show and friends + +function showarray(io::IO, x::AbstractSparseVector; + header::Bool=true, limit::Bool=Base._limit_output, + rows = Base.tty_size()[1], repr=false) + + n = length(x) + nzind = nonzeroinds(x) + nzval = nonzeros(x) + xnnz = length(nzind) + + if header + print(io, "Sparse vector, length = ", n, + ", with ", xnnz, " ", eltype(nzval), " entries:", "\n") + end + half_screen_rows = limit ? div(rows - 8, 2) : typemax(Int) + pad = ndigits(n) + k = 0 + sep = "\n\t" + for k = 1:length(nzind) + if k < half_screen_rows || k > xnnz - half_screen_rows + print(io, " ", '[', rpad(nzind[k], pad), "] = ") + showcompact(io, nzval[k]) + elseif k == half_screen_rows + print(io, sep, '\u22ee') + end + print(io, "\n") + k += 1 + end +end + +show(io::IO, x::AbstractSparseVector) = showarray(io, x) +writemime(io::IO, ::MIME"text/plain", x::AbstractSparseVector) = show(io, x) + + +### Comparison + +function exact_equal(x::AbstractSparseVector, y::AbstractSparseVector) + eltype(x) == eltype(y) && + eltype(nonzeroinds(x)) == eltype(nonzeroinds(y)) && + length(x) == length(y) && + nonzeroinds(x) == nonzeroinds(y) && + nonzeros(x) == nonzeros(y) +end + +### Conversion to matrix + +function convert{TvD,TiD,Tv,Ti}(::Type{SparseMatrixCSC{TvD,TiD}}, x::AbstractSparseVector{Tv,Ti}) + n = length(x) + xnzind = nonzeroinds(x) + xnzval = nonzeros(x) + m = length(xnzind) + colptr = TiD[1, m+1] + rowval = _copy_convert(TiD, xnzind) + nzval = _copy_convert(TvD, xnzval) + SparseMatrixCSC(n, 1, colptr, rowval, nzval) +end + +convert{TvD,Tv,Ti}(::Type{SparseMatrixCSC{TvD}}, x::AbstractSparseVector{Tv,Ti}) = + convert(SparseMatrixCSC{TvD,Ti}, x) + +convert{Tv,Ti}(::Type{SparseMatrixCSC}, x::AbstractSparseVector{Tv,Ti}) = + convert(SparseMatrixCSC{Tv,Ti}, x) + + +### Array manipulation + +function full{Tv}(x::AbstractSparseVector{Tv}) + n = length(x) + nzind = nonzeroinds(x) + nzval = nonzeros(x) + r = zeros(Tv, n) + for i = 1:length(nzind) + r[nzind[i]] = nzval[i] + end + return r +end + +vec(x::AbstractSparseVector) = x +copy(x::AbstractSparseVector) = + SparseVector(length(x), copy(nonzeroinds(x)), copy(nonzeros(x))) + +function reinterpret{T,Tv}(::Type{T}, x::AbstractSparseVector{Tv}) + sizeof(T) == sizeof(Tv) || + throw(ArgumentError("reinterpret of sparse vectors only supports element types of the same size.")) + SparseVector(length(x), copy(nonzeroinds(x)), reinterpret(T, nonzeros(x))) +end + +float{Tv<:FloatingPoint}(x::AbstractSparseVector{Tv}) = x +float(x::AbstractSparseVector) = + SparseVector(length(x), copy(nonzeroinds(x)), float(nonzeros(x))) + +complex{Tv<:Complex}(x::AbstractSparseVector{Tv}) = x +complex(x::AbstractSparseVector) = + SparseVector(length(x), copy(nonzeroinds(x)), complex(nonzeros(x))) + + +### Concatenation + +function hcat{Tv,Ti}(X::AbstractSparseVector{Tv,Ti}...) + # check sizes + n = length(X) + m = length(X[1]) + tnnz = nnz(X[1]) + for j = 2:n + length(X[j]) == m || + throw(DimensionMismatch("Inconsistent column lengths.")) + tnnz += nnz(X[j]) + end + + # construction + colptr = Array(Ti, n+1) + nzrow = Array(Ti, tnnz) + nzval = Array(Tv, tnnz) + roff = 1 + @inbounds for j = 1:n + xj = X[j] + xnzind = nonzeroinds(xj) + xnzval = nonzeros(xj) + colptr[j] = roff + copy!(nzrow, roff, xnzind) + copy!(nzval, roff, xnzval) + roff += length(xnzind) + end + colptr[n+1] = roff + SparseMatrixCSC{Tv,Ti}(m, n, colptr, nzrow, nzval) +end + +function vcat{Tv,Ti}(X::AbstractSparseVector{Tv,Ti}...) + # check sizes + n = length(X) + tnnz = 0 + for j = 1:n + tnnz += nnz(X[j]) + end + + # construction + rnzind = Array(Ti, tnnz) + rnzval = Array(Tv, tnnz) + ir = 0 + len = 0 + @inbounds for j = 1:n + xj = X[j] + xnzind = nonzeroinds(xj) + xnzval = nonzeros(xj) + xnnz = length(xnzind) + for i = 1:xnnz + rnzind[ir + i] = xnzind[i] + len + end + copy!(rnzval, ir+1, xnzval) + ir += xnnz + len += length(xj) + end + SparseVector(len, rnzind, rnzval) +end + +### math.jl + + +### Unary Map + +# zero-preserving functions (z->z, nz->nz) +for op in [:-, :abs, :abs2, :conj] + @eval begin + $(op)(x::AbstractSparseVector) = + SparseVector(length(x), copy(nonzeroinds(x)), $(op)(nonzeros(x))) + end +end + +# functions f, such that +# f(x) can be zero or non-zero when x != 0 +# f(x) = 0 when x == 0 +# +macro unarymap_nz2z_z2z(op, TF) + esc(quote + function $(op){Tv<:$(TF),Ti<:Integer}(x::AbstractSparseVector{Tv,Ti}) + R = typeof($(op)(zero(Tv))) + xnzind = nonzeroinds(x) + xnzval = nonzeros(x) + m = length(xnzind) + + ynzind = Array(Ti, m) + ynzval = Array(R, m) + ir = 0 + @inbounds for j = 1:m + i = xnzind[j] + v = $(op)(xnzval[j]) + if v != zero(v) + ir += 1 + ynzind[ir] = i + ynzval[ir] = v + end + end + resize!(ynzind, ir) + resize!(ynzval, ir) + SparseVector(length(x), ynzind, ynzval) + end + end) +end + +real{T<:Real}(x::AbstractSparseVector{T}) = x +@unarymap_nz2z_z2z real Complex + +imag{Tv<:Real,Ti<:Integer}(x::AbstractSparseVector{Tv,Ti}) = SparseVector(length(x), Ti[], Tv[]) +@unarymap_nz2z_z2z imag Complex + +for op in [:floor, :ceil, :trunc, :round] + @eval @unarymap_nz2z_z2z $(op) Real +end + +for op in [:log1p, :expm1, + :sin, :tan, :sinpi, :sind, :tand, + :asin, :atan, :asind, :atand, + :sinh, :tanh, :asinh, :atanh] + @eval @unarymap_nz2z_z2z $(op) Number +end + +# function that does not preserve zeros + +macro unarymap_z2nz(op, TF) + esc(quote + function $(op){Tv<:$(TF),Ti<:Integer}(x::AbstractSparseVector{Tv,Ti}) + v0 = $(op)(zero(Tv)) + R = typeof(v0) + xnzind = nonzeroinds(x) + xnzval = nonzeros(x) + n = length(x) + m = length(xnzind) + y = fill(v0, n) + @inbounds for j = 1:m + y[xnzind[j]] = $(op)(xnzval[j]) + end + y + end + end) +end + +for op in [:exp, :exp2, :exp10, :log, :log2, :log10, + :cos, :csc, :cot, :sec, :cospi, + :cosd, :cscd, :cotd, :secd, + :acos, :acot, :acosd, :acotd, + :cosh, :csch, :coth, :sech, + :acsch, :asech] + @eval @unarymap_z2nz $(op) Number +end + + +### Binary Map + +# mode: +# 0: f(nz, nz) -> nz, f(z, nz) -> z, f(nz, z) -> z +# 1: f(nz, nz) -> z/nz, f(z, nz) -> nz, f(nz, z) -> nz +# 2: f(nz, nz) -> z/nz, f(z, nz) -> z/nz, f(nz, z) -> z/nz + +function _binarymap{Tx,Ty}(f::BinaryOp, + x::AbstractSparseVector{Tx}, + y::AbstractSparseVector{Ty}, + mode::Int) + + 0 <= mode <= 2 || throw(ArgumentError("Incorrect mode $mode.")) + R = typeof(call(f, zero(Tx), zero(Ty))) + n = length(x) + length(y) == n || throw(DimensionMismatch()) + + xnzind = nonzeroinds(x) + xnzval = nonzeros(x) + ynzind = nonzeroinds(y) + ynzval = nonzeros(y) + mx = length(xnzind) + my = length(ynzind) + (mx == 0 || my == 0) && return SparseVector(R, 0) + cap = (mode == 0 ? min(mx, my) : mx + my)::Int + + rind = Array(Int, cap) + rval = Array(R, cap) + ir = 0 + ix = 1 + iy = 1 + + ir = ( + mode == 0 ? _binarymap_mode_0!(f, mx, my, + xnzind, xnzval, ynzind, ynzval, rind, rval) : + mode == 1 ? _binarymap_mode_1!(f, mx, my, + xnzind, xnzval, ynzind, ynzval, rind, rval) : + _binarymap_mode_2!(f, mx, my, + xnzind, xnzval, ynzind, ynzval, rind, rval) + )::Int + + resize!(rind, ir) + resize!(rval, ir) + return SparseVector(n, rind, rval) +end + +function _binarymap_mode_0!(f::BinaryOp, mx::Int, my::Int, + xnzind, xnzval, ynzind, ynzval, rind, rval) + # f(nz, nz) -> nz, f(z, nz) -> z, f(nz, z) -> z + ir = 0; ix = 1; iy = 1 + @inbounds while ix <= mx && iy <= my + jx = xnzind[ix] + jy = ynzind[iy] + if jx == jy + v = call(f, xnzval[ix], ynzval[iy]) + ir += 1; rind[ir] = jx; rval[ir] = v + ix += 1; iy += 1 + elseif jx < jy + ix += 1 + else + iy += 1 + end + end + return ir +end + +function _binarymap_mode_1!{Tx,Ty}(f::BinaryOp, mx::Int, my::Int, + xnzind, xnzval::AbstractVector{Tx}, + ynzind, ynzval::AbstractVector{Ty}, + rind, rval) + # f(nz, nz) -> z/nz, f(z, nz) -> nz, f(nz, z) -> nz + ir = 0; ix = 1; iy = 1 + @inbounds while ix <= mx && iy <= my + jx = xnzind[ix] + jy = ynzind[iy] + if jx == jy + v = call(f, xnzval[ix], ynzval[iy]) + if v != zero(v) + ir += 1; rind[ir] = jx; rval[ir] = v + end + ix += 1; iy += 1 + elseif jx < jy + v = call(f, xnzval[ix], zero(Ty)) + ir += 1; rind[ir] = jx; rval[ir] = v + ix += 1 + else + v = call(f, zero(Tx), ynzval[iy]) + ir += 1; rind[ir] = jy; rval[ir] = v + iy += 1 + end + end + @inbounds while ix <= mx + v = call(f, xnzval[ix], zero(Ty)) + ir += 1; rind[ir] = xnzind[ix]; rval[ir] = v + ix += 1 + end + @inbounds while iy <= my + v = call(f, zero(Tx), ynzval[iy]) + ir += 1; rind[ir] = ynzind[iy]; rval[ir] = v + iy += 1 + end + return ir +end + +function _binarymap_mode_2!{Tx,Ty}(f::BinaryOp, mx::Int, my::Int, + xnzind, xnzval::AbstractVector{Tx}, + ynzind, ynzval::AbstractVector{Ty}, + rind, rval) + # f(nz, nz) -> z/nz, f(z, nz) -> z/nz, f(nz, z) -> z/nz + ir = 0; ix = 1; iy = 1 + @inbounds while ix <= mx && iy <= my + jx = xnzind[ix] + jy = ynzind[iy] + if jx == jy + v = call(f, xnzval[ix], ynzval[iy]) + if v != zero(v) + ir += 1; rind[ir] = jx; rval[ir] = v + end + ix += 1; iy += 1 + elseif jx < jy + v = call(f, xnzval[ix], zero(Ty)) + if v != zero(v) + ir += 1; rind[ir] = jx; rval[ir] = v + end + ix += 1 + else + v = call(f, zero(Tx), ynzval[iy]) + if v != zero(v) + ir += 1; rind[ir] = jy; rval[ir] = v + end + iy += 1 + end + end + @inbounds while ix <= mx + v = call(f, xnzval[ix], zero(Ty)) + if v != zero(v) + ir += 1; rind[ir] = xnzind[ix]; rval[ir] = v + end + ix += 1 + end + @inbounds while iy <= my + v = call(f, zero(Tx), ynzval[iy]) + if v != zero(v) + ir += 1; rind[ir] = ynzind[iy]; rval[ir] = v + end + iy += 1 + end + return ir +end + +function _binarymap{Tx,Ty}(f::BinaryOp, + x::AbstractVector{Tx}, + y::AbstractSparseVector{Ty}, + mode::Int) + + 0 <= mode <= 2 || throw(ArgumentError("Incorrect mode $mode.")) + R = typeof(call(f, zero(Tx), zero(Ty))) + n = length(x) + length(y) == n || throw(DimensionMismatch()) + + ynzind = nonzeroinds(y) + ynzval = nonzeros(y) + m = length(ynzind) + + dst = Array(R, n) + if mode == 0 + ii = 1 + @inbounds for i = 1:m + j = ynzind[i] + while ii < j + dst[ii] = zero(R); ii += 1 + end + dst[j] = call(f, x[j], ynzval[i]); ii += 1 + end + @inbounds while ii <= n + dst[ii] = zero(R); ii += 1 + end + else # mode >= 1 + ii = 1 + @inbounds for i = 1:m + j = ynzind[i] + while ii < j + dst[ii] = call(f, x[ii], zero(Ty)); ii += 1 + end + dst[j] = call(f, x[j], ynzval[i]); ii += 1 + end + @inbounds while ii <= n + dst[ii] = call(f, x[ii], zero(Ty)); ii += 1 + end + end + return dst +end + +function _binarymap{Tx,Ty}(f::BinaryOp, + x::AbstractSparseVector{Tx}, + y::AbstractVector{Ty}, + mode::Int) + + 0 <= mode <= 2 || throw(ArgumentError("Incorrect mode $mode.")) + R = typeof(call(f, zero(Tx), zero(Ty))) + n = length(x) + length(y) == n || throw(DimensionMismatch()) + + xnzind = nonzeroinds(x) + xnzval = nonzeros(x) + m = length(xnzind) + + dst = Array(R, n) + if mode == 0 + ii = 1 + @inbounds for i = 1:m + j = xnzind[i] + while ii < j + dst[ii] = zero(R); ii += 1 + end + dst[j] = call(f, xnzval[i], y[j]); ii += 1 + end + @inbounds while ii <= n + dst[ii] = zero(R); ii += 1 + end + else # mode >= 1 + ii = 1 + @inbounds for i = 1:m + j = xnzind[i] + while ii < j + dst[ii] = call(f, zero(Tx), y[ii]); ii += 1 + end + dst[j] = call(f, xnzval[i], y[j]); ii += 1 + end + @inbounds while ii <= n + dst[ii] = call(f, zero(Tx), y[ii]); ii += 1 + end + end + return dst +end + + +### Binary arithmetics: +, -, * + +for (vop, fun, mode) in [(:_vadd, :AddFun, 1), + (:_vsub, :SubFun, 1), + (:_vmul, :MulFun, 0)] + @eval begin + $(vop)(x::AbstractSparseVector, y::AbstractSparseVector) = _binarymap($(fun)(), x, y, $mode) + $(vop)(x::StridedVector, y::AbstractSparseVector) = _binarymap($(fun)(), x, y, $mode) + $(vop)(x::AbstractSparseVector, y::StridedVector) = _binarymap($(fun)(), x, y, $mode) + end +end + +# to workaround the ambiguities with vectorized dates/arithmetic.jl functions +if VERSION > v"0.4-dev" + +{T<:Dates.TimeType,P<:Dates.GeneralPeriod}(x::StridedVector{P}, y::AbstractSparseVector{T}) = _vadd(x, y) + -{T<:Dates.TimeType,P<:Dates.GeneralPeriod}(x::StridedVector{P}, y::AbstractSparseVector{T}) = _vsub(x, y) + +{T<:Dates.TimeType,P<:Dates.GeneralPeriod}(x::AbstractSparseVector{T}, y::StridedVector{P}) = _vadd(x, y) + -{T<:Dates.TimeType,P<:Dates.GeneralPeriod}(x::AbstractSparseVector{T}, y::StridedVector{P}) = _vsub(x, y) +end + +# to workaround the ambiguities with BitVector +.*(x::BitVector, y::AbstractSparseVector{Bool}) = _vmul(x, y) +.*(x::AbstractSparseVector{Bool}, y::BitVector) = _vmul(x, y) + +# definition of operators + +for (op, vop) in [(:+, :_vadd), (:(.+), :_vadd), + (:-, :_vsub), (:(.-), :_vsub), + (:.*, :_vmul)] + @eval begin + $(op)(x::AbstractSparseVector, y::AbstractSparseVector) = $(vop)(x, y) + $(op)(x::StridedVector, y::AbstractSparseVector) = $(vop)(x, y) + $(op)(x::AbstractSparseVector, y::StridedVector) = $(vop)(x, y) + end +end + +# definition of other binary functions + +for (op, fun, TF, mode) in [(:max, :MaxFun, :Real, 2), + (:min, :MinFun, :Real, 2), + (:complex, :ComplexFun, :Real, 1)] + @eval begin + $(op){Tx<:$(TF),Ty<:$(TF)}(x::AbstractSparseVector{Tx}, y::AbstractSparseVector{Ty}) = + _binarymap($(fun)(), x, y, $mode) + $(op){Tx<:$(TF),Ty<:$(TF)}(x::StridedVector{Tx}, y::AbstractSparseVector{Ty}) = + _binarymap($(fun)(), x, y, $mode) + $(op){Tx<:$(TF),Ty<:$(TF)}(x::AbstractSparseVector{Tx}, y::StridedVector{Ty}) = + _binarymap($(fun)(), x, y, $mode) + end +end + +### Reduction + +sum(x::AbstractSparseVector) = sum(nonzeros(x)) +sumabs(x::AbstractSparseVector) = sumabs(nonzeros(x)) +sumabs2(x::AbstractSparseVector) = sumabs2(nonzeros(x)) + +function maximum{T<:Real}(x::AbstractSparseVector{T}) + n = length(x) + n > 0 || throw(ArgumentError("maximum over empty array is not allowed.")) + m = nnz(x) + (m == 0 ? zero(T) : + m == n ? maximum(nonzeros(x)) : + max(zero(T), maximum(nonzeros(x))))::T +end + +function minimum{T<:Real}(x::AbstractSparseVector{T}) + n = length(x) + n > 0 || throw(ArgumentError("minimum over empty array is not allowed.")) + m = nnz(x) + (m == 0 ? zero(T) : + m == n ? minimum(nonzeros(x)) : + min(zero(T), minimum(nonzeros(x))))::T +end + +maxabs{T<:Number}(x::AbstractSparseVector{T}) = maxabs(nonzeros(x)) +minabs{T<:Number}(x::AbstractSparseVector{T}) = nnz(x) < length(x) ? abs(zero(T)) : minabs(nonzeros(x)) + +vecnorm(x::AbstractSparseVector, p::Real=2) = vecnorm(nonzeros(x), p) + +### linalg.jl + + +### BLAS Level-1 + +# axpy + +function LinAlg.axpy!(a::Number, x::AbstractSparseVector, y::StridedVector) + length(x) == length(y) || throw(DimensionMismatch()) + nzind = nonzeroinds(x) + nzval = nonzeros(x) + m = length(nzind) + + if a == one(a) + for i = 1:m + @inbounds ii = nzind[i] + @inbounds v = nzval[i] + y[ii] += v + end + elseif a == -one(a) + for i = 1:m + @inbounds ii = nzind[i] + @inbounds v = nzval[i] + y[ii] -= v + end + else + for i = 1:m + @inbounds ii = nzind[i] + @inbounds v = nzval[i] + y[ii] += a * v + end + end + return y +end + + +# scale + +scale!(x::AbstractSparseVector, a::Real) = (scale!(nonzeros(x), a); x) +scale!(x::AbstractSparseVector, a::Complex) = (scale!(nonzeros(x), a); x) +scale!(a::Real, x::AbstractSparseVector) = scale!(nonzeros(x), a) +scale!(a::Complex, x::AbstractSparseVector) = scale!(nonzeros(x), a) + +scale(x::AbstractSparseVector, a::Real) = + SparseVector(length(x), copy(nonzeroinds(x)), scale(nonzeros(x), a)) +scale(x::AbstractSparseVector, a::Complex) = + SparseVector(length(x), copy(nonzeroinds(x)), scale(nonzeros(x), a)) + +scale(a::Real, x::AbstractSparseVector) = scale(x, a) +scale(a::Complex, x::AbstractSparseVector) = scale(x, a) + +*(x::AbstractSparseVector, a::Number) = scale(x, a) +*(a::Number, x::AbstractSparseVector) = scale(x, a) +.*(x::AbstractSparseVector, a::Number) = scale(x, a) +.*(a::Number, x::AbstractSparseVector) = scale(x, a) + + +# dot + +function dot{Tx<:Number,Ty<:Number}(x::StridedVector{Tx}, y::AbstractSparseVector{Ty}) + n = length(x) + length(y) == n || throw(DimensionMismatch()) + nzind = nonzeroinds(y) + nzval = nonzeros(y) + s = zero(Tx) * zero(Ty) + for i = 1:length(nzind) + s += _dot(x[nzind[i]], nzval[i]) + end + return s +end + +function dot{Tx<:Number,Ty<:Number}(x::AbstractSparseVector{Tx}, y::AbstractVector{Ty}) + n = length(y) + length(x) == n || throw(DimensionMismatch()) + nzind = nonzeroinds(x) + nzval = nonzeros(x) + s = zero(Tx) * zero(Ty) + for i = 1:length(nzind) + s += _dot(nzval[i], y[nzind[i]]) + end + return s +end + +function _spdot(f::BinaryOp, + xj::Int, xj_last::Int, xnzind, xnzval, + yj::Int, yj_last::Int, ynzind, ynzval) + # dot product between ranges of non-zeros, + s = zero(eltype(xnzval)) * zero(eltype(ynzval)) + @inbounds while xj <= xj_last && yj <= yj_last + ix = xnzind[xj] + iy = ynzind[yj] + if ix == iy + s += call(f, xnzval[xj], ynzval[yj]) + xj += 1 + yj += 1 + elseif ix < iy + xj += 1 + else + yj += 1 + end + end + s +end + +function dot{Tx<:Number,Ty<:Number}(x::AbstractSparseVector{Tx}, y::AbstractSparseVector{Ty}) + is(x, y) && return sumabs2(x) + n = length(x) + length(y) == n || throw(DimensionMismatch()) + + xnzind = nonzeroinds(x) + ynzind = nonzeroinds(y) + xnzval = nonzeros(x) + ynzval = nonzeros(y) + + _spdot(DotFun(), + 1, length(xnzind), xnzind, xnzval, + 1, length(ynzind), ynzind, ynzval) +end + + +### BLAS-2 / dense A * sparse x -> dense y + +# A_mul_B + +function *{Ta,Tx}(A::StridedMatrix{Ta}, x::AbstractSparseVector{Tx}) + m, n = size(A) + length(x) == n || throw(DimensionMismatch()) + Ty = promote_type(Ta, Tx) + y = Array(Ty, m) + A_mul_B!(y, A, x) +end + +A_mul_B!{Tx,Ty}(y::StridedVector{Ty}, A::StridedMatrix, x::AbstractSparseVector{Tx}) = + A_mul_B!(one(Tx), A, x, zero(Ty), y) + +function A_mul_B!(α::Number, A::StridedMatrix, x::AbstractSparseVector, β::Number, y::StridedVector) + m, n = size(A) + length(x) == n && length(y) == m || throw(DimensionMismatch()) + m == 0 && return y + if β != one(β) + β == zero(β) ? fill!(y, zero(eltype(y))) : scale!(y, β) + end + α == zero(α) && return y + + xnzind = nonzeroinds(x) + xnzval = nonzeros(x) + @inbounds for i = 1:length(xnzind) + v = xnzval[i] + if v != zero(v) + j = xnzind[i] + αv = v * α + for r = 1:m + y[r] += A[r,j] * αv + end + end + end + return y +end + +# At_mul_B + +function At_mul_B{Ta,Tx}(A::StridedMatrix{Ta}, x::AbstractSparseVector{Tx}) + m, n = size(A) + length(x) == m || throw(DimensionMismatch()) + Ty = promote_type(Ta, Tx) + y = Array(Ty, n) + At_mul_B!(y, A, x) +end + +At_mul_B!{Tx,Ty}(y::StridedVector{Ty}, A::StridedMatrix, x::AbstractSparseVector{Tx}) = + At_mul_B!(one(Tx), A, x, zero(Ty), y) + +function At_mul_B!(α::Number, A::StridedMatrix, x::AbstractSparseVector, β::Number, y::StridedVector) + m, n = size(A) + length(x) == m && length(y) == n || throw(DimensionMismatch()) + n == 0 && return y + if β != one(β) + β == zero(β) ? fill!(y, zero(eltype(y))) : scale!(y, β) + end + α == zero(α) && return y + + xnzind = nonzeroinds(x) + xnzval = nonzeros(x) + _nnz = length(xnzind) + _nnz == 0 && return y + + s0 = zero(eltype(A)) * zero(eltype(x)) + @inbounds for j = 1:n + s = zero(s0) + for i = 1:_nnz + s += A[xnzind[i], j] * xnzval[i] + end + y[j] += s * α + end + return y +end + + +### BLAS-2 / sparse A * sparse x -> dense y + +function densemv(A::SparseMatrixCSC, x::AbstractSparseVector; trans::Char='N') + xlen::Int + ylen::Int + m, n = size(A) + if trans == 'N' || trans == 'n' + xlen = n; ylen = m + elseif trans == 'T' || trans == 't' || trans == 'C' || trans == 'c' + xlen = m; ylen = n + else + throw(ArgumentError("Invalid trans character $trans")) + end + xlen == length(x) || throw(DimensionMismatch()) + T = promote_type(eltype(A), eltype(x)) + y = Array(T, ylen) + if trans == 'N' || trans == 'N' + A_mul_B!(y, A, x) + elseif trans == 'T' || trans == 't' + At_mul_B!(y, A, x) + elseif trans == 'C' || trans == 'c' + Ac_mul_B!(y, A, x) + else + throw(ArgumentError("Invalid trans character $trans")) + end + y +end + +# A_mul_B + +A_mul_B!{Tx,Ty}(y::StridedVector{Ty}, A::SparseMatrixCSC, x::AbstractSparseVector{Tx}) = + A_mul_B!(one(Tx), A, x, zero(Ty), y) + +function A_mul_B!(α::Number, A::SparseMatrixCSC, x::AbstractSparseVector, β::Number, y::StridedVector) + m, n = size(A) + length(x) == n && length(y) == m || throw(DimensionMismatch()) + m == 0 && return y + if β != one(β) + β == zero(β) ? fill!(y, zero(eltype(y))) : scale!(y, β) + end + α == zero(α) && return y + + xnzind = nonzeroinds(x) + xnzval = nonzeros(x) + Acolptr = A.colptr + Arowval = A.rowval + Anzval = A.nzval + + @inbounds for i = 1:length(xnzind) + v = xnzval[i] + if v != zero(v) + αv = v * α + j = xnzind[i] + for r = A.colptr[j]:(Acolptr[j+1]-1) + y[Arowval[r]] += Anzval[r] * αv + end + end + end + return y +end + +# At_mul_B + +At_mul_B!{Tx,Ty}(y::StridedVector{Ty}, A::SparseMatrixCSC, x::AbstractSparseVector{Tx}) = + At_mul_B!(one(Tx), A, x, zero(Ty), y) + +At_mul_B!{Tx,Ty}(α::Number, A::SparseMatrixCSC, x::AbstractSparseVector{Tx}, β::Number, y::StridedVector{Ty}) = + _At_or_Ac_mul_B!(MulFun(), α, A, x, β, y) + +Ac_mul_B!{Tx,Ty}(y::StridedVector{Ty}, A::SparseMatrixCSC, x::AbstractSparseVector{Tx}) = + Ac_mul_B!(one(Tx), A, x, zero(Ty), y) + +Ac_mul_B!{Tx,Ty}(α::Number, A::SparseMatrixCSC, x::AbstractSparseVector{Tx}, β::Number, y::StridedVector{Ty}) = + _At_or_Ac_mul_B!(DotFun(), α, A, x, β, y) + +function _At_or_Ac_mul_B!{Tx,Ty}(tfun::BinaryOp, + α::Number, A::SparseMatrixCSC, x::AbstractSparseVector{Tx}, + β::Number, y::StridedVector{Ty}) + m, n = size(A) + length(x) == m && length(y) == n || throw(DimensionMismatch()) + n == 0 && return y + if β != one(β) + β == zero(β) ? fill!(y, zero(eltype(y))) : scale!(y, β) + end + α == zero(α) && return y + + xnzind = nonzeroinds(x) + xnzval = nonzeros(x) + Acolptr = A.colptr + Arowval = A.rowval + Anzval = A.nzval + mx = length(xnzind) + + for j = 1:n + # s <- dot(A[:,j], x) + s = _spdot(tfun, Acolptr[j], Acolptr[j+1]-1, Arowval, Anzval, + 1, mx, xnzind, xnzval) + @inbounds y[j] += s * α + end + return y +end + + +### BLAS-2 / sparse A * sparse x -> dense y + +function *(A::SparseMatrixCSC, x::AbstractSparseVector) + y = densemv(A, x) + initcap = min(nnz(A), size(A,1)) + _dense2sparsevec(y, initcap) +end + +At_mul_B(A::SparseMatrixCSC, x::AbstractSparseVector) = + _At_or_Ac_mul_B(MulFun(), A, x) + +Ac_mul_B(A::SparseMatrixCSC, x::AbstractSparseVector) = + _At_or_Ac_mul_B(DotFun(), A, x) + +function _At_or_Ac_mul_B{TvA,TiA,TvX,TiX}(tfun::BinaryOp, A::SparseMatrixCSC{TvA,TiA}, x::AbstractSparseVector{TvX,TiX}) + m, n = size(A) + length(x) == m || throw(DimensionMismatch()) + Tv = promote_type(TvA, TvX) + Ti = promote_type(TiA, TiX) + + xnzind = nonzeroinds(x) + xnzval = nonzeros(x) + Acolptr = A.colptr + Arowval = A.rowval + Anzval = A.nzval + mx = length(xnzind) + + ynzind = Array(Ti, n) + ynzval = Array(Tv, n) + + jr = 0 + for j = 1:n + s = _spdot(tfun, Acolptr[j], Acolptr[j+1]-1, Arowval, Anzval, + 1, mx, xnzind, xnzval) + if s != zero(s) + jr += 1 + ynzind[jr] = j + ynzval[jr] = s + end + end + if jr < n + resize!(ynzind, jr) + resize!(ynzval, jr) + end + SparseVector(n, ynzind, ynzval) +end diff --git a/test/sparsedir/sparsevector.jl b/test/sparsedir/sparsevector.jl new file mode 100644 index 0000000000000..f568a437e5188 --- /dev/null +++ b/test/sparsedir/sparsevector.jl @@ -0,0 +1,675 @@ +## sparsevec.jl + +### Data + +spv_x1 = SparseVector(8, [2, 5, 6], [1.25, -0.75, 3.5]) +_x2 = SparseVector(8, [1, 2, 6, 7], [3.25, 4.0, -5.5, -6.0]) +spv_x2 = view(_x2) + +@test isa(spv_x1, SparseVector{Float64,Int}) +@test isa(spv_x2, SparseVectorView{Float64,Int}) + +x1_full = zeros(length(spv_x1)) +x1_full[nonzeroinds(spv_x1)] = nonzeros(spv_x1) + +x2_full = zeros(length(spv_x2)) +x2_full[nonzeroinds(spv_x2)] = nonzeros(spv_x2) + + +### Basic Properties + +let x = spv_x1 + @test eltype(x) == Float64 + @test ndims(x) == 1 + @test length(x) == 8 + @test size(x) == (8,) + @test size(x,1) == 8 + @test size(x,2) == 1 + @test !isempty(x) + + @test countnz(x) == 3 + @test nnz(x) == 3 + @test nonzeroinds(x) == [2, 5, 6] + @test nonzeros(x) == [1.25, -0.75, 3.5] +end + +let x = spv_x2 + @test eltype(x) == Float64 + @test ndims(x) == 1 + @test length(x) == 8 + @test size(x) == (8,) + @test size(x,1) == 8 + @test size(x,2) == 1 + @test !isempty(x) + + @test countnz(x) == 4 + @test nnz(x) == 4 + @test nonzeroinds(x) == [1, 2, 6, 7] + @test nonzeros(x) == [3.25, 4.0, -5.5, -6.0] +end + +# full + +for (x, xf) in [(spv_x1, x1_full), (spv_x2, x2_full)] + @test isa(full(x), Vector{Float64}) + @test full(x) == xf +end + +### Show + +@test string(spv_x1) == "Sparse vector, length = 8, with 3 Float64 entries:\n" * +" [2] = 1.25\n" * +" [5] = -0.75\n" * +" [6] = 3.5\n" + + +### Other Constructors + +# construct empty sparse vector + +@test exact_equal(sparsevector(Float64, 8), SparseVector(8, Int[], Float64[])) + +# from list of indices and values + +@test exact_equal( + sparsevector(Int[], Float64[], 8), + SparseVector(8, Int[], Float64[])) + +@test exact_equal( + sparsevector(Int[], Float64[]), + SparseVector(0, Int[], Float64[])) + +@test exact_equal( + sparsevector([3, 3], [5.0, -5.0], 8), + sparsevector(Float64, 8)) + +@test exact_equal( + sparsevector([2, 3, 6], [12.0, 18.0, 25.0]), + SparseVector(6, [2, 3, 6], [12.0, 18.0, 25.0])) + +let x0 = SparseVector(8, [2, 3, 6], [12.0, 18.0, 25.0]) + @test exact_equal( + sparsevector([2, 3, 6], [12.0, 18.0, 25.0], 8), x0) + + @test exact_equal( + sparsevector([3, 6, 2], [18.0, 25.0, 12.0], 8), x0) + + @test exact_equal( + sparsevector([2, 3, 4, 4, 6], [12.0, 18.0, 5.0, -5.0, 25.0], 8), + x0) + + @test exact_equal( + sparsevector([1, 1, 1, 2, 3, 3, 6], [2.0, 3.0, -5.0, 12.0, 10.0, 8.0, 25.0], 8), + x0) + + @test exact_equal( + sparsevector([2, 3, 6, 7, 7], [12.0, 18.0, 25.0, 5.0, -5.0], 8), x0) +end + +# from dictionary + +function my_intmap(x) + a = Dict{Int,eltype(x)}() + for i in nonzeroinds(x) + a[i] = x[i] + end + return a +end + +let x = spv_x1 + a = my_intmap(x) + xc = sparsevector(a, 8) + @test exact_equal(x, xc) + + xc = sparsevector(a) + @test exact_equal(xc, SparseVector(6, [2, 5, 6], [1.25, -0.75, 3.5])) +end + +# sprand & sprandn + +let xr = sprand(1000, 0.3) + @test isa(xr, SparseVector{Float64,Int}) + @test length(xr) == 1000 + @test all(nonzeros(xr) .>= 0.0) +end + +let xr = sprand(1000, 0.3, Float32) + @test isa(xr, SparseVector{Float32,Int}) + @test length(xr) == 1000 + @test all(nonzeros(xr) .>= 0.0) +end + +let xr = sprandn(1000, 0.3) + @test isa(xr, SparseVector{Float64,Int}) + @test length(xr) == 1000 + @test any(nonzeros(xr) .> 0.0) && any(nonzeros(xr) .< 0.0) +end + + +### Element access + +# getindex + +# single integer index +for (x, xf) in [(spv_x1, x1_full), (spv_x2, x2_full)] + for i = 1:length(x) + @test x[i] == xf[i] + end +end + +# range index +let x = spv_x2 + # range that contains no non-zeros + @test exact_equal(x[3:2], sparsevector(Float64, 0)) + @test exact_equal(x[3:3], sparsevector(Float64, 1)) + @test exact_equal(x[3:5], sparsevector(Float64, 3)) + + # range with non-zeros + @test exact_equal(x[1:length(x)], x) + @test exact_equal(x[1:5], SparseVector(5, [1,2], [3.25, 4.0])) + @test exact_equal(x[2:6], SparseVector(5, [1,5], [4.0, -5.5])) + @test exact_equal(x[2:8], SparseVector(7, [1,5,6], [4.0, -5.5, -6.0])) +end + +# generic array index +let x = sprand(100, 0.5) + I = rand(1:length(x), 20) + @which x[I] + r = x[I] + @test isa(r, SparseVector{Float64,Int}) + @test all(nonzeros(r) .!= 0.0) + @test full(r) == full(x)[I] +end + +# setindex + +let xc = sparsevector(Float64, 8) + xc[3] = 2.0 + @test exact_equal(xc, SparseVector(8, [3], [2.0])) +end + +let xc = copy(spv_x1) + xc[5] = 2.0 + @test exact_equal(xc, SparseVector(8, [2, 5, 6], [1.25, 2.0, 3.5])) +end + +let xc = copy(spv_x1) + xc[3] = 4.0 + @test exact_equal(xc, SparseVector(8, [2, 3, 5, 6], [1.25, 4.0, -0.75, 3.5])) + + xc[1] = 6.0 + @test exact_equal(xc, SparseVector(8, [1, 2, 3, 5, 6], [6.0, 1.25, 4.0, -0.75, 3.5])) + + xc[8] = -1.5 + @test exact_equal(xc, SparseVector(8, [1, 2, 3, 5, 6, 8], [6.0, 1.25, 4.0, -0.75, 3.5, -1.5])) +end + +let xc = copy(spv_x1) + xc[5] = 0.0 + @test exact_equal(xc, SparseVector(8, [2, 6], [1.25, 3.5])) + + xc[6] = 0.0 + @test exact_equal(xc, SparseVector(8, [2], [1.25])) + + xc[2] = 0.0 + @test exact_equal(xc, SparseVector(8, Int[], Float64[])) +end + + +### Array manipulation + +# copy + +let x = spv_x1 + xc = copy(x) + @test isa(xc, SparseVector{Float64,Int}) + @test !is(x.nzind, xc.nzval) + @test !is(x.nzval, xc.nzval) + @test exact_equal(x, xc) +end + +let a = SparseVector(8, [2, 5, 6], Int32[12, 35, 72]) + # reinterpret + au = reinterpret(UInt32, a) + @test isa(au, SparseVector{UInt32,Int}) + @test exact_equal(au, SparseVector(8, [2, 5, 6], UInt32[12, 35, 72])) + + # float + af = float(a) + @test isa(af, SparseVector{Float64,Int}) + @test exact_equal(af, SparseVector(8, [2, 5, 6], [12., 35., 72.])) + + # complex + acp = complex(af) + @test isa(acp, SparseVector{Complex128,Int}) + @test exact_equal(acp, SparseVector(8, [2, 5, 6], complex([12., 35., 72.]))) +end + +### Type conversion + +let x = convert(SparseVector, sparse([2, 5, 6], [1, 1, 1], [1.25, -0.75, 3.5], 8, 1)) + @test isa(x, SparseVector{Float64,Int}) + @test exact_equal(x, spv_x1) +end + +let x = spv_x1, xf = x1_full + xc = convert(SparseVector, xf) + @test isa(xc, SparseVector{Float64,Int}) + @test exact_equal(xc, x) + + xc = convert(SparseVector{Float32,Int}, x) + xf32 = SparseVector(8, [2, 5, 6], [1.25f0, -0.75f0, 3.5f0]) + @test isa(xc, SparseVector{Float32,Int}) + @test exact_equal(xc, xf32) + + xc = convert(SparseVector{Float32}, x) + @test isa(xc, SparseVector{Float32,Int}) + @test exact_equal(xc, xf32) + + xm = convert(SparseMatrixCSC, x) + @test isa(xm, SparseMatrixCSC{Float64,Int}) + @test full(xm) == reshape(xf, 8, 1) + + xm = convert(SparseMatrixCSC{Float32}, x) + @test isa(xm, SparseMatrixCSC{Float32,Int}) + @test full(xm) == reshape(convert(Vector{Float32}, xf), 8, 1) +end + + +### Concatenation + +let m = 80, n = 100 + A = Array(SparseVector{Float64,Int}, n) + tnnz = 0 + for i = 1:length(A) + A[i] = sprand(m, 0.3) + tnnz += nnz(A[i]) + end + + H = hcat(A...) + @test isa(H, SparseMatrixCSC{Float64,Int}) + @test size(H) == (m, n) + @test nnz(H) == tnnz + Hr = zeros(m, n) + for j = 1:n + Hr[:,j] = full(A[j]) + end + @test full(H) == Hr + + V = vcat(A...) + @test isa(V, SparseVector{Float64,Int}) + @test length(V) == m * n + Vr = vec(Hr) + @test full(V) == Vr +end + + +## sparsemat.jl + +let S = sprand(4, 8, 0.5) + Sf = full(S) + @assert isa(Sf, Matrix{Float64}) + + # get a single column + for j = 1:size(S,2) + col = getcol(S, j) + @test isa(col, SparseVector{Float64,Int}) + @test length(col) == size(S,1) + @test full(col) == Sf[:,j] + end + + # column views + for j = 1:size(S,2) + col = view(S, :, j) + @test isa(col, SparseVectorView{Float64,Int}) + @test length(col) == size(S,1) + @test full(col) == Sf[:,j] + end + + # column-range views + + # non-empty range + V = unsafe_colrange(S, 2:6) + @test isa(V, SparseMatrixCSC{Float64,Int}) + @test size(V) == (4, 5) + @test full(V) == Sf[:, 2:6] + @test !isempty(V) + + # empty range + V0 = unsafe_colrange(S, 2:1) + @test isa(V0, SparseMatrixCSC{Float64,Int}) + @test size(V0) == (4, 0) + @test isempty(V0) + +end + + +## math.jl + +### Data + +rnd_x0 = sprand(50, 0.6) +rnd_x0f = full(rnd_x0) + +rnd_x1 = sprand(50, 0.7) * 4.0 +rnd_x1f = full(rnd_x1) + +spv_x1 = SparseVector(8, [2, 5, 6], [1.25, -0.75, 3.5]) +_x2 = SparseVector(8, [1, 2, 6, 7], [3.25, 4.0, -5.5, -6.0]) +spv_x2 = view(_x2) + + +### Arithmetic operations + +let x = spv_x1, x2 = spv_x2 + # negate + @test exact_equal(-x, SparseVector(8, [2, 5, 6], [-1.25, 0.75, -3.5])) + + # abs and abs2 + @test exact_equal(abs(x), SparseVector(8, [2, 5, 6], abs([1.25, -0.75, 3.5]))) + @test exact_equal(abs2(x), SparseVector(8, [2, 5, 6], abs2([1.25, -0.75, 3.5]))) + + # plus and minus + xa = SparseVector(8, [1,2,5,6,7], [3.25,5.25,-0.75,-2.0,-6.0]) + + @test exact_equal(x + x, x * 2) + @test exact_equal(x + x2, xa) + @test exact_equal(x2 + x, xa) + + xb = SparseVector(8, [1,2,5,6,7], [-3.25,-2.75,-0.75,9.0,6.0]) + @test exact_equal(x - x, SparseVector(8, Int[], Float64[])) + @test exact_equal(x - x2, xb) + @test exact_equal(x2 - x, -xb) + + @test full(x) + x2 == full(xa) + @test full(x) - x2 == full(xb) + @test x + full(x2) == full(xa) + @test x - full(x2) == full(xb) + + # multiplies + xm = SparseVector(8, [2, 6], [5.0, -19.25]) + @test exact_equal(x .* x, abs2(x)) + @test exact_equal(x .* x2, xm) + @test exact_equal(x2 .* x, xm) + + @test full(x) .* x2 == full(xm) + @test x .* full(x2) == full(xm) + + # max & min + @test exact_equal(max(x, x), x) + @test exact_equal(min(x, x), x) + @test exact_equal(max(x, x2), + SparseVector(8, Int[1, 2, 6], Float64[3.25, 4.0, 3.5])) + @test exact_equal(min(x, x2), + SparseVector(8, Int[2, 5, 6, 7], Float64[1.25, -0.75, -5.5, -6.0])) +end + +### Complex + +let x = spv_x1, x2 = spv_x2 + # complex + @test exact_equal(complex(x, x), + SparseVector(8, [2,5,6], [1.25+1.25im, -0.75-0.75im, 3.5+3.5im])) + @test exact_equal(complex(x, x2), + SparseVector(8, [1,2,5,6,7], [3.25im, 1.25+4.0im, -0.75+0.im, 3.5-5.5im, -6.0im])) + @test exact_equal(complex(x2, x), + SparseVector(8, [1,2,5,6,7], [3.25+0.im, 4.0+1.25im, -0.75im, -5.5+3.5im, -6.0+0.im])) + + # real & imag + + @test is(real(x), x) + @test exact_equal(imag(x), sparsevector(Float64, length(x))) + + xcp = complex(x, x2) + @test exact_equal(real(xcp), x) + @test exact_equal(imag(xcp), x2) +end + +### Zero-preserving math functions: sparse -> sparse + +function check_nz2z_z2z{T}(f::Function, x::SparseVector{T}, xf::Vector{T}) + R = typeof(f(zero(T))) + r = f(x) + isa(r, AbstractSparseVector) || error("$f(x) is not a sparse vector.") + eltype(r) == R || error("$f(x) results in eltype = $(eltype(r)), expect $R") + all(r.nzval .!= 0) || error("$f(x) contains zeros in nzval.") + full(r) == f(xf) || error("Incorrect results found in $f(x).") +end + +for f in [floor, ceil, trunc, round] + check_nz2z_z2z(f, rnd_x1, rnd_x1f) +end + +for f in [log1p, expm1, + sin, tan, sinpi, sind, tand, + asin, atan, asind, atand, + sinh, tanh, asinh, atanh] + check_nz2z_z2z(f, rnd_x0, rnd_x0f) +end + +### Non-zero-preserving math functions: sparse -> dense + +function check_z2nz{T}(f::Function, x::SparseVector{T}, xf::Vector{T}) + R = typeof(f(zero(T))) + r = f(x) + isa(r, Vector) || error("$f(x) is not a dense vector.") + eltype(r) == R || error("$f(x) results in eltype = $(eltype(r)), expect $R") + r == f(xf) || error("Incorrect results found in $f(x).") +end + +for f in [exp, exp2, exp10, log, log2, log10, + cos, csc, cot, sec, cospi, + cosd, cscd, cotd, secd, + acos, acot, acosd, acotd, + cosh, csch, coth, sech, acsch, asech] + check_z2nz(f, rnd_x0, rnd_x0f) +end + + +### Reduction + +# sum, sumabs, sumabs2, vecnorm + +let x = spv_x1 + @test sum(x) == 4.0 + @test sumabs(x) == 5.5 + @test sumabs2(x) == 14.375 + + @test vecnorm(x) == sqrt(14.375) + @test vecnorm(x, 1) == 5.5 + @test vecnorm(x, 2) == sqrt(14.375) + @test vecnorm(x, Inf) == 3.5 +end + +# maximum, minimum, maxabs, minabs + +let x = spv_x1 + @test maximum(x) == 3.5 + @test minimum(x) == -0.75 + @test maxabs(x) == 3.5 + @test minabs(x) == 0.0 +end + +let x = abs(spv_x1) + @test maximum(x) == 3.5 + @test minimum(x) == 0.0 +end + +let x = -abs(spv_x1) + @test maximum(x) == 0.0 + @test minimum(x) == -3.5 +end + +let x = SparseVector(3, [1, 2, 3], [-4.5, 2.5, 3.5]) + @test maximum(x) == 3.5 + @test minimum(x) == -4.5 + @test maxabs(x) == 4.5 + @test minabs(x) == 2.5 +end + +let x = sparsevector(Float64, 8) + @test maximum(x) == 0.0 + @test minimum(x) == 0.0 + @test maxabs(x) == 0.0 + @test minabs(x) == 0.0 +end + + +### linalg.jl + +### BLAS Level-1 + +let x = sprand(16, 0.5), x2 = sprand(16, 0.4) + xf = full(x) + xf2 = full(x2) + + # axpy! + for c in [1.0, -1.0, 2.0, -2.0] + y = full(x) + @test is(axpy!(c, x2, y), y) + @test y == full(x2 * c + x) + end + + # scale + let sx = SparseVector(x.n, x.nzind, x.nzval * 2.5) + @test exact_equal(scale(x, 2.5), sx) + @test exact_equal(scale(2.5, x), sx) + @test exact_equal(x * 2.5, sx) + @test exact_equal(2.5 * x, sx) + @test exact_equal(x .* 2.5, sx) + @test exact_equal(2.5 .* x, sx) + + xc = copy(x) + @test is(scale!(xc, 2.5), xc) + @test exact_equal(xc, sx) + end + + # dot + let dv = dot(xf, xf2) + @test dot(x, x) == sumabs2(x) + @test dot(x2, x2) == sumabs2(x2) + @test_approx_eq dot(x, x2) dv + @test_approx_eq dot(x2, x) dv + @test_approx_eq dot(full(x), x2) dv + @test_approx_eq dot(x, full(x2)) dv + end +end + +let x = complex(sprand(32, 0.6), sprand(32, 0.6)), + y = complex(sprand(32, 0.6), sprand(32, 0.6)) + xf = full(x)::Vector{Complex128} + yf = full(y)::Vector{Complex128} + @test_approx_eq dot(x, x) dot(xf, xf) + @test_approx_eq dot(x, y) dot(xf, yf) +end + + + +### BLAS Level-2: + +## dense A * sparse x -> dense y + +let A = randn(9, 16), x = sprand(16, 0.7) + xf = full(x) + for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] + y = rand(9) + rr = α * A * xf + β * y + @test is(A_mul_B!(α, A, x, β, y), y) + @test_approx_eq y rr + end + y = A * x + @test isa(y, Vector{Float64}) + @test_approx_eq A * x A * xf +end + +let A = randn(16, 9), x = sprand(16, 0.7) + xf = full(x) + for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] + y = rand(9) + rr = α * A'xf + β * y + @test is(At_mul_B!(α, A, x, β, y), y) + @test_approx_eq y rr + end + y = At_mul_B(A, x) + @test isa(y, Vector{Float64}) + @test_approx_eq y At_mul_B(A, xf) +end + +## sparse A * sparse x -> dense y + +let A = sprandn(9, 16, 0.5), x = sprand(16, 0.7) + Af = full(A) + xf = full(x) + for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] + y = rand(9) + rr = α * Af * xf + β * y + @test is(A_mul_B!(α, A, x, β, y), y) + @test_approx_eq y rr + end + y = densemv(A, x) + @test isa(y, Vector{Float64}) + @test_approx_eq y Af * xf +end + +let A = sprandn(16, 9, 0.5), x = sprand(16, 0.7) + Af = full(A) + xf = full(x) + for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] + y = rand(9) + rr = α * Af'xf + β * y + @test is(At_mul_B!(α, A, x, β, y), y) + @test_approx_eq y rr + end + y = densemv(A, x; trans='T') + @test isa(y, Vector{Float64}) + @test_approx_eq y At_mul_B(Af, xf) +end + +let A = complex(sprandn(7, 8, 0.5), sprandn(7, 8, 0.5)), + x = complex(sprandn(8, 0.6), sprandn(8, 0.6)), + x2 = complex(sprandn(7, 0.75), sprandn(7, 0.75)) + Af = full(A) + xf = full(x) + x2f = full(x2) + @test_approx_eq densemv(A, x; trans='N') Af * xf + @test_approx_eq densemv(A, x2; trans='T') Af.' * x2f + @test_approx_eq densemv(A, x2; trans='C') Af'x2f +end + +## sparse A * sparse x -> sparse y + +let A = sprandn(9, 16, 0.5), x = sprand(16, 0.7), x2 = sprand(9, 0.7) + Af = full(A) + xf = full(x) + x2f = full(x2) + + y = A * x + @test isa(y, SparseVector{Float64,Int}) + @test all(nonzeros(y) .!= 0.0) + @test_approx_eq full(y) Af * xf + + y = At_mul_B(A, x2) + @test isa(y, SparseVector{Float64,Int}) + @test all(nonzeros(y) .!= 0.0) + @test_approx_eq full(y) Af'x2f +end + +let A = complex(sprandn(7, 8, 0.5), sprandn(7, 8, 0.5)), + x = complex(sprandn(8, 0.6), sprandn(8, 0.6)), + x2 = complex(sprandn(7, 0.75), sprandn(7, 0.75)) + Af = full(A) + xf = full(x) + x2f = full(x2) + + y = A * x + @test isa(y, SparseVector{Complex128,Int}) + @test_approx_eq full(y) Af * xf + + y = At_mul_B(A, x2) + @test isa(y, SparseVector{Complex128,Int}) + @test_approx_eq full(y) Af.' * x2f + + y = Ac_mul_B(A, x2) + @test isa(y, SparseVector{Complex128,Int}) + @test_approx_eq full(y) Af'x2f +end From b53644ddd23ee650c2515b35bf4e8625aece65a8 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Sat, 3 Oct 2015 13:10:58 -0400 Subject: [PATCH 0541/1938] Integrate SparseVectors.jl with Base sparse module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renamings: * Rename sparse module to SparseArrays, and improve its separation from base. This makes it very simple to dynamically reload the sparse module. Move docstrings to their proper place * _copy_convert → collect * Rename sparsevector to the existing spzeros and sparsevec. * Use call overloading instead of call Remove functionality from SparseVectors.jl: * Simplify and remove some functors * Remove SparseVectorView * Remove no-longer-needed ambiguity preventers Add functionality for SparseVectors: * Add similar for SparseVector * Allow sparsevec(::AbstractArray), not just vectors * Add spzeros(n); adapt some tests to SparseVector * Allow CHOLMOD linalg with vectors * Implement (c)transpose(::SparseVector). Returns a dense vector since a one-row CSC structure is effectively dense but with worse performance. * Add vector sprandbool and allow passing RNG to all vector sprand* functions. Harden tests against random failures. * Implement, test and doc spones(::SparseVector) Improve performance for SparseVector indexing: * Punt to SparseMatrix for some indexing behaviors. Since the datastructures are compatible and SparseMatrix's routines are more optimized, it is easiest to just construct a temporary SparseMatrix and index into that. This is faster in all but the smallest of cases (N<10) * Use searchsorted for indexing SparseVector by UnitRange. This is about 20% slower on very small vectors, but is faster in general. Change SparseMatrix behaviors to take advantage of vectors * Add indexing behaviors for SparseMatrix->SparseVector * `vec` and `sparsevec` for CSC return SparseVectors * Update documentation to incorporate vectors Minor bugfixes and changes to SparseVectors: * Compare to literal 0 in vector construction and setindex. This matches SparseMatrix semantics, and makes indexing semantics consistent with regard to stored zeros * Use checkbounds more thoroughly * Use error types that are consistent with SparseMatrixCSC * Minor sparse vector display tweaks. Turn on output limiting by default, remove unused variable `k`, implement and use Base.summary * Fix missing return, add test Add some tests: * Add a test and comment to ensure nobody else tries to share data between vectors and matrices * Ensure indexing is consistent between one-column sparse matrices and sparse vectors, with special attention to stored zeros. --- base/deprecated.jl | 2 + base/docs/helpdb.jl | 34 --- base/exports.jl | 20 -- base/irrationals.jl | 3 +- base/precompile.jl | 1 - base/sparse.jl | 10 +- base/sparse/cholmod.jl | 47 +++- base/sparse/csparse.jl | 9 + base/sparse/linalg.jl | 4 +- base/sparse/sparsematrix.jl | 192 ++++--------- base/sparse/sparsevector.jl | 494 +++++++++++++++++---------------- base/sparse/spqr.jl | 6 +- base/sparse/umfpack.jl | 4 +- base/sysimg.jl | 31 ++- doc/stdlib/arrays.rst | 56 ++-- doc/stdlib/linalg.rst | 2 +- test/sparse.jl | 1 + test/sparsedir/cholmod.jl | 8 +- test/sparsedir/sparse.jl | 59 ++-- test/sparsedir/sparsevector.jl | 218 ++++++++------- test/sparsedir/spqr.jl | 4 +- test/sparsedir/umfpack.jl | 6 +- 22 files changed, 584 insertions(+), 627 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 1219c1107bffe..687ff736c9316 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -852,3 +852,5 @@ end @deprecate cor(X::AbstractMatrix; vardim=1, mean=Base.mean(X, vardim)) corm(X, mean, vardim) @deprecate cor(x::AbstractVector, y::AbstractVector; mean=(Base.mean(x), Base.mean(y))) corm(x, mean[1], y, mean[2]) @deprecate cor(X::AbstractVecOrMat, Y::AbstractVecOrMat; vardim=1, mean=(Base.mean(X, vardim), Base.mean(Y, vardim))) corm(X, mean[1], Y, mean[2], vardim) + +@deprecate_binding SparseMatrix SparseArrays diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 8cccb985bd338..730ba60ae2b43 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -2387,17 +2387,6 @@ Right division operator: multiplication of `x` by the inverse of `y` on the righ """ Base.(:(/)) -doc""" - ldltfact(::Union{SparseMatrixCSC,Symmetric{Float64,SparseMatrixCSC{Flaot64,SuiteSparse_long}},Hermitian{Complex{Float64},SparseMatrixCSC{Complex{Float64},SuiteSparse_long}}}; shift=0, perm=Int[]) -> CHOLMOD.Factor - -Compute the `LDLt` factorization of a sparse symmetric or Hermitian matrix. A fill-reducing permutation is used. `F = ldltfact(A)` is most frequently used to solve systems of equations `A*x = b` with `F\b`, but also the methods `diag`, `det`, `logdet` are defined for `F`. You can also extract individual factors from `F`, using `F[:L]`. However, since pivoting is on by default, the factorization is internally represented as `A == P'*L*D*L'*P` with a permutation matrix `P`; using just `L` without accounting for `P` will give incorrect answers. To include the effects of permutation, it's typically preferable to extact "combined" factors like `PtL = F[:PtL]` (the equivalent of `P'*L`) and `LtP = F[:UP]` (the equivalent of `L'*P`). The complete list of supported factors is `:L, :PtL, :D, :UP, :U, :LD, :DU, :PtLD, :DUP`. - -Setting optional `shift` keyword argument computes the factorization of `A+shift*I` instead of `A`. If the `perm` argument is nonempty, it should be a permutation of `1:size(A,1)` giving the ordering to use (instead of CHOLMOD's default AMD ordering). - -The function calls the C library CHOLMOD and many other functions from the library are wrapped but not exported. -""" -ldltfact(A::SparseMatrixCSC; shift=0, perm=Int[]) - doc""" connect([host],port) -> TcpSocket @@ -7256,13 +7245,6 @@ Tests whether `A` or its elements are of type `T`. """ iseltype -doc""" - symperm(A, p) - -Return the symmetric permutation of `A`, which is `A[p,p]`. `A` should be symmetric and sparse, where only the upper triangular part of the matrix is stored. This algorithm ignores the lower triangular part of the matrix. Only the upper triangular part of the result is returned as well. -""" -symperm - doc""" min(x, y, ...) @@ -9993,15 +9975,6 @@ Like permute!, but the inverse of the given permutation is applied. """ ipermute! -doc""" -```rst -.. full(S) - -Convert a sparse matrix ``S`` into a dense matrix. -``` -""" -full(::AbstractSparseMatrix) - doc""" ```rst .. full(F) @@ -10487,13 +10460,6 @@ k]``.) """ eigfact(A,B) -doc""" - rowvals(A) - -Return a vector of the row indices of `A`, and any modifications to the returned vector will mutate `A` as well. Given the internal storage format of sparse matrices, providing access to how the row indices are stored internally can be useful in conjuction with iterating over structural nonzero values. See `nonzeros(A)` and `nzrange(A, col)`. -""" -rowvals - doc""" mkdir(path, [mode]) diff --git a/base/exports.jl b/base/exports.jl index c19f89a6c1604..7caf6944ff079 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -20,16 +20,12 @@ export BLAS, LAPACK, Serializer, - SparseMatrix, Docs, Markdown, # Types AbstractChannel, AbstractMatrix, - AbstractSparseArray, - AbstractSparseMatrix, - AbstractSparseVector, AbstractVector, AbstractVecOrMat, Array, @@ -107,7 +103,6 @@ export SharedArray, SharedMatrix, SharedVector, - SparseMatrixCSC, StatStruct, StepRange, StridedArray, @@ -562,7 +557,6 @@ export minimum, minmax, ndims, - nnz, nonzeros, nthperm!, nthperm, @@ -715,21 +709,7 @@ export ×, # sparse - etree, full, - issparse, - sparse, - sparsevec, - spdiagm, - speye, - spones, - sprand, - sprandbool, - sprandn, - spzeros, - symperm, - rowvals, - nzrange, # bitarrays bitpack, diff --git a/base/irrationals.jl b/base/irrationals.jl index 7e7d1342f4a1f..24346506fead6 100644 --- a/base/irrationals.jl +++ b/base/irrationals.jl @@ -122,10 +122,9 @@ const golden = φ for T in (Irrational, Rational, Integer, Number) ^(::Irrational{:e}, x::T) = exp(x) end -for T in (Range, BitArray, SparseMatrixCSC, StridedArray, AbstractArray) +for T in (Range, BitArray, StridedArray, AbstractArray) .^(::Irrational{:e}, x::T) = exp(x) end -^(::Irrational{:e}, x::AbstractMatrix) = expm(x) log(::Irrational{:e}) = 1 # use 1 to correctly promote expressions like log(x)/log(e) log(::Irrational{:e}, x) = log(x) diff --git a/base/precompile.jl b/base/precompile.jl index a418295527902..42c326b0a7a38 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -294,7 +294,6 @@ precompile(Base.next, (Dict{Symbol,Any},Int)) precompile(Base.next, (IntSet, Int)) precompile(Base.next, (UnitRange{Int},Int)) precompile(Base.nextind, (ASCIIString, Int)) -precompile(Base.nnz, (BitArray{1},)) precompile(Base.normpath, (ASCIIString, ASCIIString)) precompile(Base.normpath, (ASCIIString,)) precompile(Base.normpath, (UTF8String, UTF8String)) diff --git a/base/sparse.jl b/base/sparse.jl index 92573e83a184d..d40603bc3f3f0 100644 --- a/base/sparse.jl +++ b/base/sparse.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -module SparseMatrix +module SparseArrays using Base: Func, AddFun, OrFun, ConjFun, IdFun using Base.Sort: Forward @@ -17,12 +17,14 @@ import Base: @get!, abs, abs2, broadcast, ceil, complex, cond, conj, convert, co import Base.Broadcast: eltype_plus, broadcast_shape export AbstractSparseArray, AbstractSparseMatrix, AbstractSparseVector, SparseMatrixCSC, - blkdiag, dense, droptol!, dropzeros!, etree, issparse, nnz, nonzeros, nzrange, + SparseVector, + blkdiag, dense, droptol!, dropzeros!, etree, issparse, nonzeros, nzrange, rowvals, sparse, sparsevec, spdiagm, speye, spones, sprand, sprandbool, sprandn, - spzeros, symperm + spzeros, symperm, nnz include("sparse/abstractsparse.jl") include("sparse/sparsematrix.jl") +include("sparse/sparsevector.jl") include("sparse/csparse.jl") include("sparse/linalg.jl") @@ -32,4 +34,4 @@ if Base.USE_GPL_LIBS include("sparse/spqr.jl") end -end # module SparseMatrix +end diff --git a/base/sparse/cholmod.jl b/base/sparse/cholmod.jl index ec7e980a7ed49..cb084c4910d5d 100644 --- a/base/sparse/cholmod.jl +++ b/base/sparse/cholmod.jl @@ -9,14 +9,14 @@ import Base.LinAlg: (\), A_mul_Bc, A_mul_Bt, Ac_ldiv_B, Ac_mul_B, At_ldiv_B, At_ cholfact, det, diag, ishermitian, isposdef, issym, ldltfact, logdet -import Base.SparseMatrix: sparse, nnz +importall ..SparseArrays export Dense, Factor, Sparse -using Base.SparseMatrix: AbstractSparseMatrix, SparseMatrixCSC, increment, indtype +import ..SparseArrays: AbstractSparseMatrix, SparseMatrixCSC, increment, indtype ######### # Setup # @@ -853,6 +853,9 @@ function convert{Tv<:VTypes}(::Type{Sparse}, A::SparseMatrixCSC{Tv,SuiteSparse_l return o end + +# convert SparseVectors into CHOLMOD Sparse types through a mx1 CSC matrix +convert{Tv<:VTypes}(::Type{Sparse}, A::SparseVector{Tv,SuiteSparse_long}) = convert(Sparse, convert(SparseMatrixCSC, A)) function convert{Tv<:VTypes}(::Type{Sparse}, A::SparseMatrixCSC{Tv,SuiteSparse_long}) o = Sparse(A, 0) # check if array is symmetric and change stype if it is @@ -994,7 +997,7 @@ function sparse(F::Factor) L, d = getLd!(LD) A = scale(L, d)*L' end - SparseMatrix.sortSparseMatrixCSC!(A) + SparseArrays.sortSparseMatrixCSC!(A) p = get_perm(F) if p != [1:s.n;] pinv = Array(Int, length(p)) @@ -1216,6 +1219,32 @@ function cholfact(A::Sparse; kws...) return F end +doc""" + ldltfact(::Union{SparseMatrixCSC,Symmetric{Float64,SparseMatrixCSC{Flaot64,SuiteSparse_long}},Hermitian{Complex{Float64},SparseMatrixCSC{Complex{Float64},SuiteSparse_long}}}; shift=0, perm=Int[]) -> CHOLMOD.Factor + +Compute the `LDLt` factorization of a sparse symmetric or Hermitian matrix. A +fill-reducing permutation is used. `F = ldltfact(A)` is most frequently used to +solve systems of equations `A*x = b` with `F\b`. The returned factorization +object `F` also supports the methods `diag`, `det`, and `logdet`. You can +extract individual factors from `F` using `F[:L]`. However, since pivoting is +on by default, the factorization is internally represented as `A == P'*L*D*L'*P` +with a permutation matrix `P`; using just `L` without accounting for `P` will +give incorrect answers. To include the effects of permutation, it's typically +preferable to extact "combined" factors like `PtL = F[:PtL]` (the equivalent of +`P'*L`) and `LtP = F[:UP]` (the equivalent of `L'*P`). The complete list of +supported factors is `:L, :PtL, :D, :UP, :U, :LD, :DU, :PtLD, :DUP`. + +Setting optional `shift` keyword argument computes the factorization of +`A+shift*I` instead of `A`. If the `perm` argument is nonempty, it should be a +permutation of `1:size(A,1)` giving the ordering to use (instead of CHOLMOD's +default AMD ordering). + +The function calls the C library CHOLMOD and many other functions from the +library are wrapped but not exported. + +""" +ldltfact(A::SparseMatrixCSC; shift=0, perm=Int[]) + function ldltfact(A::Sparse; kws...) cm = defaults(common()) # setting the common struct to default values. Should only be done when creating new factorization. set_print_level(cm, 0) # no printing from CHOLMOD by default @@ -1294,13 +1323,15 @@ for (T, f) in ((:Dense, :solve), (:Sparse, :spsolve)) end end +typealias SparseVecOrMat{Tv,Ti} Union{SparseVector{Tv,Ti}, SparseMatrixCSC{Tv,Ti}} + function (\)(L::FactorComponent, b::Vector) reshape(convert(Matrix, L\Dense(b)), length(b)) end function (\)(L::FactorComponent, B::Matrix) convert(Matrix, L\Dense(B)) end -function (\)(L::FactorComponent, B::SparseMatrixCSC) +function (\)(L::FactorComponent, B::SparseVecOrMat) sparse(L\Sparse(B,0)) end @@ -1311,12 +1342,12 @@ Ac_ldiv_B(L::FactorComponent, B) = ctranspose(L)\B (\)(L::Factor, B::Matrix) = convert(Matrix, solve(CHOLMOD_A, L, Dense(B))) (\)(L::Factor, B::Sparse) = spsolve(CHOLMOD_A, L, B) # When right hand side is sparse, we have to ensure that the rhs is not marked as symmetric. -(\)(L::Factor, B::SparseMatrixCSC) = sparse(spsolve(CHOLMOD_A, L, Sparse(B, 0))) +(\)(L::Factor, B::SparseVecOrMat) = sparse(spsolve(CHOLMOD_A, L, Sparse(B, 0))) Ac_ldiv_B(L::Factor, B::Dense) = solve(CHOLMOD_A, L, B) Ac_ldiv_B(L::Factor, B::VecOrMat) = convert(Matrix, solve(CHOLMOD_A, L, Dense(B))) Ac_ldiv_B(L::Factor, B::Sparse) = spsolve(CHOLMOD_A, L, B) -Ac_ldiv_B(L::Factor, B::SparseMatrixCSC) = Ac_ldiv_B(L, Sparse(B)) +Ac_ldiv_B(L::Factor, B::SparseVecOrMat) = Ac_ldiv_B(L, Sparse(B)) ## Other convenience methods function diag{Tv}(F::Factor{Tv}) @@ -1398,7 +1429,7 @@ function ishermitian(A::Sparse{Complex{Float64}}) end end -(*){Ti}(A::Symmetric{Float64,SparseMatrixCSC{Float64,Ti}}, B::SparseMatrixCSC{Float64,Ti}) = sparse(Sparse(A)*Sparse(B)) -(*){Ti}(A::Hermitian{Complex{Float64},SparseMatrixCSC{Complex{Float64},Ti}}, B::SparseMatrixCSC{Complex{Float64},Ti}) = sparse(Sparse(A)*Sparse(B)) +(*){Ti}(A::Symmetric{Float64,SparseMatrixCSC{Float64,Ti}}, B::SparseVecOrMat{Float64,Ti}) = sparse(Sparse(A)*Sparse(B)) +(*){Ti}(A::Hermitian{Complex{Float64},SparseMatrixCSC{Complex{Float64},Ti}}, B::SparseVecOrMat{Complex{Float64},Ti}) = sparse(Sparse(A)*Sparse(B)) end #module diff --git a/base/sparse/csparse.jl b/base/sparse/csparse.jl index 695ad08236213..16977b6d19056 100644 --- a/base/sparse/csparse.jl +++ b/base/sparse/csparse.jl @@ -313,8 +313,17 @@ function csc_permute{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, pinv::Vector{Ti}, q::Vect (C.').' # double transpose to order the columns end + # based on cs_symperm p. 21, "Direct Methods for Sparse Linear Systems" # form A[p,p] for a symmetric A stored in the upper triangle +doc""" + symperm(A, p) + +Return the symmetric permutation of `A`, which is `A[p,p]`. `A` should be +symmetric, sparse, and only contain nonzeros in the upper triangular part of the +matrix is stored. This algorithm ignores the lower triangular part of the +matrix. Only the upper triangular part of the result is returned. +""" function symperm{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, pinv::Vector{Ti}) m, n = size(A) if m != n diff --git a/base/sparse/linalg.jl b/base/sparse/linalg.jl index ffad8c5f4b51a..9475575073df7 100644 --- a/base/sparse/linalg.jl +++ b/base/sparse/linalg.jl @@ -160,7 +160,7 @@ function spmatmul{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, B::SparseMatrixCSC{Tv,Ti}; # The Gustavson algorithm does not guarantee the product to have sorted row indices. Cunsorted = SparseMatrixCSC(mA, nB, colptrC, rowvalC, nzvalC) - C = Base.SparseMatrix.sortSparseMatrixCSC!(Cunsorted, sortindices=sortindices) + C = SparseArrays.sortSparseMatrixCSC!(Cunsorted, sortindices=sortindices) return C end @@ -752,7 +752,7 @@ inv(A::SparseMatrixCSC) = error("The inverse of a sparse matrix can often be den ## scale methods -# Copy colptr and rowval from one SparseMatrix to another +# Copy colptr and rowval from one sparse matrix to another function copyinds!(C::SparseMatrixCSC, A::SparseMatrixCSC) if C.colptr !== A.colptr resize!(C.colptr, length(A.colptr)) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index c29100678abd3..027012dea94e2 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -29,7 +29,7 @@ size(S::SparseMatrixCSC) = (S.m, S.n) """ nnz(A) -Returns the number of stored (filled) elements in a sparse matrix. +Returns the number of stored (filled) elements in a sparse array. """ nnz(S::SparseMatrixCSC) = Int(S.colptr[end]-1) countnz(S::SparseMatrixCSC) = countnz(S.nzval) @@ -37,17 +37,26 @@ countnz(S::SparseMatrixCSC) = countnz(S.nzval) """ nonzeros(A) -Return a vector of the structural nonzero values in sparse matrix `A`. This -includes zeros that are explicitly stored in the sparse matrix. The returned +Return a vector of the structural nonzero values in sparse array `A`. This +includes zeros that are explicitly stored in the sparse array. The returned vector points directly to the internal nonzero storage of `A`, and any modifications to the returned vector will mutate `A` as well. See `rowvals(A)` and `nzrange(A, col)`. """ nonzeros(S::SparseMatrixCSC) = S.nzval + +""" + rowvals(A::SparseMatrixCSC) + +Return a vector of the row indices of `A`. Any modifications to the returned +vector will mutate `A` as well. Providing access to how the row indices are +stored internally can be useful in conjuction with iterating over structural +nonzero values. See also `nonzeros(A)` and `nzrange(A, col)`. +""" rowvals(S::SparseMatrixCSC) = S.rowval """ - nzrange(A, col) + nzrange(A::SparseMatrixCSC, col) Return the range of indices to the structural nonzero values of a sparse matrix column. In conjunction with `nonzeros(A)` and `rowvals(A)`, this allows for @@ -221,6 +230,14 @@ end convert{T}(::Type{AbstractMatrix{T}}, A::SparseMatrixCSC) = convert(SparseMatrixCSC{T}, A) convert(::Type{Matrix}, S::SparseMatrixCSC) = full(S) + +""" + full(S) + +Convert a sparse matrix or vector `S` into a dense matrix or vector. +""" +full + function full{Tv}(S::SparseMatrixCSC{Tv}) # Handle cases where zero(Tv) is not defined but the array is dense. # (Should we really worry about this?) @@ -239,78 +256,8 @@ complex(A::SparseMatrixCSC, B::SparseMatrixCSC) = A + im*B # Construct a sparse vector -function vec{Tv,Ti}(S::SparseMatrixCSC{Tv,Ti}) - colptr = Array(Ti,2) - rowval = similar(S.rowval) - lS = length(S) - sparse_compute_reshaped_colptr_and_rowval(colptr, rowval, lS, 1, S.colptr, S.rowval, S.m, S.n) - SparseMatrixCSC(lS, 1, colptr, rowval, copy(S.nzval)) -end - -sparsevec(A::AbstractMatrix) = reshape(sparse(A), (length(A),1)) -sparsevec(S::SparseMatrixCSC) = vec(S) - -""" - sparsevec(D::Dict, [m]) - -Create a sparse matrix of size `m x 1` where the row values are keys from -the dictionary, and the nonzero values are the values from the dictionary. -""" -sparsevec{K<:Integer,V}(d::Dict{K,V}, len::Int) = sparsevec(collect(keys(d)), collect(values(d)), len) - -sparsevec{K<:Integer,V}(d::Dict{K,V}) = sparsevec(collect(keys(d)), collect(values(d))) - -sparsevec(I::AbstractVector, V, m::Integer) = sparsevec(I, V, m, AddFun()) - -sparsevec(I::AbstractVector, V) = sparsevec(I, V, maximum(I), AddFun()) - -""" - sparsevec(I, V, [m, combine]) - -Create a sparse matrix `S` of size `m x 1` such that `S[I[k]] = V[k]`. -Duplicates are combined using the `combine` function, which defaults to -`+` if it is not provided. In julia, sparse vectors are really just sparse -matrices with one column. Given Julia's Compressed Sparse Columns (CSC) -storage format, a sparse column matrix with one column is sparse, whereas -a sparse row matrix with one row ends up being dense. -""" -function sparsevec(I::AbstractVector, V, m::Integer, combine::Union{Function,Base.Func}) - nI = length(I) - if isa(V, Number) - V = fill(V, nI) - end - if nI != length(V) - throw(ArgumentError("index and value vectors must be the same length")) - end - p = sortperm(I) - @inbounds I = I[p] - if nI > 0 - if I[1] <= 0 - throw(ArgumentError("I index values must be ≥ 0")) - end - if I[end] > m - throw(ArgumentError("all I index values must be ≤ length(sparsevec)")) - end - end - V = V[p] - sparse_IJ_sorted!(I, ones(eltype(I), nI), V, m, 1, combine) -end - -""" - sparsevec(A) - -Convert a dense vector `A` into a sparse matrix of size `m x 1`. In julia, -sparse vectors are really just sparse matrices with one column. -""" -function sparsevec(a::Vector) - n = length(a) - I = find(a) - J = ones(Int, n) - V = a[I] - return sparse_IJ_sorted!(I,J,V,n,1,AddFun()) -end - -sparse(a::Vector) = sparsevec(a) +# Note that unlike `vec` for arrays, this does not share data +vec(S::SparseMatrixCSC) = S[:] """ sparse(A) @@ -500,14 +447,15 @@ end """ ```rst -.. sprand([rng,] m,n,p [,rfn]) - -Create a random ``m`` by ``n`` sparse matrix, in which the probability of any -element being nonzero is independently given by ``p`` (and hence the mean -density of nonzeros is also exactly ``p``). Nonzero values are sampled from -the distribution specified by ``rfn``. The uniform distribution is used in -case ``rfn`` is not specified. The optional ``rng`` argument specifies a -random number generator, see :ref:`Random Numbers `. +.. sprand([rng],m,[n],p::AbstractFloat,[rfn]) + +Create a random length ``m`` sparse vector or ``m`` by ``n`` sparse matrix, in +which the probability of any element being nonzero is independently given by +``p`` (and hence the mean density of nonzeros is also exactly ``p``). Nonzero +values are sampled from the distribution specified by ``rfn``. The uniform +distribution is used in case ``rfn`` is not specified. The optional ``rng`` +argument specifies a random number generator, see :ref:`Random Numbers +`. ``` """ function sprand{T}(r::AbstractRNG, m::Integer, n::Integer, density::AbstractFloat, @@ -535,11 +483,11 @@ sprand(m::Integer, n::Integer, density::AbstractFloat) = sprand(GLOBAL_RNG,m,n,d sprandn(r::AbstractRNG, m::Integer, n::Integer, density::AbstractFloat) = sprand(r,m,n,density,randn,Float64) """ - sprandn(m,n,p) + sprandn(m[,n],p::AbstractFloat) -Create a random `m` by `n` sparse matrix with the specified (independent) -probability `p` of any entry being nonzero, where nonzero values are -sampled from the normal distribution. +Create a random sparse vector of length `m` or sparse matrix of size `m` by `n` +with the specified (independent) probability `p` of any entry being nonzero, +where nonzero values are sampled from the normal distribution. """ sprandn( m::Integer, n::Integer, density::AbstractFloat) = sprandn(GLOBAL_RNG,m,n,density) @@ -547,27 +495,29 @@ truebools(r::AbstractRNG, n::Integer) = ones(Bool, n) sprandbool(r::AbstractRNG, m::Integer, n::Integer, density::AbstractFloat) = sprand(r,m,n,density,truebools,Bool) """ - sprandbool(m,n,p) + sprandbool(m[,n],p) -Create a random `m` by `n` sparse boolean matrix with the specified -(independent) probability `p` of any entry being `true`. +Create a random `m` by `n` sparse boolean matrix or length `m` sparse boolean +vector with the specified (independent) probability `p` of any entry being +`true`. """ sprandbool(m::Integer, n::Integer, density::AbstractFloat) = sprandbool(GLOBAL_RNG,m,n,density) """ spones(S) -Create a sparse matrix with the same structure as that of `S`, but with every nonzero +Create a sparse array with the same structure as that of `S`, but with every nonzero element having the value `1.0`. """ spones{T}(S::SparseMatrixCSC{T}) = SparseMatrixCSC(S.m, S.n, copy(S.colptr), copy(S.rowval), ones(T, S.colptr[end]-1)) """ - spzeros(m,n) + spzeros(m[,n]) -Create a sparse matrix of size `m x n`. This sparse matrix will not contain any -nonzero values. No storage will be allocated for nonzero values during construction. +Create a sparse vector of length `m` or sparse matrix of size `m x n`. This +sparse array will not contain any nonzero values. No storage will be allocated +for nonzero values during construction. """ spzeros(m::Integer, n::Integer) = spzeros(Float64, m, n) spzeros(Tv::Type, m::Integer, n::Integer) = spzeros(Tv, Int, m, n) @@ -1041,6 +991,7 @@ end # macro (.^)(A::SparseMatrixCSC, B::Number) = B==0 ? sparse(ones(typeof(one(eltype(A)).^B), A.m, A.n)) : SparseMatrixCSC(A.m, A.n, copy(A.colptr), copy(A.rowval), A.nzval .^ B) +(.^)(::Irrational{:e}, B::SparseMatrixCSC) = exp(B) (.^)(A::Number, B::SparseMatrixCSC) = (.^)(A, full(B)) (.^)(A::SparseMatrixCSC, B::Array) = (.^)(full(A), B) (.^)(A::Array, B::SparseMatrixCSC) = (.^)(A, full(B)) @@ -1400,13 +1351,10 @@ function getindex{T}(A::SparseMatrixCSC{T}, i0::Integer, i1::Integer) ((r1 > r2) || (A.rowval[r1] != i0)) ? zero(T) : A.nzval[r1] end -getindex{T<:Integer}(A::SparseMatrixCSC, I::AbstractVector{T}, j::Integer) = getindex(A,I,[j]) getindex{T<:Integer}(A::SparseMatrixCSC, i::Integer, J::AbstractVector{T}) = getindex(A,[i],J) -# Colon translation (this could be done more efficiently) -getindex(A::SparseMatrixCSC, ::Colon) = getindex(A, 1:length(A)) -getindex(A::SparseMatrixCSC, ::Colon, ::Colon) = getindex(A, 1:size(A, 1), 1:size(A, 2)) -getindex(A::SparseMatrixCSC, ::Colon, j) = getindex(A, 1:size(A, 1), j) +# Colon translation +getindex(A::SparseMatrixCSC, ::Colon, ::Colon) = copy(A) getindex(A::SparseMatrixCSC, i, ::Colon) = getindex(A, i, 1:size(A, 2)) function getindex_cols{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, J::AbstractVector) @@ -1730,8 +1678,8 @@ end function getindex_general{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, I::AbstractVector, J::AbstractVector) pI = sortperm(I) - @inbounds I = I[pI] - permute_rows!(getindex_I_sorted(A, I, J), pI) + @inbounds Is = I[pI] + permute_rows!(getindex_I_sorted(A, Is, J), pI) end # the general case: @@ -1812,46 +1760,6 @@ getindex(A::SparseMatrixCSC, I::AbstractVector{Bool}, J::AbstractVector{Bool}) = getindex{T<:Integer}(A::SparseMatrixCSC, I::AbstractVector{T}, J::AbstractVector{Bool}) = A[I,find(J)] getindex{T<:Integer}(A::SparseMatrixCSC, I::AbstractVector{Bool}, J::AbstractVector{T}) = A[find(I),J] -function getindex{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractArray{Bool}) - checkbounds(A, I) - n = sum(I) - - colptrA = A.colptr; rowvalA = A.rowval; nzvalA = A.nzval - colptrB = Int[1,n+1] - rowvalB = Array(Int, n) - nzvalB = Array(Tv, n) - c = 1 - rowB = 1 - - @inbounds for col in 1:A.n - r1 = colptrA[col] - r2 = colptrA[col+1]-1 - - for row in 1:A.m - if I[row, col] - while (r1 <= r2) && (rowvalA[r1] < row) - r1 += 1 - end - if (r1 <= r2) && (rowvalA[r1] == row) - nzvalB[c] = nzvalA[r1] - rowvalB[c] = rowB - c += 1 - end - rowB += 1 - (rowB > n) && break - end - end - (rowB > n) && break - end - colptrB[end] = c - n = length(nzvalB) - if n > (c-1) - deleteat!(nzvalB, c:n) - deleteat!(rowvalB, c:n) - end - SparseMatrixCSC(n, 1, colptrB, rowvalB, nzvalB) -end - ## setindex! function setindex!{T,Ti}(A::SparseMatrixCSC{T,Ti}, v, i0::Integer, i1::Integer) i0 = convert(Ti, i0) diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index b580271f605d6..43ea1dbf5cbc5 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -1,47 +1,19 @@ -### common.jl +# This file is a part of Julia. License is MIT: http://julialang.org/license -# not exported, used mainly for testing +### Common definitions -_copy_convert{T}(::Type{T}, x::Vector{T}) = copy(x) -_copy_convert{R,T}(::Type{R}, x::AbstractVector{T}) = convert(Vector{R}, x) - -import Base: Func, AddFun, MulFun, MaxFun, MinFun - -if isdefined(Base, :call) - import Base: call -else - call(f::Function, x) = f(x) - call(f::Function, x, y) = f(x, y) - call(f::Func{1}, x) = Base.evaluate(f, x) - call(f::Func{2}, x, y) = Base.evaluate(f, x, y) -end - -if isdefined(Base, :SubFun) - import Base: SubFun -else - immutable SubFun <: Func{2} end - call(::SubFun, x, y) = x - y -end - -immutable RealFun <: Func{1} end -call(::RealFun, x) = real(x) - -immutable ImagFun <: Func{1} end -call(::ImagFun, x) = imag(x) +import Base: Func, AddFun, MulFun, MaxFun, MinFun, SubFun immutable ComplexFun <: Func{2} end call(::ComplexFun, x::Real, y::Real) = complex(x, y) immutable DotFun <: Func{2} end -_dot(x::Number, y::Number) = conj(x) * y -_dot(x::Real, y::Real) = x * y -call(::DotFun, x::Number, y::Number) = _dot(x, y) - -typealias UnaryOp Union(Function, Func{1}) -typealias BinaryOp Union(Function, Func{2}) +call(::DotFun, x::Number, y::Number) = conj(x) * y +typealias UnaryOp Union{Function, Func{1}} +typealias BinaryOp Union{Function, Func{2}} -### sparsevec.jl +### The SparseVector ### Types @@ -53,7 +25,7 @@ immutable SparseVector{Tv,Ti<:Integer} <: AbstractSparseVector{Tv,Ti} function SparseVector(n::Integer, nzind::Vector{Ti}, nzval::Vector{Tv}) n >= 0 || throw(ArgumentError("The number of elements must be non-negative.")) length(nzind) == length(nzval) || - throw(DimensionMismatch("The lengths of nzind and nzval are inconsistent.")) + throw(ArgumentError("index and value vectors must be the same length")) new(convert(Int, n), nzind, nzval) end end @@ -70,9 +42,15 @@ countnz(x::SparseVector) = countnz(x.nzval) nonzeros(x::SparseVector) = x.nzval nonzeroinds(x::SparseVector) = x.nzind +similar{T}(x::SparseVector, ::Type{T}, D::Dims) = spzeros(T, D...) + ### Construct empty sparse vector -sparsevector{T}(::Type{T}, len::Integer) = SparseVector(len, Int[], T[]) +spzeros(len::Integer) = spzeros(Float64, len) +spzeros{T}(::Type{T}, len::Integer) = SparseVector(len, Int[], T[]) + +# Construction of same structure, but with all ones +spones{T}(x::SparseVector{T}) = SparseVector(x.n, copy(x.nzind), ones(T, length(x.nzval))) ### Construction from lists of indices and values @@ -96,13 +74,13 @@ function _sparsevector!{Tv,Ti<:Integer}(I::Vector{Ti}, V::Vector{Tv}, len::Integ # advances to the first non-zero element r = 1 # index of last processed entry while r <= m - if V[r] == zero(Tv) + if V[r] == 0 r += 1 else break end end - r > m && SparseVector(len, Ti[], Tv[]) + r > m && return SparseVector(len, Ti[], Tv[]) # move r-th to l-th l = 1 # length of processed part @@ -116,10 +94,10 @@ function _sparsevector!{Tv,Ti<:Integer}(I::Vector{Ti}, V::Vector{Tv}, len::Integ r += 1 i2 = I[r] if i2 == i # accumulate r-th to the l-th entry - V[l] = call(combine, V[l], V[r]) + V[l] = combine(V[l], V[r]) else # advance l, and move r-th to l-th pv = V[l] - if pv != zero(Tv) + if pv != 0 l += 1 end i = i2 @@ -128,7 +106,7 @@ function _sparsevector!{Tv,Ti<:Integer}(I::Vector{Ti}, V::Vector{Tv}, len::Integ end end end - if V[l] == zero(Tv) + if V[l] == 0 l -= 1 end if l < m @@ -139,9 +117,16 @@ function _sparsevector!{Tv,Ti<:Integer}(I::Vector{Ti}, V::Vector{Tv}, len::Integ SparseVector(len, I, V) end -function sparsevector{Tv,Ti<:Integer}(I::AbstractVector{Ti}, V::AbstractVector{Tv}, combine::BinaryOp) +""" + sparsevec(I, V, [m, combine]) + +Create a sparse vector `S` of length `m` such that `S[I[k]] = V[k]`. +Duplicates are combined using the `combine` function, which defaults to +`+` if it is not provided. +""" +function sparsevec{Tv,Ti<:Integer}(I::AbstractVector{Ti}, V::AbstractVector{Tv}, combine::BinaryOp) length(I) == length(V) || - throw(DimensionMismatch("The lengths of I and V are inconsistent.")) + throw(ArgumentError("index and value vectors must be the same length")) len = 0 for i in I i >= 1 || error("Index must be positive.") @@ -149,41 +134,46 @@ function sparsevector{Tv,Ti<:Integer}(I::AbstractVector{Ti}, V::AbstractVector{T len = i end end - _sparsevector!(_copy_convert(Ti, I), _copy_convert(Tv, V), len, combine) + _sparsevector!(collect(Ti, I), collect(Tv, V), len, combine) end -function sparsevector{Tv,Ti<:Integer}(I::AbstractVector{Ti}, V::AbstractVector{Tv}, len::Integer, combine::BinaryOp) +function sparsevec{Tv,Ti<:Integer}(I::AbstractVector{Ti}, V::AbstractVector{Tv}, len::Integer, combine::BinaryOp) length(I) == length(V) || - throw(DimensionMismatch("The lengths of I and V are inconsistent.")) + throw(ArgumentError("index and value vectors must be the same length")) maxi = convert(Ti, len) for i in I - 1 <= i <= maxi || error("An index is out of bound.") + 1 <= i <= maxi || throw(ArgumentError("An index is out of bound.")) end - _sparsevector!(_copy_convert(Ti, I), _copy_convert(Tv, V), len, combine) + _sparsevector!(collect(Ti, I), collect(Tv, V), len, combine) end -sparsevector{Ti<:Integer}(I::AbstractVector{Ti}, V::AbstractVector) = - sparsevector(I, V, AddFun()) +sparsevec{Ti<:Integer}(I::AbstractVector{Ti}, V::AbstractVector) = + sparsevec(I, V, AddFun()) -sparsevector{Ti<:Integer}(I::AbstractVector{Ti}, V::AbstractVector, len::Integer) = - sparsevector(I, V, len, AddFun()) +sparsevec{Ti<:Integer}(I::AbstractVector{Ti}, V::AbstractVector, len::Integer) = + sparsevec(I, V, len, AddFun()) -sparsevector{Ti<:Integer}(I::AbstractVector{Ti}, v::Number, combine::BinaryOp) = - sparsevector(I, fill(v, length(I)), combine) +sparsevec{Ti<:Integer}(I::AbstractVector{Ti}, v::Number, combine::BinaryOp) = + sparsevec(I, fill(v, length(I)), combine) -sparsevector{Ti<:Integer}(I::AbstractVector{Ti}, v::Number, len::Integer, combine::BinaryOp) = - sparsevector(I, fill(v, length(I)), len, combine) +sparsevec{Ti<:Integer}(I::AbstractVector{Ti}, v::Number, len::Integer, combine::BinaryOp) = + sparsevec(I, fill(v, length(I)), len, combine) -sparsevector{Ti<:Integer}(I::AbstractVector{Ti}, v::Number) = - sparsevector(I, v, AddFun()) +sparsevec{Ti<:Integer}(I::AbstractVector{Ti}, v::Number) = + sparsevec(I, v, AddFun()) -sparsevector{Ti<:Integer}(I::AbstractVector{Ti}, v::Number, len::Integer) = - sparsevector(I, v, len, AddFun()) +sparsevec{Ti<:Integer}(I::AbstractVector{Ti}, v::Number, len::Integer) = + sparsevec(I, v, len, AddFun()) ### Construction from dictionary +""" + sparsevec(D::Dict, [m]) -function sparsevector{Tv,Ti<:Integer}(dict::Associative{Ti,Tv}) +Create a sparse vector of length `m` where the nonzero indices are keys from +the dictionary, and the nonzero values are the values from the dictionary. +""" +function sparsevec{Tv,Ti<:Integer}(dict::Associative{Ti,Tv}) m = length(dict) nzind = Array(Ti, m) nzval = Array(Tv, m) @@ -191,11 +181,11 @@ function sparsevector{Tv,Ti<:Integer}(dict::Associative{Ti,Tv}) cnt = 0 len = zero(Ti) for (k, v) in dict - k >= 1 || error("Index must be positive.") + k >= 1 || throw(ArgumentError("index must be positive.")) if k > len len = k end - if v != zero(v) + if v != 0 cnt += 1 @inbounds nzind[cnt] = k @inbounds nzval[cnt] = v @@ -206,7 +196,7 @@ function sparsevector{Tv,Ti<:Integer}(dict::Associative{Ti,Tv}) _sparsevector!(nzind, nzval, len) end -function sparsevector{Tv,Ti<:Integer}(dict::Associative{Ti,Tv}, len::Integer) +function sparsevec{Tv,Ti<:Integer}(dict::Associative{Ti,Tv}, len::Integer) m = length(dict) nzind = Array(Ti, m) nzval = Array(Tv, m) @@ -214,8 +204,8 @@ function sparsevector{Tv,Ti<:Integer}(dict::Associative{Ti,Tv}, len::Integer) cnt = 0 maxk = convert(Ti, len) for (k, v) in dict - 1 <= k <= maxk || error("An index (key) is out of bound.") - if v != zero(v) + 1 <= k <= maxk || throw(ArgumentError("an index (key) is out of bound.")) + if v != 0 cnt += 1 @inbounds nzind[cnt] = k @inbounds nzval[cnt] = v @@ -230,20 +220,21 @@ end ### Element access function setindex!{Tv,Ti<:Integer}(x::SparseVector{Tv,Ti}, v::Tv, i::Ti) + checkbounds(x, i) nzind = nonzeroinds(x) nzval = nonzeros(x) m = length(nzind) k = searchsortedfirst(nzind, i) if 1 <= k <= m && nzind[k] == i # i found - if v == zero(v) + if v == 0 deleteat!(nzind, k) deleteat!(nzval, k) else nzval[k] = v end else # i not found - if v != zero(v) + if v != 0 insert!(nzind, k, i) insert!(nzval, k, v) end @@ -271,16 +262,26 @@ convert{Tv,Ti}(::Type{SparseVector}, s::SparseMatrixCSC{Tv,Ti}) = # convert Vector to SparseVector -function _dense2sparsevec{Tv}(s::Vector{Tv}, initcap::Int) - # pre-condition: initcap > 0 +""" + sparsevec(A) + +Convert a vector `A` into a sparse vector of length `m`. +""" +sparsevec{T}(a::AbstractVector{T}) = convert(SparseVector{T, Int}, a) +sparsevec(a::AbstractArray) = sparsevec(vec(a)) +sparsevec(a::AbstractSparseArray) = vec(a) +sparse(a::AbstractVector) = sparsevec(a) + +function _dense2sparsevec{Tv,Ti}(s::AbstractArray{Tv}, initcap::Ti) + # pre-condition: initcap > 0; the initcap determines the index type n = length(s) cap = initcap - nzind = Array(Int, cap) + nzind = Array(Ti, cap) nzval = Array(Tv, cap) c = 0 @inbounds for i = 1:n v = s[i] - if v != zero(v) + if v != 0 if c >= cap cap *= 2 resize!(nzind, cap) @@ -298,13 +299,13 @@ function _dense2sparsevec{Tv}(s::Vector{Tv}, initcap::Int) SparseVector(n, nzind, nzval) end -convert{Tv}(::Type{SparseVector{Tv,Int}}, s::Vector{Tv}) = - _dense2sparsevec(s, max(8, div(length(s), 8))) +convert{Tv,Ti}(::Type{SparseVector{Tv,Ti}}, s::AbstractVector{Tv}) = + _dense2sparsevec(s, convert(Ti, max(8, div(length(s), 8)))) -convert{Tv}(::Type{SparseVector{Tv}}, s::Vector{Tv}) = +convert{Tv}(::Type{SparseVector{Tv}}, s::AbstractVector{Tv}) = convert(SparseVector{Tv,Int}, s) -convert{Tv}(::Type{SparseVector}, s::Vector{Tv}) = +convert{Tv}(::Type{SparseVector}, s::AbstractVector{Tv}) = convert(SparseVector{Tv,Int}, s) @@ -317,101 +318,136 @@ convert{Tv,TvS,TiS}(::Type{SparseVector{Tv}}, s::SparseVector{TvS,TiS}) = ### Rand Construction - -function sprand{T}(n::Integer, p::FloatingPoint, rfn::Function, ::Type{T}) - I = randsubseq(1:convert(Int, n), p) - V = rfn(T, length(I)) +sprand{T}(n::Integer, p::AbstractFloat, rfn::Function, ::Type{T}) = sprand(GLOBAL_RNG, n, p, rfn, T) +function sprand{T}(r::AbstractRNG, n::Integer, p::AbstractFloat, rfn::Function, ::Type{T}) + I = randsubseq(r, 1:convert(Int, n), p) + V = rfn(r, T, length(I)) SparseVector(n, I, V) end -function sprand(n::Integer, p::FloatingPoint, rfn::Function) - I = randsubseq(1:convert(Int, n), p) - V = rfn(length(I)) +sprand(n::Integer, p::AbstractFloat, rfn::Function) = sprand(GLOBAL_RNG, n, p, rfn) +function sprand(r::AbstractRNG, n::Integer, p::AbstractFloat, rfn::Function) + I = randsubseq(r, 1:convert(Int, n), p) + V = rfn(r, length(I)) SparseVector(n, I, V) end -sprand{T}(n::Integer, p::FloatingPoint, ::Type{T}) = sprand(n, p, rand, T) -sprand(n::Integer, p::FloatingPoint) = sprand(n, p, rand) -sprandn(n::Integer, p::FloatingPoint) = sprand(n, p, randn) - - -### sparsevecview.jl +sprand{T}(n::Integer, p::AbstractFloat, ::Type{T}) = sprand(GLOBAL_RNG, n, p, rand, T) +sprand(n::Integer, p::AbstractFloat) = sprand(GLOBAL_RNG, n, p, rand) +sprand{T}(r::AbstractRNG, n::Integer, p::AbstractFloat, ::Type{T}) = sprand(r, n, p, rand, T) +sprand(r::AbstractRNG, n::Integer, p::AbstractFloat) = sprand(r, n, p, rand) -using ArrayViews -import ArrayViews: view +sprandn(n::Integer, p::AbstractFloat) = sprand(GLOBAL_RNG, n, p, randn) +sprandn(r::AbstractRNG, n::Integer, p::AbstractFloat) = sprand(r, n, p, randn) -typealias CVecView{T} ContiguousView{T,1,Vector{T}} +sprandbool(n::Integer, p::AbstractFloat) = sprand(GLOBAL_RNG, n, p, truebools) +sprandbool(r::AbstractRNG, n::Integer, p::AbstractFloat) = sprand(r, n, p, truebools) -immutable SparseVectorView{Tv,Ti<:Integer} <: AbstractSparseVector{Tv,Ti} - n::Int # the number of elements - nzind::CVecView{Ti} # the indices of nonzeros - nzval::CVecView{Tv} # the values of nonzeros +## Indexing into Matrices can return SparseVectors - function SparseVectorView(n::Integer, nzind::CVecView{Ti}, nzval::CVecView{Tv}) - n >= 0 || throw(ArgumentError("The number of elements must be non-negative.")) - length(nzind) == length(nzval) || - throw(DimensionMismatch("The lengths of nzind and nzval are inconsistent.")) - new(convert(Int, n), nzind, nzval) - end +function getindex(x::SparseMatrixCSC, ::Colon, j::Integer) + checkbounds(x, :, j) + r1 = convert(Int, x.colptr[j]) + r2 = convert(Int, x.colptr[j+1]) - 1 + SparseVector(x.m, x.rowval[r1:r2], x.nzval[r1:r2]) end -### Construction - -SparseVectorView{Tv,Ti}(n::Integer, nzind::CVecView{Ti}, nzval::CVecView{Tv}) = - SparseVectorView{Tv,Ti}(n, nzind, nzval) - -view(x::AbstractSparseVector) = - SparseVectorView(length(x), view(nonzeroinds(x)), view(nonzeros(x))) - -### Basic properties - -length(x::SparseVectorView) = x.n -size(x::SparseVectorView) = (x.n,) -nnz(x::SparseVectorView) = length(x.nzval) -countnz(x::SparseVectorView) = countnz(x.nzval) -nonzeros(x::SparseVectorView) = x.nzval -nonzeroinds(x::SparseVectorView) = x.nzind +function getindex(x::SparseMatrixCSC, I::UnitRange, j::Integer) + checkbounds(x, I, j) + # Get the selected column + c1 = convert(Int, x.colptr[j]) + c2 = convert(Int, x.colptr[j+1]) - 1 + # Restrict to the selected rows + r1 = searchsortedfirst(x.rowval, first(I), c1, c2, Forward) + r2 = searchsortedlast(x.rowval, last(I), c1, c2, Forward) + SparseVector(length(I), x.rowval[r1:r2] - first(I) + 1, x.nzval[r1:r2]) +end -### sparsematview.jl +# In the general case, we piggy back upon SparseMatrixCSC's optimized solution +@inline function getindex{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, I::AbstractVector, J::Integer) + M = A[I, [J]] + SparseVector(M.m, M.rowval, M.nzval) +end -### Views +# Logical and linear indexing into SparseMatrices +getindex{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractVector{Bool}) = _logical_index(A, I) # Ambiguities +getindex{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractArray{Bool}) = _logical_index(A, I) +function _logical_index{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractArray{Bool}) + checkbounds(A, I) + n = sum(I) -jvec_rgn(x::Vector, first::Int, n::Int) = pointer_to_array(pointer(x, first), n, false) + colptrA = A.colptr; rowvalA = A.rowval; nzvalA = A.nzval + rowvalB = Array(Int, n) + nzvalB = Array(Tv, n) + c = 1 + rowB = 1 -as_jvec{T}(x::ContiguousView{T,1,Vector{T}}) = pointer_to_array(pointer(x), length(x), false) + @inbounds for col in 1:A.n + r1 = colptrA[col] + r2 = colptrA[col+1]-1 -function view(x::SparseMatrixCSC, ::Colon, j::Integer) - 1 <= j <= x.n || throw(BoundsError()) - r1 = convert(Int, x.colptr[j]) - r2 = convert(Int, x.colptr[j+1]) - 1 - rgn = r1:r2 - SparseVectorView(x.m, view(x.rowval, rgn), view(x.nzval, rgn)) + for row in 1:A.m + if I[row, col] + while (r1 <= r2) && (rowvalA[r1] < row) + r1 += 1 + end + if (r1 <= r2) && (rowvalA[r1] == row) + nzvalB[c] = nzvalA[r1] + rowvalB[c] = rowB + c += 1 + end + rowB += 1 + (rowB > n) && break + end + end + (rowB > n) && break + end + n = length(nzvalB) + if n > (c-1) + deleteat!(nzvalB, c:n) + deleteat!(rowvalB, c:n) + end + SparseVector(n, rowvalB, nzvalB) end -function getcol(x::SparseMatrixCSC, j::Integer) - 1 <= j <= x.n || throw(BoundsError()) - r1 = convert(Int, x.colptr[j]) - r2 = convert(Int, x.colptr[j+1]) - 1 - SparseVector(x.m, x.rowval[r1:r2], x.nzval[r1:r2]) -end +# TODO: huge optimizations are available for I::Range and ::Colon +getindex(A::SparseMatrixCSC, ::Colon) = A[1:end] +function getindex{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractVector) + szA = size(A) + nA = szA[1]*szA[2] + colptrA = A.colptr + rowvalA = A.rowval + nzvalA = A.nzval -function unsafe_colrange{Tv,Ti}(x::SparseMatrixCSC{Tv,Ti}, J::UnitRange) - jfirst = first(J) - jlast = last(J) - (1 <= jfirst <= x.n && jlast <= x.n) || throw(BoundsError()) - r1 = x.colptr[jfirst] - r2 = x.colptr[jlast+1] - one(r1) - newcolptr = view(x.colptr, jfirst:jlast+1) - (r1 - one(r1)) - - fi = convert(Int, r1) - nc = convert(Int, r2 - r1) + 1 - SparseMatrixCSC{Tv, Ti}(x.m, length(J), newcolptr, - jvec_rgn(x.rowval, fi, nc), jvec_rgn(x.nzval, fi, nc)) + n = length(I) + rowvalB = Array(Int, n) + nzvalB = Array(Tv, n) + + rowB = 1 + idxB = 1 + + for i in 1:n + ((I[i] < 1) | (I[i] > nA)) && throw(BoundsError(A, I)) + row,col = ind2sub(szA, I[i]) + for r in colptrA[col]:(colptrA[col+1]-1) + @inbounds if rowvalA[r] == row + rowvalB[idxB] = i + nzvalB[idxB] = nzvalA[r] + idxB += 1 + break + end + end + end + if n > (idxB-1) + deleteat!(nzvalB, idxB:n) + deleteat!(rowvalB, idxB:n) + end + SparseVector(n, rowvalB, nzvalB) end -### generics.jl -# Generic functions operating on AbstractSparseVector + +### Generic functions operating on AbstractSparseVector ### getindex @@ -420,29 +456,25 @@ function _spgetindex{Tv,Ti}(m::Int, nzind::AbstractVector{Ti}, nzval::AbstractVe (ii <= m && nzind[ii] == i) ? nzval[ii] : zero(Tv) end -getindex{Tv}(x::AbstractSparseVector{Tv}, i::Integer) = +function getindex{Tv}(x::AbstractSparseVector{Tv}, i::Integer) + checkbounds(x, i) _spgetindex(nnz(x), nonzeroinds(x), nonzeros(x), i) +end function getindex{Tv,Ti}(x::AbstractSparseVector{Tv,Ti}, I::UnitRange) + checkbounds(x, I) xlen = length(x) i0 = first(I) i1 = last(I) - (i0 >= 1 && i1 <= xlen) || throw(BoundsError()) xnzind = nonzeroinds(x) xnzval = nonzeros(x) m = length(xnzind) # locate the first j0, s.t. xnzind[j0] >= i0 - j0 = 1 - while j0 <= m && xnzind[j0] < i0 - j0 += 1 - end + j0 = searchsortedfirst(xnzind, i0) # locate the last j1, s.t. xnzind[j1] <= i1 - j1 = j0 - 1 - while j1 < m && xnzind[j1+1] <= i1 - j1 += 1 - end + j1 = searchsortedlast(xnzind, i1, j0, m, Forward) # compute the number of non-zeros jrgn = j0:j1 @@ -460,29 +492,22 @@ function getindex{Tv,Ti}(x::AbstractSparseVector{Tv,Ti}, I::UnitRange) SparseVector(length(I), rind, rval) end +getindex{Tv,Ti}(x::AbstractSparseVector{Tv,Ti}, I::AbstractVector{Bool}) = x[find(I)] +getindex{Tv,Ti}(x::AbstractSparseVector{Tv,Ti}, I::AbstractArray{Bool}) = x[find(I)] +@inline function getindex{Tv,Ti}(x::AbstractSparseVector{Tv,Ti}, I::AbstractVector) + # SparseMatrixCSC has a nicely optimized routine for this; punt + S = SparseMatrixCSC(x.n, 1, [1,length(x.nzind)+1], x.nzind, x.nzval) + S[I, 1] +end + function getindex{Tv,Ti}(x::AbstractSparseVector{Tv,Ti}, I::AbstractArray) - xnzind = nonzeroinds(x) - xnzval = nonzeros(x) - m = length(xnzind) - n = length(I) - nzind = Array(Ti, n) - nzval = Array(Tv, n) - c = 0 - for j = 1:n - v = _spgetindex(m, xnzind, xnzval, I[j]) - if v != zero(v) - c += 1 - nzind[c] = convert(Ti, j) - nzval[c] = v - end - end - if c < n - resize!(nzind, c) - resize!(nzval, c) - end - SparseVector(n, nzind, nzval) + # punt to SparseMatrixCSC + S = SparseMatrixCSC(x.n, 1, [1,length(x.nzind)+1], x.nzind, x.nzval) + S[I] end +getindex(x::AbstractSparseVector, ::Colon) = copy(x) + ### show and friends function showarray(io::IO, x::AbstractSparseVector; @@ -495,39 +520,30 @@ function showarray(io::IO, x::AbstractSparseVector; xnnz = length(nzind) if header - print(io, "Sparse vector, length = ", n, - ", with ", xnnz, " ", eltype(nzval), " entries:", "\n") + println(io, summary(x)) end half_screen_rows = limit ? div(rows - 8, 2) : typemax(Int) pad = ndigits(n) - k = 0 sep = "\n\t" for k = 1:length(nzind) if k < half_screen_rows || k > xnnz - half_screen_rows print(io, " ", '[', rpad(nzind[k], pad), "] = ") showcompact(io, nzval[k]) + println(io) elseif k == half_screen_rows - print(io, sep, '\u22ee') + println(io, " ", " "^pad, " \u22ee") end - print(io, "\n") - k += 1 end end -show(io::IO, x::AbstractSparseVector) = showarray(io, x) -writemime(io::IO, ::MIME"text/plain", x::AbstractSparseVector) = show(io, x) - - -### Comparison - -function exact_equal(x::AbstractSparseVector, y::AbstractSparseVector) - eltype(x) == eltype(y) && - eltype(nonzeroinds(x)) == eltype(nonzeroinds(y)) && - length(x) == length(y) && - nonzeroinds(x) == nonzeroinds(y) && - nonzeros(x) == nonzeros(y) +function summary(x::AbstractSparseVector) + string("Sparse vector of length ", length(x), ", with ", length(nonzeros(x)), + " ", eltype(x), " nonzero entries:") end +show(io::IO, x::AbstractSparseVector) = showarray(io, x) +writemime(io::IO, ::MIME"text/plain", x::AbstractSparseVector) = Base.with_output_limit(()->show(io, x)) + ### Conversion to matrix function convert{TvD,TiD,Tv,Ti}(::Type{SparseMatrixCSC{TvD,TiD}}, x::AbstractSparseVector{Tv,Ti}) @@ -536,8 +552,10 @@ function convert{TvD,TiD,Tv,Ti}(::Type{SparseMatrixCSC{TvD,TiD}}, x::AbstractSpa xnzval = nonzeros(x) m = length(xnzind) colptr = TiD[1, m+1] - rowval = _copy_convert(TiD, xnzind) - nzval = _copy_convert(TvD, xnzval) + # Note that this *cannot* share data like normal array conversions, since + # modifying one would put the other in an inconsistent state + rowval = collect(TiD, xnzind) + nzval = collect(TvD, xnzval) SparseMatrixCSC(n, 1, colptr, rowval, nzval) end @@ -552,6 +570,7 @@ convert{Tv,Ti}(::Type{SparseMatrixCSC}, x::AbstractSparseVector{Tv,Ti}) = function full{Tv}(x::AbstractSparseVector{Tv}) n = length(x) + n == 0 && return Array(Tv, 0) nzind = nonzeroinds(x) nzval = nonzeros(x) r = zeros(Tv, n) @@ -571,7 +590,7 @@ function reinterpret{T,Tv}(::Type{T}, x::AbstractSparseVector{Tv}) SparseVector(length(x), copy(nonzeroinds(x)), reinterpret(T, nonzeros(x))) end -float{Tv<:FloatingPoint}(x::AbstractSparseVector{Tv}) = x +float{Tv<:AbstractFloat}(x::AbstractSparseVector{Tv}) = x float(x::AbstractSparseVector) = SparseVector(length(x), copy(nonzeroinds(x)), float(nonzeros(x))) @@ -639,8 +658,7 @@ function vcat{Tv,Ti}(X::AbstractSparseVector{Tv,Ti}...) SparseVector(len, rnzind, rnzval) end -### math.jl - +### math functions ### Unary Map @@ -743,7 +761,7 @@ function _binarymap{Tx,Ty}(f::BinaryOp, mode::Int) 0 <= mode <= 2 || throw(ArgumentError("Incorrect mode $mode.")) - R = typeof(call(f, zero(Tx), zero(Ty))) + R = typeof(f(zero(Tx), zero(Ty))) n = length(x) length(y) == n || throw(DimensionMismatch()) @@ -784,7 +802,7 @@ function _binarymap_mode_0!(f::BinaryOp, mx::Int, my::Int, jx = xnzind[ix] jy = ynzind[iy] if jx == jy - v = call(f, xnzval[ix], ynzval[iy]) + v = f(xnzval[ix], ynzval[iy]) ir += 1; rind[ir] = jx; rval[ir] = v ix += 1; iy += 1 elseif jx < jy @@ -806,28 +824,28 @@ function _binarymap_mode_1!{Tx,Ty}(f::BinaryOp, mx::Int, my::Int, jx = xnzind[ix] jy = ynzind[iy] if jx == jy - v = call(f, xnzval[ix], ynzval[iy]) + v = f(xnzval[ix], ynzval[iy]) if v != zero(v) ir += 1; rind[ir] = jx; rval[ir] = v end ix += 1; iy += 1 elseif jx < jy - v = call(f, xnzval[ix], zero(Ty)) + v = f(xnzval[ix], zero(Ty)) ir += 1; rind[ir] = jx; rval[ir] = v ix += 1 else - v = call(f, zero(Tx), ynzval[iy]) + v = f(zero(Tx), ynzval[iy]) ir += 1; rind[ir] = jy; rval[ir] = v iy += 1 end end @inbounds while ix <= mx - v = call(f, xnzval[ix], zero(Ty)) + v = f(xnzval[ix], zero(Ty)) ir += 1; rind[ir] = xnzind[ix]; rval[ir] = v ix += 1 end @inbounds while iy <= my - v = call(f, zero(Tx), ynzval[iy]) + v = f(zero(Tx), ynzval[iy]) ir += 1; rind[ir] = ynzind[iy]; rval[ir] = v iy += 1 end @@ -844,19 +862,19 @@ function _binarymap_mode_2!{Tx,Ty}(f::BinaryOp, mx::Int, my::Int, jx = xnzind[ix] jy = ynzind[iy] if jx == jy - v = call(f, xnzval[ix], ynzval[iy]) + v = f(xnzval[ix], ynzval[iy]) if v != zero(v) ir += 1; rind[ir] = jx; rval[ir] = v end ix += 1; iy += 1 elseif jx < jy - v = call(f, xnzval[ix], zero(Ty)) + v = f(xnzval[ix], zero(Ty)) if v != zero(v) ir += 1; rind[ir] = jx; rval[ir] = v end ix += 1 else - v = call(f, zero(Tx), ynzval[iy]) + v = f(zero(Tx), ynzval[iy]) if v != zero(v) ir += 1; rind[ir] = jy; rval[ir] = v end @@ -864,14 +882,14 @@ function _binarymap_mode_2!{Tx,Ty}(f::BinaryOp, mx::Int, my::Int, end end @inbounds while ix <= mx - v = call(f, xnzval[ix], zero(Ty)) + v = f(xnzval[ix], zero(Ty)) if v != zero(v) ir += 1; rind[ir] = xnzind[ix]; rval[ir] = v end ix += 1 end @inbounds while iy <= my - v = call(f, zero(Tx), ynzval[iy]) + v = f(zero(Tx), ynzval[iy]) if v != zero(v) ir += 1; rind[ir] = ynzind[iy]; rval[ir] = v end @@ -886,7 +904,7 @@ function _binarymap{Tx,Ty}(f::BinaryOp, mode::Int) 0 <= mode <= 2 || throw(ArgumentError("Incorrect mode $mode.")) - R = typeof(call(f, zero(Tx), zero(Ty))) + R = typeof(f(zero(Tx), zero(Ty))) n = length(x) length(y) == n || throw(DimensionMismatch()) @@ -902,7 +920,7 @@ function _binarymap{Tx,Ty}(f::BinaryOp, while ii < j dst[ii] = zero(R); ii += 1 end - dst[j] = call(f, x[j], ynzval[i]); ii += 1 + dst[j] = f(x[j], ynzval[i]); ii += 1 end @inbounds while ii <= n dst[ii] = zero(R); ii += 1 @@ -912,12 +930,12 @@ function _binarymap{Tx,Ty}(f::BinaryOp, @inbounds for i = 1:m j = ynzind[i] while ii < j - dst[ii] = call(f, x[ii], zero(Ty)); ii += 1 + dst[ii] = f(x[ii], zero(Ty)); ii += 1 end - dst[j] = call(f, x[j], ynzval[i]); ii += 1 + dst[j] = f(x[j], ynzval[i]); ii += 1 end @inbounds while ii <= n - dst[ii] = call(f, x[ii], zero(Ty)); ii += 1 + dst[ii] = f(x[ii], zero(Ty)); ii += 1 end end return dst @@ -929,7 +947,7 @@ function _binarymap{Tx,Ty}(f::BinaryOp, mode::Int) 0 <= mode <= 2 || throw(ArgumentError("Incorrect mode $mode.")) - R = typeof(call(f, zero(Tx), zero(Ty))) + R = typeof(f(zero(Tx), zero(Ty))) n = length(x) length(y) == n || throw(DimensionMismatch()) @@ -945,7 +963,7 @@ function _binarymap{Tx,Ty}(f::BinaryOp, while ii < j dst[ii] = zero(R); ii += 1 end - dst[j] = call(f, xnzval[i], y[j]); ii += 1 + dst[j] = f(xnzval[i], y[j]); ii += 1 end @inbounds while ii <= n dst[ii] = zero(R); ii += 1 @@ -955,12 +973,12 @@ function _binarymap{Tx,Ty}(f::BinaryOp, @inbounds for i = 1:m j = xnzind[i] while ii < j - dst[ii] = call(f, zero(Tx), y[ii]); ii += 1 + dst[ii] = f(zero(Tx), y[ii]); ii += 1 end - dst[j] = call(f, xnzval[i], y[j]); ii += 1 + dst[j] = f(xnzval[i], y[j]); ii += 1 end @inbounds while ii <= n - dst[ii] = call(f, zero(Tx), y[ii]); ii += 1 + dst[ii] = f(zero(Tx), y[ii]); ii += 1 end end return dst @@ -979,14 +997,6 @@ for (vop, fun, mode) in [(:_vadd, :AddFun, 1), end end -# to workaround the ambiguities with vectorized dates/arithmetic.jl functions -if VERSION > v"0.4-dev" - +{T<:Dates.TimeType,P<:Dates.GeneralPeriod}(x::StridedVector{P}, y::AbstractSparseVector{T}) = _vadd(x, y) - -{T<:Dates.TimeType,P<:Dates.GeneralPeriod}(x::StridedVector{P}, y::AbstractSparseVector{T}) = _vsub(x, y) - +{T<:Dates.TimeType,P<:Dates.GeneralPeriod}(x::AbstractSparseVector{T}, y::StridedVector{P}) = _vadd(x, y) - -{T<:Dates.TimeType,P<:Dates.GeneralPeriod}(x::AbstractSparseVector{T}, y::StridedVector{P}) = _vsub(x, y) -end - # to workaround the ambiguities with BitVector .*(x::BitVector, y::AbstractSparseVector{Bool}) = _vmul(x, y) .*(x::AbstractSparseVector{Bool}, y::BitVector) = _vmul(x, y) @@ -1049,6 +1059,20 @@ vecnorm(x::AbstractSparseVector, p::Real=2) = vecnorm(nonzeros(x), p) ### linalg.jl +# Transpose +# (The only sparse matrix structure in base is CSC, so a one-row sparse matrix is worse than dense) +transpose(x::SparseVector) = _ct(IdFun(), x) +ctranspose(x::SparseVector) = _ct(ConjFun(), x) +function _ct{T}(f, x::SparseVector{T}) + isempty(x) && return Array(T, 1, 0) + A = zeros(T, 1, length(x)) + xnzind = nonzeroinds(x) + xnzval = nonzeros(x) + for (i,v) in zip(xnzind, xnzval) + @inbounds A[i] = f(v) + end + A +end ### BLAS Level-1 @@ -1113,7 +1137,7 @@ function dot{Tx<:Number,Ty<:Number}(x::StridedVector{Tx}, y::AbstractSparseVecto nzval = nonzeros(y) s = zero(Tx) * zero(Ty) for i = 1:length(nzind) - s += _dot(x[nzind[i]], nzval[i]) + s += conj(x[nzind[i]]) * nzval[i] end return s end @@ -1125,7 +1149,7 @@ function dot{Tx<:Number,Ty<:Number}(x::AbstractSparseVector{Tx}, y::AbstractVect nzval = nonzeros(x) s = zero(Tx) * zero(Ty) for i = 1:length(nzind) - s += _dot(nzval[i], y[nzind[i]]) + s += conj(nzval[i]) * y[nzind[i]] end return s end @@ -1139,7 +1163,7 @@ function _spdot(f::BinaryOp, ix = xnzind[xj] iy = ynzind[yj] if ix == iy - s += call(f, xnzval[xj], ynzval[yj]) + s += f(xnzval[xj], ynzval[yj]) xj += 1 yj += 1 elseif ix < iy diff --git a/base/sparse/spqr.jl b/base/sparse/spqr.jl index aa32492695213..aed43b4dbf08e 100644 --- a/base/sparse/spqr.jl +++ b/base/sparse/spqr.jl @@ -38,12 +38,12 @@ const RTX_EQUALS_B = Int32(2) # solve R'*X=B or X = R'\B const RTX_EQUALS_ETB = Int32(3) # solve R'*X=E'*B or X = R'\(E'*B) -using Base.SparseMatrix: SparseMatrixCSC -using Base.SparseMatrix.CHOLMOD: C_Dense, C_Sparse, Dense, ITypes, Sparse, VTypes, common +using ..SparseArrays: SparseMatrixCSC +using ..SparseArrays.CHOLMOD: C_Dense, C_Sparse, Dense, ITypes, Sparse, VTypes, common import Base: size import Base.LinAlg: qrfact -import Base.SparseMatrix.CHOLMOD: convert, free! +import ..SparseArrays.CHOLMOD: convert, free! diff --git a/base/sparse/umfpack.jl b/base/sparse/umfpack.jl index 6310fb8f1d0a5..9ed382a8a282e 100644 --- a/base/sparse/umfpack.jl +++ b/base/sparse/umfpack.jl @@ -7,8 +7,8 @@ export UmfpackLU import Base: (\), Ac_ldiv_B, At_ldiv_B, findnz, getindex, show, size import Base.LinAlg: A_ldiv_B!, Ac_ldiv_B!, At_ldiv_B!, Factorization, det, lufact -importall Base.SparseMatrix -import Base.SparseMatrix: increment, increment!, decrement, decrement!, nnz +importall ..SparseArrays +import ..SparseArrays: increment, increment!, decrement, decrement!, nnz include("umfpack_h.jl") type MatrixIllConditionedException <: Exception diff --git a/base/sysimg.jl b/base/sysimg.jl index c555a41cf6d6c..5568676d519d1 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -248,10 +248,6 @@ importall .Broadcast # statistics include("statistics.jl") -# sparse matrices and sparse linear algebra -include("sparse.jl") -importall .SparseMatrix - # irrational mathematical constants include("irrationals.jl") @@ -288,6 +284,33 @@ importall .Profile include("Dates.jl") import .Dates: Date, DateTime, now +# sparse matrices, vectors, and sparse linear algebra +include("sparse.jl") +importall .SparseArrays +# SparseArrays module re-exports +export + SparseArrays, + AbstractSparseArray, + AbstractSparseMatrix, + AbstractSparseVector, + SparseMatrixCSC, + SparseVector, + etree, + issparse, + sparse, + sparsevec, + spdiagm, + speye, + spones, + sprand, + sprandbool, + sprandn, + spzeros, + symperm, + rowvals, + nzrange, + nnz + # Documentation include("markdown/Markdown.jl") diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index af88e9342a287..058a189a2a585 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -909,10 +909,11 @@ BitArrays .. _stdlib-sparse: -Sparse Matrices ---------------- +Sparse Vectors and Matrices +--------------------------- -Sparse matrices support much of the same set of operations as dense matrices. The following functions are specific to sparse matrices. +Sparse vectors and matrices largely support the same set of operations as their +dense counterparts. The following functions are specific to sparse arrays. .. function:: sparse(I,J,V,[m,n,combine]) @@ -924,13 +925,13 @@ Sparse matrices support much of the same set of operations as dense matrices. Th .. Docstring generated from Julia source - Create a sparse matrix ``S`` of size ``m x 1`` such that ``S[I[k]] = V[k]``\ . Duplicates are combined using the ``combine`` function, which defaults to ``+`` if it is not provided. In julia, sparse vectors are really just sparse matrices with one column. Given Julia's Compressed Sparse Columns (CSC) storage format, a sparse column matrix with one column is sparse, whereas a sparse row matrix with one row ends up being dense. + Create a sparse vector ``S`` of length ``m`` such that ``S[I[k]] = V[k]``\ . Duplicates are combined using the ``combine`` function, which defaults to ``+`` if it is not provided. .. function:: sparsevec(D::Dict, [m]) .. Docstring generated from Julia source - Create a sparse matrix of size ``m x 1`` where the row values are keys from the dictionary, and the nonzero values are the values from the dictionary. + Create a sparse vector of length ``m`` where the nonzero indices are keys from the dictionary, and the nonzero values are the values from the dictionary. .. function:: issparse(S) @@ -948,31 +949,31 @@ Sparse matrices support much of the same set of operations as dense matrices. Th .. Docstring generated from Julia source - Convert a dense vector ``A`` into a sparse matrix of size ``m x 1``\ . In julia, sparse vectors are really just sparse matrices with one column. + Convert a vector ``A`` into a sparse vector of length ``m``\ . .. function:: full(S) .. Docstring generated from Julia source - Convert a sparse matrix ``S`` into a dense matrix. + Convert a sparse matrix or vector ``S`` into a dense matrix or vector. .. function:: nnz(A) .. Docstring generated from Julia source - Returns the number of stored (filled) elements in a sparse matrix. + Returns the number of stored (filled) elements in a sparse array. -.. function:: spzeros(m,n) +.. function:: spzeros(m[,n]) .. Docstring generated from Julia source - Create a sparse matrix of size ``m x n``\ . This sparse matrix will not contain any nonzero values. No storage will be allocated for nonzero values during construction. + Create a sparse vector of length ``m`` or sparse matrix of size ``m x n``\ . This sparse array will not contain any nonzero values. No storage will be allocated for nonzero values during construction. .. function:: spones(S) .. Docstring generated from Julia source - Create a sparse matrix with the same structure as that of ``S``\ , but with every nonzero element having the value ``1.0``\ . + Create a sparse array with the same structure as that of ``S``\ , but with every nonzero element having the value ``1.0``\ . .. function:: speye(type,m[,n]) @@ -986,28 +987,29 @@ Sparse matrices support much of the same set of operations as dense matrices. Th Construct a sparse diagonal matrix. ``B`` is a tuple of vectors containing the diagonals and ``d`` is a tuple containing the positions of the diagonals. In the case the input contains only one diagonal, ``B`` can be a vector (instead of a tuple) and ``d`` can be the diagonal position (instead of a tuple), defaulting to 0 (diagonal). Optionally, ``m`` and ``n`` specify the size of the resulting sparse matrix. -.. function:: sprand([rng,] m,n,p [,rfn]) +.. function:: sprand([rng],m,[n],p::AbstractFloat,[rfn]) .. Docstring generated from Julia source - Create a random ``m`` by ``n`` sparse matrix, in which the probability of any - element being nonzero is independently given by ``p`` (and hence the mean - density of nonzeros is also exactly ``p``). Nonzero values are sampled from - the distribution specified by ``rfn``. The uniform distribution is used in - case ``rfn`` is not specified. The optional ``rng`` argument specifies a - random number generator, see :ref:`Random Numbers `. + Create a random length ``m`` sparse vector or ``m`` by ``n`` sparse matrix, in + which the probability of any element being nonzero is independently given by + ``p`` (and hence the mean density of nonzeros is also exactly ``p``). Nonzero + values are sampled from the distribution specified by ``rfn``. The uniform + distribution is used in case ``rfn`` is not specified. The optional ``rng`` + argument specifies a random number generator, see :ref:`Random Numbers + `. -.. function:: sprandn(m,n,p) +.. function:: sprandn(m[,n],p::AbstractFloat) .. Docstring generated from Julia source - Create a random ``m`` by ``n`` sparse matrix with the specified (independent) probability ``p`` of any entry being nonzero, where nonzero values are sampled from the normal distribution. + Create a random sparse vector of length ``m`` or sparse matrix of size ``m`` by ``n`` with the specified (independent) probability ``p`` of any entry being nonzero, where nonzero values are sampled from the normal distribution. -.. function:: sprandbool(m,n,p) +.. function:: sprandbool(m[,n],p) .. Docstring generated from Julia source - Create a random ``m`` by ``n`` sparse boolean matrix with the specified (independent) probability ``p`` of any entry being ``true``\ . + Create a random ``m`` by ``n`` sparse boolean matrix or length ``m`` sparse boolean vector with the specified (independent) probability ``p`` of any entry being ``true``\ . .. function:: etree(A[, post]) @@ -1019,21 +1021,21 @@ Sparse matrices support much of the same set of operations as dense matrices. Th .. Docstring generated from Julia source - Return the symmetric permutation of ``A``\ , which is ``A[p,p]``\ . ``A`` should be symmetric and sparse, where only the upper triangular part of the matrix is stored. This algorithm ignores the lower triangular part of the matrix. Only the upper triangular part of the result is returned as well. + Return the symmetric permutation of ``A``\ , which is ``A[p,p]``\ . ``A`` should be symmetric, sparse, and only contain nonzeros in the upper triangular part of the matrix is stored. This algorithm ignores the lower triangular part of the matrix. Only the upper triangular part of the result is returned. .. function:: nonzeros(A) .. Docstring generated from Julia source - Return a vector of the structural nonzero values in sparse matrix ``A``\ . This includes zeros that are explicitly stored in the sparse matrix. The returned vector points directly to the internal nonzero storage of ``A``\ , and any modifications to the returned vector will mutate ``A`` as well. See ``rowvals(A)`` and ``nzrange(A, col)``\ . + Return a vector of the structural nonzero values in sparse array ``A``\ . This includes zeros that are explicitly stored in the sparse array. The returned vector points directly to the internal nonzero storage of ``A``\ , and any modifications to the returned vector will mutate ``A`` as well. See ``rowvals(A)`` and ``nzrange(A, col)``\ . -.. function:: rowvals(A) +.. function:: rowvals(A::SparseMatrixCSC) .. Docstring generated from Julia source - Return a vector of the row indices of ``A``\ , and any modifications to the returned vector will mutate ``A`` as well. Given the internal storage format of sparse matrices, providing access to how the row indices are stored internally can be useful in conjuction with iterating over structural nonzero values. See ``nonzeros(A)`` and ``nzrange(A, col)``\ . + Return a vector of the row indices of ``A``\ . Any modifications to the returned vector will mutate ``A`` as well. Providing access to how the row indices are stored internally can be useful in conjuction with iterating over structural nonzero values. See also ``nonzeros(A)`` and ``nzrange(A, col)``\ . -.. function:: nzrange(A, col) +.. function:: nzrange(A::SparseMatrixCSC, col) .. Docstring generated from Julia source diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 591222d739e54..76135a1999b08 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -149,7 +149,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Compute the ``LDLt`` factorization of a sparse symmetric or Hermitian matrix. A fill-reducing permutation is used. ``F = ldltfact(A)`` is most frequently used to solve systems of equations ``A*x = b`` with ``F\b``\ , but also the methods ``diag``\ , ``det``\ , ``logdet`` are defined for ``F``\ . You can also extract individual factors from ``F``\ , using ``F[:L]``\ . However, since pivoting is on by default, the factorization is internally represented as ``A == P'*L*D*L'*P`` with a permutation matrix ``P``\ ; using just ``L`` without accounting for ``P`` will give incorrect answers. To include the effects of permutation, it's typically preferable to extact "combined" factors like ``PtL = F[:PtL]`` (the equivalent of ``P'*L``\ ) and ``LtP = F[:UP]`` (the equivalent of ``L'*P``\ ). The complete list of supported factors is ``:L, :PtL, :D, :UP, :U, :LD, :DU, :PtLD, :DUP``\ . + Compute the ``LDLt`` factorization of a sparse symmetric or Hermitian matrix. A fill-reducing permutation is used. ``F = ldltfact(A)`` is most frequently used to solve systems of equations ``A*x = b`` with ``F\b``\ . The returned factorization object ``F`` also supports the methods ``diag``\ , ``det``\ , and ``logdet``\ . You can extract individual factors from ``F`` using ``F[:L]``\ . However, since pivoting is on by default, the factorization is internally represented as ``A == P'*L*D*L'*P`` with a permutation matrix ``P``\ ; using just ``L`` without accounting for ``P`` will give incorrect answers. To include the effects of permutation, it's typically preferable to extact "combined" factors like ``PtL = F[:PtL]`` (the equivalent of ``P'*L``\ ) and ``LtP = F[:UP]`` (the equivalent of ``L'*P``\ ). The complete list of supported factors is ``:L, :PtL, :D, :UP, :U, :LD, :DU, :PtLD, :DUP``\ . Setting optional ``shift`` keyword argument computes the factorization of ``A+shift*I`` instead of ``A``\ . If the ``perm`` argument is nonempty, it should be a permutation of ``1:size(A,1)`` giving the ordering to use (instead of CHOLMOD's default AMD ordering). diff --git a/test/sparse.jl b/test/sparse.jl index 79dcd9c27f413..c57b889fa11f8 100644 --- a/test/sparse.jl +++ b/test/sparse.jl @@ -1,6 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license include("sparsedir/sparse.jl") +include("sparsedir/sparsevector.jl") if Base.USE_GPL_LIBS include("sparsedir/umfpack.jl") include("sparsedir/cholmod.jl") diff --git a/test/sparsedir/cholmod.jl b/test/sparsedir/cholmod.jl index 8026c0039bf9e..37363dafcdf1f 100644 --- a/test/sparsedir/cholmod.jl +++ b/test/sparsedir/cholmod.jl @@ -3,7 +3,7 @@ srand(123) using Base.Test -using Base.SparseMatrix.CHOLMOD +using Base.SparseArrays.CHOLMOD # based on deps/SuiteSparse-4.0.2/CHOLMOD/Demo/ @@ -318,8 +318,8 @@ for elty in (Float64, Complex{Float64}) A1pdSparse = CHOLMOD.Sparse( A1pd.m, A1pd.n, - Base.SparseMatrix.decrement(A1pd.colptr), - Base.SparseMatrix.decrement(A1pd.rowval), + Base.SparseArrays.decrement(A1pd.colptr), + Base.SparseArrays.decrement(A1pd.rowval), A1pd.nzval) ## High level interface @@ -564,7 +564,7 @@ b = rand(3) Asp = As[p,p] LDp = sparse(ldltfact(Asp, perm=[1,2,3])[:LD]) # LDp = sparse(Fs[:LD]) -Lp, dp = Base.SparseMatrix.CHOLMOD.getLd!(copy(LDp)) +Lp, dp = Base.SparseArrays.CHOLMOD.getLd!(copy(LDp)) Dp = spdiagm(dp) @test_approx_eq Fs\b Af\b @test_approx_eq Fs[:UP]\(Fs[:PtLD]\b) Af\b diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 1504d6d3e6c79..27e363354888b 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -190,8 +190,8 @@ for i = 1:5 a = sprand(10, 5, 0.7) b = sprand(5, 15, 0.3) @test maximum(abs(a*b - full(a)*full(b))) < 100*eps() - @test maximum(abs(Base.SparseMatrix.spmatmul(a,b,sortindices=:sortcols) - full(a)*full(b))) < 100*eps() - @test maximum(abs(Base.SparseMatrix.spmatmul(a,b,sortindices=:doubletranspose) - full(a)*full(b))) < 100*eps() + @test maximum(abs(Base.SparseArrays.spmatmul(a,b,sortindices=:sortcols) - full(a)*full(b))) < 100*eps() + @test maximum(abs(Base.SparseArrays.spmatmul(a,b,sortindices=:doubletranspose) - full(a)*full(b))) < 100*eps() @test full(kron(a,b)) == kron(full(a), full(b)) @test full(kron(full(a),b)) == kron(full(a), full(b)) @test full(kron(a,full(b))) == kron(full(a), full(b)) @@ -381,18 +381,18 @@ for (aa116, ss116) in [(a116, s116), (ad116, sd116)] # range indexing @test full(ss116[i,:]) == aa116[i,:] - @test full(ss116[:,j]) == aa116[:,j]'' # sparse matrices/vectors always have ndims==2: + @test full(ss116[:,j]) == aa116[:,j] @test full(ss116[i,1:2:end]) == aa116[i,1:2:end] - @test full(ss116[1:2:end,j]) == aa116[1:2:end,j]'' + @test full(ss116[1:2:end,j]) == aa116[1:2:end,j] @test full(ss116[i,end:-2:1]) == aa116[i,end:-2:1] - @test full(ss116[end:-2:1,j]) == aa116[end:-2:1,j]'' + @test full(ss116[end:-2:1,j]) == aa116[end:-2:1,j] # float-range indexing is not supported # sorted vector indexing @test full(ss116[i,[3:2:end-3;]]) == aa116[i,[3:2:end-3;]] - @test full(ss116[[3:2:end-3;],j]) == aa116[[3:2:end-3;],j]'' + @test full(ss116[[3:2:end-3;],j]) == aa116[[3:2:end-3;],j] @test full(ss116[i,[end-3:-2:1;]]) == aa116[i,[end-3:-2:1;]] - @test full(ss116[[end-3:-2:1;],j]) == aa116[[end-3:-2:1;],j]'' + @test full(ss116[[end-3:-2:1;],j]) == aa116[[end-3:-2:1;],j] # unsorted vector indexing with repetition p = [4, 1, 2, 3, 2, 6] @@ -403,7 +403,7 @@ for (aa116, ss116) in [(a116, s116), (ad116, sd116)] # bool indexing li = bitrand(size(aa116,1)) lj = bitrand(size(aa116,2)) - @test full(ss116[li,j]) == aa116[li,j]'' + @test full(ss116[li,j]) == aa116[li,j] @test full(ss116[li,:]) == aa116[li,:] @test full(ss116[i,lj]) == aa116[i,lj] @test full(ss116[:,lj]) == aa116[:,lj] @@ -412,7 +412,7 @@ for (aa116, ss116) in [(a116, s116), (ad116, sd116)] # empty indices for empty in (1:0, Int[]) @test full(ss116[empty,:]) == aa116[empty,:] - @test full(ss116[:,empty]) == aa116[:,empty]'' + @test full(ss116[:,empty]) == aa116[:,empty] @test full(ss116[empty,lj]) == aa116[empty,lj] @test full(ss116[li,empty]) == aa116[li,empty] @test full(ss116[empty,empty]) == aa116[empty,empty] @@ -445,7 +445,7 @@ S1290 = SparseMatrixCSC(3, 3, UInt8[1,1,1,1], UInt8[], Int64[]) S1290[end] = 3 @test S1290[end] == (S1290[1] + S1290[2,2]) @test 6 == sum(diag(S1290)) - @test (full(S1290)[[3,1],1])'' == full(S1290[[3,1],1]) + @test full(S1290)[[3,1],1] == full(S1290[[3,1],1]) # end @@ -457,7 +457,7 @@ let a = spzeros(Int, 10, 10) @test a[1,:] == sparse(ones(Int,1,10)) a[:,2] = 2 @test countnz(a) == 19 - @test a[:,2] == 2*sparse(ones(Int,10,1)) + @test a[:,2] == 2*sparse(ones(Int,10)) a[1,:] = 1:10 @test a[1,:] == sparse([1:10;]') @@ -500,9 +500,9 @@ let ASZ = 1000, TSZ = 800 end let A = speye(Int, 5), I=1:10, X=reshape([trues(10); falses(15)],5,5) - @test A[I] == A[X] == reshape([1,0,0,0,0,0,1,0,0,0], 10, 1) + @test A[I] == A[X] == [1,0,0,0,0,0,1,0,0,0] A[I] = [1:10;] - @test A[I] == A[X] == reshape(1:10, 10, 1) + @test A[I] == A[X] == collect(1:10) end let S = sprand(50, 30, 0.5, x->round(Int,rand(x)*100)), I = sprandbool(50, 30, 0.2) @@ -540,7 +540,7 @@ let S = sprand(50, 30, 0.5, x->round(Int,rand(x)*100)) end #Issue 7507 -@test (i7507=sparsevec(Dict{Int64, Float64}(), 10))==spzeros(10,1) +@test (i7507=sparsevec(Dict{Int64, Float64}(), 10))==spzeros(10) #Issue 7650 let S = spzeros(3, 3) @@ -552,9 +552,10 @@ let X = eye(5), M = rand(5,4), C = spzeros(3,3) VX = vec(X); VSX = vec(SX) VM = vec(M); VSM1 = vec(SM); VSM2 = sparsevec(M) VC = vec(C) - @test reshape(VX, (25,1)) == VSX - @test reshape(VM, (20,1)) == VSM1 == VSM2 - @test size(VC) == (9,1) + @test VX == VSX + @test VM == VSM1 + @test VM == VSM2 + @test size(VC) == (9,) @test nnz(VC) == 0 @test nnz(VSX) == 5 end @@ -641,9 +642,9 @@ function test_getindex_algs{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, I::AbstractVector, ((minj < 1) || (maxj > n)) && BoundsError() end - (alg == 0) ? Base.SparseMatrix.getindex_I_sorted_bsearch_A(A, I, J) : - (alg == 1) ? Base.SparseMatrix.getindex_I_sorted_bsearch_I(A, I, J) : - Base.SparseMatrix.getindex_I_sorted_linear(A, I, J) + (alg == 0) ? Base.SparseArrays.getindex_I_sorted_bsearch_A(A, I, J) : + (alg == 1) ? Base.SparseArrays.getindex_I_sorted_bsearch_I(A, I, J) : + Base.SparseArrays.getindex_I_sorted_linear(A, I, J) end let M=2^14, N=2^4 @@ -761,7 +762,7 @@ end # issue #9917 @test sparse([]') == reshape(sparse([]), 1, 0) -@test full(sparse([])) == zeros(0, 1) +@test full(sparse([])) == zeros(0) @test_throws BoundsError sparse([])[1] @test_throws BoundsError sparse([])[1] = 1 x = speye(100) @@ -1071,7 +1072,7 @@ Ai = ceil(Int,Ar*100) @test_approx_eq vecnorm(Ai) vecnorm(full(Ai)) # test sparse matrix cond -A = sparse([1.0]) +A = sparse(reshape([1.0],1,1)) Ac = sprandn(20,20,.5) + im* sprandn(20,20,.5) Ar = sprandn(20,20,.5) @test cond(A,1) == 1.0 @@ -1092,12 +1093,12 @@ Ac = sprandn(20,20,.5) + im* sprandn(20,20,.5) Aci = ceil(Int64,100*sprand(20,20,.5))+ im*ceil(Int64,sprand(20,20,.5)) Ar = sprandn(20,20,.5) Ari = ceil(Int64,100*Ar) -@test_approx_eq_eps Base.SparseMatrix.normestinv(Ac,3) norm(inv(full(Ac)),1) 1e-4 -@test_approx_eq_eps Base.SparseMatrix.normestinv(Aci,3) norm(inv(full(Aci)),1) 1e-4 -@test_approx_eq_eps Base.SparseMatrix.normestinv(Ar) norm(inv(full(Ar)),1) 1e-4 -@test_throws ArgumentError Base.SparseMatrix.normestinv(Ac,0) -@test_throws ArgumentError Base.SparseMatrix.normestinv(Ac,21) -@test_throws DimensionMismatch Base.SparseMatrix.normestinv(sprand(3,5,.9)) +@test_approx_eq_eps Base.SparseArrays.normestinv(Ac,3) norm(inv(full(Ac)),1) 1e-4 +@test_approx_eq_eps Base.SparseArrays.normestinv(Aci,3) norm(inv(full(Aci)),1) 1e-4 +@test_approx_eq_eps Base.SparseArrays.normestinv(Ar) norm(inv(full(Ar)),1) 1e-4 +@test_throws ArgumentError Base.SparseArrays.normestinv(Ac,0) +@test_throws ArgumentError Base.SparseArrays.normestinv(Ac,21) +@test_throws DimensionMismatch Base.SparseArrays.normestinv(sprand(3,5,.9)) @test_throws ErrorException transpose(sub(sprandn(10, 10, 0.3), 1:4, 1:4)) @test_throws ErrorException ctranspose(sub(sprandn(10, 10, 0.3), 1:4, 1:4)) @@ -1106,7 +1107,7 @@ Ari = ceil(Int64,100*Ar) A = sprand(10,10,0.2) p = randperm(10) q = randperm(10) -@test Base.SparseMatrix.csc_permute(A, invperm(p), q) == full(A)[p, q] +@test Base.SparseArrays.csc_permute(A, invperm(p), q) == full(A)[p, q] # issue #13008 @test_throws ArgumentError sparse(collect(1:100), collect(1:100), fill(5,100), 5, 5) diff --git a/test/sparsedir/sparsevector.jl b/test/sparsedir/sparsevector.jl index f568a437e5188..d7c1a1eba6177 100644 --- a/test/sparsedir/sparsevector.jl +++ b/test/sparsedir/sparsevector.jl @@ -1,20 +1,13 @@ -## sparsevec.jl +# This file is a part of Julia. License is MIT: http://julialang.org/license ### Data spv_x1 = SparseVector(8, [2, 5, 6], [1.25, -0.75, 3.5]) -_x2 = SparseVector(8, [1, 2, 6, 7], [3.25, 4.0, -5.5, -6.0]) -spv_x2 = view(_x2) @test isa(spv_x1, SparseVector{Float64,Int}) -@test isa(spv_x2, SparseVectorView{Float64,Int}) x1_full = zeros(length(spv_x1)) -x1_full[nonzeroinds(spv_x1)] = nonzeros(spv_x1) - -x2_full = zeros(length(spv_x2)) -x2_full[nonzeroinds(spv_x2)] = nonzeros(spv_x2) - +x1_full[SparseArrays.nonzeroinds(spv_x1)] = nonzeros(spv_x1) ### Basic Properties @@ -29,88 +22,80 @@ let x = spv_x1 @test countnz(x) == 3 @test nnz(x) == 3 - @test nonzeroinds(x) == [2, 5, 6] + @test SparseArrays.nonzeroinds(x) == [2, 5, 6] @test nonzeros(x) == [1.25, -0.75, 3.5] end -let x = spv_x2 - @test eltype(x) == Float64 - @test ndims(x) == 1 - @test length(x) == 8 - @test size(x) == (8,) - @test size(x,1) == 8 - @test size(x,2) == 1 - @test !isempty(x) - - @test countnz(x) == 4 - @test nnz(x) == 4 - @test nonzeroinds(x) == [1, 2, 6, 7] - @test nonzeros(x) == [3.25, 4.0, -5.5, -6.0] -end - # full -for (x, xf) in [(spv_x1, x1_full), (spv_x2, x2_full)] +for (x, xf) in [(spv_x1, x1_full)] @test isa(full(x), Vector{Float64}) @test full(x) == xf end ### Show -@test string(spv_x1) == "Sparse vector, length = 8, with 3 Float64 entries:\n" * -" [2] = 1.25\n" * -" [5] = -0.75\n" * -" [6] = 3.5\n" - +@test contains(string(spv_x1), "1.25") +@test contains(string(spv_x1), "-0.75") +@test contains(string(spv_x1), "3.5") ### Other Constructors +### Comparison helper to ensure exact equality with internal structure +function exact_equal(x::AbstractSparseVector, y::AbstractSparseVector) + eltype(x) == eltype(y) && + eltype(SparseArrays.nonzeroinds(x)) == eltype(SparseArrays.nonzeroinds(y)) && + length(x) == length(y) && + SparseArrays.nonzeroinds(x) == SparseArrays.nonzeroinds(y) && + nonzeros(x) == nonzeros(y) +end + # construct empty sparse vector -@test exact_equal(sparsevector(Float64, 8), SparseVector(8, Int[], Float64[])) +@test exact_equal(spzeros(Float64, 8), SparseVector(8, Int[], Float64[])) # from list of indices and values @test exact_equal( - sparsevector(Int[], Float64[], 8), + sparsevec(Int[], Float64[], 8), SparseVector(8, Int[], Float64[])) @test exact_equal( - sparsevector(Int[], Float64[]), + sparsevec(Int[], Float64[]), SparseVector(0, Int[], Float64[])) @test exact_equal( - sparsevector([3, 3], [5.0, -5.0], 8), - sparsevector(Float64, 8)) + sparsevec([3, 3], [5.0, -5.0], 8), + spzeros(Float64, 8)) @test exact_equal( - sparsevector([2, 3, 6], [12.0, 18.0, 25.0]), + sparsevec([2, 3, 6], [12.0, 18.0, 25.0]), SparseVector(6, [2, 3, 6], [12.0, 18.0, 25.0])) let x0 = SparseVector(8, [2, 3, 6], [12.0, 18.0, 25.0]) @test exact_equal( - sparsevector([2, 3, 6], [12.0, 18.0, 25.0], 8), x0) + sparsevec([2, 3, 6], [12.0, 18.0, 25.0], 8), x0) @test exact_equal( - sparsevector([3, 6, 2], [18.0, 25.0, 12.0], 8), x0) + sparsevec([3, 6, 2], [18.0, 25.0, 12.0], 8), x0) @test exact_equal( - sparsevector([2, 3, 4, 4, 6], [12.0, 18.0, 5.0, -5.0, 25.0], 8), + sparsevec([2, 3, 4, 4, 6], [12.0, 18.0, 5.0, -5.0, 25.0], 8), x0) @test exact_equal( - sparsevector([1, 1, 1, 2, 3, 3, 6], [2.0, 3.0, -5.0, 12.0, 10.0, 8.0, 25.0], 8), + sparsevec([1, 1, 1, 2, 3, 3, 6], [2.0, 3.0, -5.0, 12.0, 10.0, 8.0, 25.0], 8), x0) @test exact_equal( - sparsevector([2, 3, 6, 7, 7], [12.0, 18.0, 25.0, 5.0, -5.0], 8), x0) + sparsevec([2, 3, 6, 7, 7], [12.0, 18.0, 25.0, 5.0, -5.0], 8), x0) end # from dictionary function my_intmap(x) a = Dict{Int,eltype(x)}() - for i in nonzeroinds(x) + for i in SparseArrays.nonzeroinds(x) a[i] = x[i] end return a @@ -118,59 +103,65 @@ end let x = spv_x1 a = my_intmap(x) - xc = sparsevector(a, 8) + xc = sparsevec(a, 8) @test exact_equal(x, xc) - xc = sparsevector(a) + xc = sparsevec(a) @test exact_equal(xc, SparseVector(6, [2, 5, 6], [1.25, -0.75, 3.5])) end +# spones - copies structure, but replaces nzvals with ones +let x = SparseVector(8, [2, 3, 6], [12.0, 18.0, 25.0]) + y = spones(x) + @test (x .!= 0) == (y .!= 0) + @test y == SparseVector(8, [2, 3, 6], [1.0, 1.0, 1.0]) +end + # sprand & sprandn -let xr = sprand(1000, 0.3) +let xr = sprand(1000, 0.9) @test isa(xr, SparseVector{Float64,Int}) @test length(xr) == 1000 @test all(nonzeros(xr) .>= 0.0) end -let xr = sprand(1000, 0.3, Float32) +let xr = sprand(1000, 0.9, Float32) @test isa(xr, SparseVector{Float32,Int}) @test length(xr) == 1000 @test all(nonzeros(xr) .>= 0.0) end -let xr = sprandn(1000, 0.3) +let xr = sprandn(1000, 0.9) @test isa(xr, SparseVector{Float64,Int}) @test length(xr) == 1000 - @test any(nonzeros(xr) .> 0.0) && any(nonzeros(xr) .< 0.0) + if !isempty(nonzeros(xr)) + @test any(nonzeros(xr) .> 0.0) && any(nonzeros(xr) .< 0.0) + end +end + +let xr = sprandbool(1000, 0.9) + @test isa(xr, SparseVector{Bool,Int}) + @test length(xr) == 1000 + @test all(nonzeros(xr)) end +let r1 = MersenneTwister(), r2 = MersenneTwister() + @test sprand(r1, 100, .9) == sprand(r2, 100, .9) + @test sprandn(r1, 100, .9) == sprandn(r2, 100, .9) + @test sprandbool(r1, 100, .9) == sprandbool(r2, 100, .9) +end ### Element access # getindex # single integer index -for (x, xf) in [(spv_x1, x1_full), (spv_x2, x2_full)] +for (x, xf) in [(spv_x1, x1_full)] for i = 1:length(x) @test x[i] == xf[i] end end -# range index -let x = spv_x2 - # range that contains no non-zeros - @test exact_equal(x[3:2], sparsevector(Float64, 0)) - @test exact_equal(x[3:3], sparsevector(Float64, 1)) - @test exact_equal(x[3:5], sparsevector(Float64, 3)) - - # range with non-zeros - @test exact_equal(x[1:length(x)], x) - @test exact_equal(x[1:5], SparseVector(5, [1,2], [3.25, 4.0])) - @test exact_equal(x[2:6], SparseVector(5, [1,5], [4.0, -5.5])) - @test exact_equal(x[2:8], SparseVector(7, [1,5,6], [4.0, -5.5, -6.0])) -end - # generic array index let x = sprand(100, 0.5) I = rand(1:length(x), 20) @@ -183,7 +174,7 @@ end # setindex -let xc = sparsevector(Float64, 8) +let xc = spzeros(Float64, 8) xc[3] = 2.0 @test exact_equal(xc, SparseVector(8, [3], [2.0])) end @@ -304,7 +295,7 @@ let m = 80, n = 100 end -## sparsemat.jl +## sparsemat: combinations with sparse matrix let S = sprand(4, 8, 0.5) Sf = full(S) @@ -312,39 +303,34 @@ let S = sprand(4, 8, 0.5) # get a single column for j = 1:size(S,2) - col = getcol(S, j) + col = S[:, j] @test isa(col, SparseVector{Float64,Int}) @test length(col) == size(S,1) @test full(col) == Sf[:,j] end - # column views - for j = 1:size(S,2) - col = view(S, :, j) - @test isa(col, SparseVectorView{Float64,Int}) - @test length(col) == size(S,1) - @test full(col) == Sf[:,j] + # Get a reshaped vector + v = S[:] + @test isa(v, SparseVector{Float64,Int}) + @test length(v) == length(S) + @test full(v) == Sf[:] + + # Get a linear subset + for i=0:length(S) + v = S[1:i] + @test isa(v, SparseVector{Float64,Int}) + @test length(v) == i + @test full(v) == Sf[1:i] + end + for i=1:length(S)+1 + v = S[i:end] + @test isa(v, SparseVector{Float64,Int}) + @test length(v) == length(S) - i + 1 + @test full(v) == Sf[i:end] end - - # column-range views - - # non-empty range - V = unsafe_colrange(S, 2:6) - @test isa(V, SparseMatrixCSC{Float64,Int}) - @test size(V) == (4, 5) - @test full(V) == Sf[:, 2:6] - @test !isempty(V) - - # empty range - V0 = unsafe_colrange(S, 2:1) - @test isa(V0, SparseMatrixCSC{Float64,Int}) - @test size(V0) == (4, 0) - @test isempty(V0) - end - -## math.jl +## math ### Data @@ -355,13 +341,11 @@ rnd_x1 = sprand(50, 0.7) * 4.0 rnd_x1f = full(rnd_x1) spv_x1 = SparseVector(8, [2, 5, 6], [1.25, -0.75, 3.5]) -_x2 = SparseVector(8, [1, 2, 6, 7], [3.25, 4.0, -5.5, -6.0]) -spv_x2 = view(_x2) - +spv_x2 = SparseVector(8, [1, 2, 6, 7], [3.25, 4.0, -5.5, -6.0]) ### Arithmetic operations -let x = spv_x1, x2 = spv_x2 +let x = spv_x1, x2 = x2 = spv_x2 # negate @test exact_equal(-x, SparseVector(8, [2, 5, 6], [-1.25, 0.75, -3.5])) @@ -418,7 +402,7 @@ let x = spv_x1, x2 = spv_x2 # real & imag @test is(real(x), x) - @test exact_equal(imag(x), sparsevector(Float64, length(x))) + @test exact_equal(imag(x), spzeros(Float64, length(x))) xcp = complex(x, x2) @test exact_equal(real(xcp), x) @@ -507,7 +491,7 @@ let x = SparseVector(3, [1, 2, 3], [-4.5, 2.5, 3.5]) @test minabs(x) == 2.5 end -let x = sparsevector(Float64, 8) +let x = spzeros(Float64, 8) @test maximum(x) == 0.0 @test minimum(x) == 0.0 @test maxabs(x) == 0.0 @@ -515,7 +499,7 @@ let x = sparsevector(Float64, 8) end -### linalg.jl +### linalg ### BLAS Level-1 @@ -526,7 +510,7 @@ let x = sprand(16, 0.5), x2 = sprand(16, 0.4) # axpy! for c in [1.0, -1.0, 2.0, -2.0] y = full(x) - @test is(axpy!(c, x2, y), y) + @test is(Base.axpy!(c, x2, y), y) @test y == full(x2 * c + x) end @@ -606,7 +590,7 @@ let A = sprandn(9, 16, 0.5), x = sprand(16, 0.7) @test is(A_mul_B!(α, A, x, β, y), y) @test_approx_eq y rr end - y = densemv(A, x) + y = SparseArrays.densemv(A, x) @test isa(y, Vector{Float64}) @test_approx_eq y Af * xf end @@ -620,7 +604,7 @@ let A = sprandn(16, 9, 0.5), x = sprand(16, 0.7) @test is(At_mul_B!(α, A, x, β, y), y) @test_approx_eq y rr end - y = densemv(A, x; trans='T') + y = SparseArrays.densemv(A, x; trans='T') @test isa(y, Vector{Float64}) @test_approx_eq y At_mul_B(Af, xf) end @@ -631,9 +615,9 @@ let A = complex(sprandn(7, 8, 0.5), sprandn(7, 8, 0.5)), Af = full(A) xf = full(x) x2f = full(x2) - @test_approx_eq densemv(A, x; trans='N') Af * xf - @test_approx_eq densemv(A, x2; trans='T') Af.' * x2f - @test_approx_eq densemv(A, x2; trans='C') Af'x2f + @test_approx_eq SparseArrays.densemv(A, x; trans='N') Af * xf + @test_approx_eq SparseArrays.densemv(A, x2; trans='T') Af.' * x2f + @test_approx_eq SparseArrays.densemv(A, x2; trans='C') Af'x2f end ## sparse A * sparse x -> sparse y @@ -673,3 +657,29 @@ let A = complex(sprandn(7, 8, 0.5), sprandn(7, 8, 0.5)), @test isa(y, SparseVector{Complex128,Int}) @test_approx_eq full(y) Af'x2f end + +# It's tempting to share data between a SparseVector and a SparseArrays, +# but if that's done, then modifications to one or the other will cause +# an inconsistent state: +sv = sparse(1:10) +sm = convert(SparseMatrixCSC, sv) +sv[1] = 0 +@test full(sm)[2:end] == collect(2:10) + +# Ensure that sparsevec with all-zero values returns an array of zeros +@test sparsevec([1,2,3],[0,0,0]) == [0,0,0] + +# Compare stored zero semantics between SparseVector and SparseMatrixCSC +let S = SparseMatrixCSC(10,1,[1,6],[1,3,5,6,7],[0,1,2,0,3]), x = SparseVector(10,[1,3,5,6,7],[0,1,2,0,3]) + @test nnz(S) == nnz(x) == 5 + for I = (:, 1:10, collect(1:10)) + @test S[I,1] == S[I] == x[I] == x + @test nnz(S[I,1]) == nnz(S[I]) == nnz(x[I]) == nnz(x) + end + for I = (2:9, 1:2, 9:10, [3,6,1], [10,9,8], []) + @test S[I,1] == S[I] == x[I] + @test nnz(S[I,1]) == nnz(S[I]) == nnz(x[I]) + end + @test S[[1 3 5; 2 4 6]] == x[[1 3 5; 2 4 6]] + @test nnz(S[[1 3 5; 2 4 6]]) == nnz(x[[1 3 5; 2 4 6]]) +end diff --git a/test/sparsedir/spqr.jl b/test/sparsedir/spqr.jl index 3b9c0f9d1782a..1fee41328c1ad 100644 --- a/test/sparsedir/spqr.jl +++ b/test/sparsedir/spqr.jl @@ -2,8 +2,8 @@ using Base.Test -using Base.SparseMatrix.SPQR -using Base.SparseMatrix.CHOLMOD +using Base.SparseArrays.SPQR +using Base.SparseArrays.CHOLMOD m, n = 100, 10 nn = 100 diff --git a/test/sparsedir/umfpack.jl b/test/sparsedir/umfpack.jl index 2a7e708eca2e5..be6ea41cae4d6 100644 --- a/test/sparsedir/umfpack.jl +++ b/test/sparsedir/umfpack.jl @@ -6,14 +6,14 @@ do33 = ones(3) # based on deps/Suitesparse-4.0.2/UMFPACK/Demo/umfpack_di_demo.c -using Base.SparseMatrix.UMFPACK.increment! +using Base.SparseArrays.UMFPACK.increment! A0 = sparse(increment!([0,4,1,1,2,2,0,1,2,3,4,4]), increment!([0,4,0,2,1,2,1,4,3,2,1,2]), [2.,1.,3.,4.,-1.,-3.,3.,6.,2.,1.,4.,2.], 5, 5) for Tv in (Float64, Complex128) - for Ti in Base.SparseMatrix.UMFPACK.UMFITypes.types + for Ti in Base.SparseArrays.UMFPACK.UMFITypes.types A = convert(SparseMatrixCSC{Tv,Ti}, A0) lua = lufact(A) @test nnz(lua) == 18 @@ -40,7 +40,7 @@ for Tv in (Float64, Complex128) end Ac0 = complex(A0,A0) -for Ti in Base.SparseMatrix.UMFPACK.UMFITypes.types +for Ti in Base.SparseArrays.UMFPACK.UMFITypes.types Ac = convert(SparseMatrixCSC{Complex128,Ti}, Ac0) lua = lufact(Ac) L,U,p,q,Rs = lua[:(:)] From 12fbcab60a5cfc1aa0e7bbf44d436f718d380851 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Wed, 7 Oct 2015 17:23:37 -0400 Subject: [PATCH 0542/1938] Add SparseVector to NEWS.md --- NEWS.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/NEWS.md b/NEWS.md index 7db7662e707e6..c5c6c8a58a645 100644 --- a/NEWS.md +++ b/NEWS.md @@ -30,6 +30,11 @@ Library improvements * New method for generic QR with column pivoting ([#13480]). + * A new `SparseVector` type allows for one-dimensional sparse arrays. Slicing + and reshaping sparse matrices now return vectors when appropriate. The + `sparsevec` function returns a one-dimensional sparse vector instead of a + one-column sparse matrix. + Deprecated or removed --------------------- From e316b6092f5ae234704e260989dba0df09fdc67e Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Wed, 7 Oct 2015 19:38:55 -0400 Subject: [PATCH 0543/1938] Move SparseArrays exports back to their usual home --- base/exports.jl | 25 ++++++++++++++++++++++++- base/sysimg.jl | 23 ----------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index 7caf6944ff079..e32a5ae247046 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1414,4 +1414,27 @@ export @assert, @enum, @label, - @goto + @goto, + +# SparseArrays module re-exports + SparseArrays, + AbstractSparseArray, + AbstractSparseMatrix, + AbstractSparseVector, + SparseMatrixCSC, + SparseVector, + etree, + issparse, + sparse, + sparsevec, + spdiagm, + speye, + spones, + sprand, + sprandbool, + sprandn, + spzeros, + symperm, + rowvals, + nzrange, + nnz diff --git a/base/sysimg.jl b/base/sysimg.jl index 5568676d519d1..5b075ac237067 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -287,29 +287,6 @@ import .Dates: Date, DateTime, now # sparse matrices, vectors, and sparse linear algebra include("sparse.jl") importall .SparseArrays -# SparseArrays module re-exports -export - SparseArrays, - AbstractSparseArray, - AbstractSparseMatrix, - AbstractSparseVector, - SparseMatrixCSC, - SparseVector, - etree, - issparse, - sparse, - sparsevec, - spdiagm, - speye, - spones, - sprand, - sprandbool, - sprandn, - spzeros, - symperm, - rowvals, - nzrange, - nnz # Documentation From eb90416701eb6e782054afd33a1ec1b70a737fd9 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Tue, 13 Oct 2015 12:18:28 -0400 Subject: [PATCH 0544/1938] Add missing imports --- base/sparse.jl | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/base/sparse.jl b/base/sparse.jl index d40603bc3f3f0..3a5485ebe1821 100644 --- a/base/sparse.jl +++ b/base/sparse.jl @@ -7,20 +7,28 @@ using Base.Sort: Forward using Base.LinAlg: AbstractTriangular, PosDefException import Base: +, -, *, &, |, $, .+, .-, .*, ./, .\, .^, .<, .!=, == -import Base: A_mul_B!, Ac_mul_B, Ac_mul_B!, At_mul_B!, A_ldiv_B! -import Base: @get!, abs, abs2, broadcast, ceil, complex, cond, conj, convert, copy, - ctranspose, diagm, exp, expm1, factorize, find, findmax, findmin, findnz, float, - full, getindex, hcat, hvcat, imag, indmax, ishermitian, kron, length, log, log1p, - max, min, norm, one, promote_eltype, real, reinterpret, reshape, rot180, rotl90, - rotr90, round, scale, scale!, setindex!, similar, size, transpose, tril, triu, vcat, - vec +import Base: A_mul_B!, Ac_mul_B, Ac_mul_B!, At_mul_B, At_mul_B!, A_ldiv_B! + +import Base: @get!, acos, acosd, acot, acotd, acsch, asech, asin, asind, asinh, + atan, atand, atanh, broadcast!, chol, conj!, cos, cosc, cosd, cosh, cospi, cot, + cotd, coth, countnz, csc, cscd, csch, ctranspose!, diag, diff, done, dot, eig, + exp10, exp2, eye, findn, floor, hash, indmin, inv, issym, istril, istriu, log10, + log2, lu, maxabs, minabs, next, sec, secd, sech, show, showarray, sin, sinc, + sind, sinh, sinpi, squeeze, start, sum, sumabs, sumabs2, summary, tan, tand, + tanh, trace, transpose!, tril!, triu!, trunc, vecnorm, writemime, abs, abs2, + broadcast, call, ceil, complex, cond, conj, convert, copy, ctranspose, diagm, + exp, expm1, factorize, find, findmax, findmin, findnz, float, full, getindex, + hcat, hvcat, imag, indmax, ishermitian, kron, length, log, log1p, max, min, + maximum, minimum, norm, one, promote_eltype, real, reinterpret, reshape, rot180, + rotl90, rotr90, round, scale, scale!, setindex!, similar, size, transpose, tril, + triu, vcat, vec + import Base.Broadcast: eltype_plus, broadcast_shape -export AbstractSparseArray, AbstractSparseMatrix, AbstractSparseVector, SparseMatrixCSC, - SparseVector, - blkdiag, dense, droptol!, dropzeros!, etree, issparse, nonzeros, nzrange, - rowvals, sparse, sparsevec, spdiagm, speye, spones, sprand, sprandbool, sprandn, - spzeros, symperm, nnz +export AbstractSparseArray, AbstractSparseMatrix, AbstractSparseVector, + SparseMatrixCSC, SparseVector, blkdiag, dense, droptol!, dropzeros!, etree, + issparse, nonzeros, nzrange, rowvals, sparse, sparsevec, spdiagm, speye, spones, + sprand, sprandbool, sprandn, spzeros, symperm, nnz include("sparse/abstractsparse.jl") include("sparse/sparsematrix.jl") From a2516aa040d12cb8e3a0459f224f0cd24073e5e0 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 13 Oct 2015 12:30:33 -0400 Subject: [PATCH 0545/1938] cleanup runtime_intrinsics DLLEXPORT and license --- src/APInt-C.cpp | 8 ++- src/APInt-C.h | 131 ++++++++++++++++++++++-------------------- src/intrinsics.cpp | 11 ++-- src/runtime_ccall.cpp | 1 + 4 files changed, 83 insertions(+), 68 deletions(-) diff --git a/src/APInt-C.cpp b/src/APInt-C.cpp index 16a4c3059493e..1558f5f816b5c 100644 --- a/src/APInt-C.cpp +++ b/src/APInt-C.cpp @@ -1,10 +1,14 @@ +// This file is a part of Julia. License is MIT: http://julialang.org/license + #include "llvm-version.h" #include #include #include -#define DLLEXPORT -extern "C" DLLEXPORT void jl_error(const char *str); +extern "C" { +#include "APInt-C.h" +DLLEXPORT void jl_error(const char *str); +} using namespace llvm; diff --git a/src/APInt-C.h b/src/APInt-C.h index da7c08652d2a8..2dd97fa02e0a9 100644 --- a/src/APInt-C.h +++ b/src/APInt-C.h @@ -1,3 +1,4 @@ +// This file is a part of Julia. License is MIT: http://julialang.org/license #ifndef APINT_C_H #define APINT_C_H @@ -5,70 +6,76 @@ #ifdef __cplusplus extern "C" { #endif +#include "dtypes.h" + +#ifdef LLVM_VERSION_MAJOR +using llvm::integerPart; +#else typedef void integerPart; +#endif -void LLVMNeg(unsigned numbits, integerPart *pa, integerPart *pr); -void LLVMByteSwap(unsigned numbits, integerPart *pa, integerPart *pr); - -void LLVMAdd(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -void LLVMSub(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -void LLVMMul(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -void LLVMSDiv(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -void LLVMUDiv(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -void LLVMSRem(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -void LLVMURem(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); - -void LLVMAnd(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -void LLVMOr(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -void LLVMXor(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -void LLVMShl(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -void LLVMLShr(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -void LLVMAShr(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -void LLVMFlipAllBits(unsigned numbits, integerPart *pa, integerPart *pr); - -int LLVMICmpEQ(unsigned numbits, integerPart *pa, integerPart *pr); -int LLVMICmpNE(unsigned numbits, integerPart *pa, integerPart *pb); -int LLVMICmpSLT(unsigned numbits, integerPart *pa, integerPart *pb); -int LLVMICmpULT(unsigned numbits, integerPart *pa, integerPart *pb); -int LLVMICmpSLE(unsigned numbits, integerPart *pa, integerPart *pb); -int LLVMICmpULE(unsigned numbits, integerPart *pa, integerPart *pb); - -int LLVMAdd_uov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -int LLVMAdd_sov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -int LLVMSub_uov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -int LLVMSub_sov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -int LLVMMul_sov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -int LLVMMul_uov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); - -unsigned LLVMCountPopulation(unsigned numbits, integerPart *pa); -unsigned LLVMCountTrailingOnes(unsigned numbits, integerPart *pa); -unsigned LLVMCountTrailingZeros(unsigned numbits, integerPart *pa); -unsigned LLVMCountLeadingOnes(unsigned numbits, integerPart *pa); -unsigned LLVMCountLeadingZeros(unsigned numbits, integerPart *pa); - -void LLVMFPtoSI(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); -void LLVMFPtoUI(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); -void LLVMSItoFP(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); -void LLVMUItoFP(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); -void LLVMSExt(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); -void LLVMZExt(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); -void LLVMTrunc(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); - -int LLVMFPtoSI_exact(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); -int LLVMFPtoUI_exact(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); - -void jl_LLVMSMod(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); -void jl_LLVMFlipSign(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); - -unsigned countTrailingZeros_8(uint8_t Val); -unsigned countTrailingZeros_16(uint16_t Val); -unsigned countTrailingZeros_32(uint32_t Val); -unsigned countTrailingZeros_64(uint64_t Val); - -uint8_t getSwappedBytes_8(uint8_t Value); // no-op -uint16_t getSwappedBytes_16(uint16_t Value); -uint32_t getSwappedBytes_32(uint32_t Value); -uint64_t getSwappedBytes_64(uint64_t Value); +DLLEXPORT void LLVMNeg(unsigned numbits, integerPart *pa, integerPart *pr); +DLLEXPORT void LLVMByteSwap(unsigned numbits, integerPart *pa, integerPart *pr); + +DLLEXPORT void LLVMAdd(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT void LLVMSub(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT void LLVMMul(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT void LLVMSDiv(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT void LLVMUDiv(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT void LLVMSRem(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT void LLVMURem(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); + +DLLEXPORT void LLVMAnd(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT void LLVMOr(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT void LLVMXor(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT void LLVMShl(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT void LLVMLShr(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT void LLVMAShr(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT void LLVMFlipAllBits(unsigned numbits, integerPart *pa, integerPart *pr); + +DLLEXPORT int LLVMICmpEQ(unsigned numbits, integerPart *pa, integerPart *pr); +DLLEXPORT int LLVMICmpNE(unsigned numbits, integerPart *pa, integerPart *pb); +DLLEXPORT int LLVMICmpSLT(unsigned numbits, integerPart *pa, integerPart *pb); +DLLEXPORT int LLVMICmpULT(unsigned numbits, integerPart *pa, integerPart *pb); +DLLEXPORT int LLVMICmpSLE(unsigned numbits, integerPart *pa, integerPart *pb); +DLLEXPORT int LLVMICmpULE(unsigned numbits, integerPart *pa, integerPart *pb); + +DLLEXPORT int LLVMAdd_uov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT int LLVMAdd_sov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT int LLVMSub_uov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT int LLVMSub_sov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT int LLVMMul_sov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT int LLVMMul_uov(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); + +DLLEXPORT unsigned LLVMCountPopulation(unsigned numbits, integerPart *pa); +DLLEXPORT unsigned LLVMCountTrailingOnes(unsigned numbits, integerPart *pa); +DLLEXPORT unsigned LLVMCountTrailingZeros(unsigned numbits, integerPart *pa); +DLLEXPORT unsigned LLVMCountLeadingOnes(unsigned numbits, integerPart *pa); +DLLEXPORT unsigned LLVMCountLeadingZeros(unsigned numbits, integerPart *pa); + +DLLEXPORT void LLVMFPtoSI(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); +DLLEXPORT void LLVMFPtoUI(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); +DLLEXPORT void LLVMSItoFP(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); +DLLEXPORT void LLVMUItoFP(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); +DLLEXPORT void LLVMSExt(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); +DLLEXPORT void LLVMZExt(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); +DLLEXPORT void LLVMTrunc(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); + +DLLEXPORT int LLVMFPtoSI_exact(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); +DLLEXPORT int LLVMFPtoUI_exact(unsigned numbits, integerPart *pa, unsigned onumbits, integerPart *pr); + +DLLEXPORT void jl_LLVMSMod(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); +DLLEXPORT void jl_LLVMFlipSign(unsigned numbits, integerPart *pa, integerPart *pb, integerPart *pr); + +DLLEXPORT unsigned countTrailingZeros_8(uint8_t Val); +DLLEXPORT unsigned countTrailingZeros_16(uint16_t Val); +DLLEXPORT unsigned countTrailingZeros_32(uint32_t Val); +DLLEXPORT unsigned countTrailingZeros_64(uint64_t Val); + +//uint8_t getSwappedBytes_8(uint8_t Value); // no-op +//uint16_t getSwappedBytes_16(uint16_t Value); +//uint32_t getSwappedBytes_32(uint32_t Value); +//uint64_t getSwappedBytes_64(uint64_t Value); #ifdef __cplusplus diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 88f7e1734d877..fe6a4baf2caa0 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -495,7 +495,10 @@ static jl_cgval_t generic_unbox(jl_value_t *targ, jl_value_t *x, jl_codectx_t *c } else { if (!jl_is_leaf_type(v.typ) && !jl_is_bitstype(v.typ)) { - return jl_cgval_t(); // TODO: XXX + // TODO: currently doesn't handle the case where the type of neither argument is understood at compile time + // since codegen has no idea what size it might have + jl_error("codegen: failed during evaluation of a call to unbox"); + return jl_cgval_t(); } nb = jl_datatype_size(v.typ); llvmt = staticeval_bitstype(v.typ); @@ -515,7 +518,7 @@ static jl_cgval_t generic_unbox(jl_value_t *targ, jl_value_t *x, jl_codectx_t *c if (!jl_is_bitstype(bt)) { // TODO: to accept arbitrary types, replace this function with a call to llvm_type_rewrite - emit_error("reinterpret: expected bits type as first argument", ctx); + emit_error("unbox: expected bits type as first argument", ctx); return jl_cgval_t(); } @@ -536,7 +539,7 @@ static jl_cgval_t generic_unbox(jl_value_t *targ, jl_value_t *x, jl_codectx_t *c else { vx = v.V; if (!jl_is_bitstype(v.typ)) { - emit_error("reinterpret: expected bits type value for second argument", ctx); + emit_error("unbox: expected bits type value for second argument", ctx); return jl_cgval_t(); } } @@ -554,7 +557,7 @@ static jl_cgval_t generic_unbox(jl_value_t *targ, jl_value_t *x, jl_codectx_t *c if (vxt->getPrimitiveSizeInBits() != llvmt->getPrimitiveSizeInBits() && !(vxt->isPointerTy() && llvmt->getPrimitiveSizeInBits() == sizeof(void*)*8) && !(llvmt->isPointerTy() && vxt->getPrimitiveSizeInBits() == sizeof(void*)*8)) { - emit_error("box: argument is of incorrect size", ctx); + emit_error("unbox: argument is of incorrect size", ctx); return jl_cgval_t(); } if (vxt->isPointerTy() && !llvmt->isPointerTy()) diff --git a/src/runtime_ccall.cpp b/src/runtime_ccall.cpp index f1623fc6ac08f..81e43164c64bd 100644 --- a/src/runtime_ccall.cpp +++ b/src/runtime_ccall.cpp @@ -2,6 +2,7 @@ #include #include +#include #include "julia.h" #include "julia_internal.h" From 566acb1183a90ce69f58a34fe43d416cf8f69050 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Tue, 13 Oct 2015 13:59:34 -0400 Subject: [PATCH 0546/1938] Allow setindex! in zero part of Triangular matrices... as long as the value is zero --- base/linalg/triangular.jl | 34 +++++++++++++++++++++------------- test/linalg/triangular.jl | 16 ++++++++++++---- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/base/linalg/triangular.jl b/base/linalg/triangular.jl index 92e12f8039b73..8143a7b0b78cd 100644 --- a/base/linalg/triangular.jl +++ b/base/linalg/triangular.jl @@ -23,12 +23,12 @@ for t in (:LowerTriangular, :UnitLowerTriangular, :UpperTriangular, :UnitUpperTr convert{T,S}(::Type{Matrix}, A::$t{T,S}) = convert(Matrix{T}, A) function similar{T,S,Tnew}(A::$t{T,S}, ::Type{Tnew}, dims::Dims) - if dims[1] != dims[2] - throw(ArgumentError("Triangular matrix must be square")) - end if length(dims) != 2 throw(ArgumentError("Triangular matrix must have two dimensions")) end + if dims[1] != dims[2] + throw(ArgumentError("Triangular matrix must be square")) + end B = similar(A.data, Tnew, dims) return $t(B) end @@ -119,33 +119,41 @@ getindex{T,S}(A::UpperTriangular{T,S}, i::Integer, j::Integer) = i <= j ? A.data function setindex!(A::UpperTriangular, x, i::Integer, j::Integer) if i > j - throw(BoundsError(A,(i,j))) + x == 0 || throw(ArgumentError("cannot set index in the lower triangular part ($i, $j) of an UpperTriangular matrix to a nonzero value ($x)")) + else + A.data[i,j] = x end - A.data[i,j] = x return A end function setindex!(A::UnitUpperTriangular, x, i::Integer, j::Integer) - if i >= j - throw(BoundsError(A,(i,j))) + if i > j + x == 0 || throw(ArgumentError("cannot set index in the lower triangular part ($i, $j) of a UnitUpperTriangular matrix to a nonzero value ($x)")) + elseif i == j + x == 1 || throw(ArgumentError("cannot set index on the diagonal ($i, $j) of a UnitUpperTriangular matrix to a non-unit value ($x)")) + else + A.data[i,j] = x end - A.data[i,j] = x return A end function setindex!(A::LowerTriangular, x, i::Integer, j::Integer) if i < j - throw(BoundsError(A,(i,j))) + x == 0 || throw(ArgumentError("cannot set index in the upper triangular part ($i, $j) of a LowerTriangular matrix to a nonzero value ($x)")) + else + A.data[i,j] = x end - A.data[i,j] = x return A end function setindex!(A::UnitLowerTriangular, x, i::Integer, j::Integer) - if i <= j - throw(BoundsError(A,(i,j))) + if i < j + x == 0 || throw(ArgumentError("cannot set index in the upper triangular part ($i, $j) of a UnitLowerTriangular matrix to a nonzero value ($x)")) + elseif i == j + x == 1 || throw(ArgumentError("cannot set diagonal index ($i, $j) of a UnitLowerTriangular matrix to a non-unit value ($x)")) + else + A.data[i,j] = x end - A.data[i,j] = x return A end diff --git a/test/linalg/triangular.jl b/test/linalg/triangular.jl index cea002bd2eda0..51a25a98ce4e3 100644 --- a/test/linalg/triangular.jl +++ b/test/linalg/triangular.jl @@ -59,15 +59,23 @@ for elty1 in (Float32, Float64, Complex64, Complex128, BigFloat, Int) for i = 1:size(A1, 1) for j = 1:size(A1, 2) if uplo1 == :U - if i > j || (i == j && t1 == UnitUpperTriangular) - @test_throws BoundsError A1c[i,j] = 0 + if i > j + A1c[i,j] = 0 + @test_throws ArgumentError A1c[i,j] = 1 + elseif i == j && t1 == UnitUpperTriangular + A1c[i,j] = 1 + @test_throws ArgumentError A1c[i,j] = 0 else A1c[i,j] = 0 @test A1c[i,j] == 0 end else - if i < j || (i == j && t1 == UnitLowerTriangular) - @test_throws BoundsError A1c[i,j] = 0 + if i < j + A1c[i,j] = 0 + @test_throws ArgumentError A1c[i,j] = 1 + elseif i == j && t1 == UnitLowerTriangular + A1c[i,j] = 1 + @test_throws ArgumentError A1c[i,j] = 0 else A1c[i,j] = 0 @test A1c[i,j] == 0 From 54f898cb1e0dc49964004e34f0eb70fa7f0d94fa Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Tue, 13 Oct 2015 16:08:18 -0400 Subject: [PATCH 0547/1938] NEWS for #13387 --- NEWS.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 7db7662e707e6..b6de14271a0c6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -16,9 +16,14 @@ Compiler/Runtime improvements Library improvements -------------------- - * The package system (`Pkg`) is now based on the `libgit2` library, rather - than running the `git` program, increasing performance (especially on - Windows) ([#11196]). + * Packages: + + * The package system (`Pkg`) is now based on the `libgit2` library, rather + than running the `git` program, increasing performance (especially on + Windows) ([#11196]). + + * Package-development functions like `Pkg.tag` and `Pkg.publish` + have been moved to an external [PkgDev] package ([#13387]). * The `Base.Test` module now has a `@testset` feature to bundle tests together and delay throwing an error until the end ([#13062]). @@ -1297,6 +1302,7 @@ Too numerous to mention. [pairwise summation]: https://en.wikipedia.org/wiki/Pairwise_summation [a448e080]: https://github.com/JuliaLang/julia/commit/a448e080dc736c7fb326426dfcb2528be36973d3 [5e3f074b]: https://github.com/JuliaLang/julia/commit/5e3f074b9173044a0a4219f9b285879ff7cec041 +[PkgDev]: https://github.com/JuliaLang/PkgDev.jl [#13]: https://github.com/JuliaLang/julia/issues/13 [#69]: https://github.com/JuliaLang/julia/issues/69 @@ -1670,3 +1676,6 @@ Too numerous to mention. [#12739]: https://github.com/JuliaLang/julia/issues/12739 [#13062]: https://github.com/JuliaLang/julia/issues/13062 [#13338]: https://github.com/JuliaLang/julia/issues/13338 +[#13387]: https://github.com/JuliaLang/julia/issues/13387 +[#13465]: https://github.com/JuliaLang/julia/issues/13465 +[#13480]: https://github.com/JuliaLang/julia/issues/13480 From 8adbf7a3ace473e6ddf740f7c95077972381101c Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Tue, 13 Oct 2015 16:10:37 -0400 Subject: [PATCH 0548/1938] NEWS for #13542 --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index b6de14271a0c6..98b77f328924d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -7,6 +7,8 @@ New language features Language changes ---------------- + * `using` and `import` are now case-sensitive even on case-insensitive filesystems (common on Mac and Windows) ([#13542]). + Command-line option changes --------------------------- @@ -1679,3 +1681,4 @@ Too numerous to mention. [#13387]: https://github.com/JuliaLang/julia/issues/13387 [#13465]: https://github.com/JuliaLang/julia/issues/13465 [#13480]: https://github.com/JuliaLang/julia/issues/13480 +[#13542]: https://github.com/JuliaLang/julia/issues/13542 From 49adc8fb58674785a0b6a140f158cc32fbc697fa Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 13 Oct 2015 23:20:02 +0200 Subject: [PATCH 0549/1938] add simple check for aliasing in gemm_wrapper! This does not catch all cases of aliasing but should catch the most common mistakes that users unaware of the aliasing rules might do. --- base/linalg/matmul.jl | 5 +++++ test/linalg/matmul.jl | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/base/linalg/matmul.jl b/base/linalg/matmul.jl index 2066ff3603efe..cb7d87cd27cb3 100644 --- a/base/linalg/matmul.jl +++ b/base/linalg/matmul.jl @@ -311,12 +311,17 @@ function gemm_wrapper!{T<:BlasFloat}(C::StridedVecOrMat{T}, tA::Char, tB::Char, throw(DimensionMismatch("A has dimensions ($mA,$nA) but B has dimensions ($mB,$nB)")) end + if C === A || B === C + throw(ArgumentError("output matrix must not be aliased with input matrix")) + end + if mA == 0 || nA == 0 || nB == 0 if size(C) != (mA, nB) throw(DimensionMismatch("C has dimensions $(size(C)), should have ($mA,$nB)")) end return fill!(C,0) end + if mA == 2 && nA == 2 && nB == 2 return matmul2x2!(C,tA,tB,A,B) end diff --git a/test/linalg/matmul.jl b/test/linalg/matmul.jl index 02d3fef9f6b30..7698424580099 100644 --- a/test/linalg/matmul.jl +++ b/test/linalg/matmul.jl @@ -186,3 +186,10 @@ for elty in [Float32,Float64,Complex128,Complex64] A = rand(elty,3,3) @test Base.LinAlg.matmul3x3('T','N',A,eye(elty,3)) == A.' end + +# 13593, #13488 +a = rand(3,3) +b = rand(3,3) +@test_throws ArgumentError A_mul_B!(a, a, b) +@test_throws ArgumentError A_mul_B!(a, b, a) +@test_throws ArgumentError A_mul_B!(a, a, a) From cd37f1b73b61a09bcb66ea6463d92282ee984445 Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Tue, 13 Oct 2015 13:18:49 -0400 Subject: [PATCH 0550/1938] A few missing imports in LinAlg and DataFmt --- base/datafmt.jl | 2 +- base/linalg.jl | 5 +++-- test/linalg/bidiag.jl | 5 +++++ test/linalg/tridiag.jl | 8 ++++++++ test/readdlm.jl | 4 ++++ 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/base/datafmt.jl b/base/datafmt.jl index 595ec572216c0..ed10fd92f28b8 100644 --- a/base/datafmt.jl +++ b/base/datafmt.jl @@ -4,7 +4,7 @@ module DataFmt -import Base: _default_delims, tryparse_internal +import Base: _default_delims, tryparse_internal, writemime export countlines, readdlm, readcsv, writedlm, writecsv diff --git a/base/linalg.jl b/base/linalg.jl index 5c695f24b3ea3..6aa47e91748e4 100644 --- a/base/linalg.jl +++ b/base/linalg.jl @@ -7,8 +7,9 @@ import Base: A_mul_Bt, At_ldiv_Bt, A_rdiv_Bc, At_ldiv_B, Ac_mul_Bc, A_mul_Bc, Ac Ac_ldiv_B, Ac_ldiv_Bc, At_mul_Bt, A_rdiv_Bt, At_mul_B import Base: USE_BLAS64, abs, big, ceil, conj, convert, copy, copy!, copy_transpose!, ctranspose, ctranspose!, eltype, eye, findmax, findmin, fill!, floor, full, getindex, - imag, inv, isapprox, kron, ndims, power_by_squaring, print_matrix, real, setindex!, - show, similar, size, transpose, transpose!, unsafe_getindex, unsafe_setindex! + imag, inv, isapprox, kron, ndims, power_by_squaring, print_matrix, promote_rule, real, + round, setindex!, show, similar, size, transpose, transpose!, trunc, unsafe_getindex, + unsafe_setindex! export # Modules diff --git a/test/linalg/bidiag.jl b/test/linalg/bidiag.jl index 74e1eb1b11e8c..af3b5854915d1 100644 --- a/test/linalg/bidiag.jl +++ b/test/linalg/bidiag.jl @@ -112,9 +112,13 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) debug && println("Round,float,trunc,ceil") if elty <: BlasReal @test floor(Int,T) == Bidiagonal(floor(Int,T.dv),floor(Int,T.ev),T.isupper) + @test isa(floor(Int,T), Bidiagonal) @test trunc(Int,T) == Bidiagonal(trunc(Int,T.dv),trunc(Int,T.ev),T.isupper) + @test isa(trunc(Int,T), Bidiagonal) @test round(Int,T) == Bidiagonal(round(Int,T.dv),round(Int,T.ev),T.isupper) + @test isa(round(Int,T), Bidiagonal) @test ceil(Int,T) == Bidiagonal(ceil(Int,T.dv),ceil(Int,T.ev),T.isupper) + @test isa(ceil(Int,T), Bidiagonal) end debug && println("Generic Mat-vec ops") @@ -184,5 +188,6 @@ end A = Bidiagonal(ones(Float32,10),ones(Float32,9),true) B = rand(Float64,10,10) C = Tridiagonal(rand(Float64,9),rand(Float64,10),rand(Float64,9)) +@test promote_rule(Matrix{Float64}, Bidiagonal{Float64}) == Matrix{Float64} @test promote(B,A) == (B,convert(Matrix{Float64},full(A))) @test promote(C,A) == (C,Tridiagonal(zeros(Float64,9),convert(Vector{Float64},A.dv),convert(Vector{Float64},A.ev))) diff --git a/test/linalg/tridiag.jl b/test/linalg/tridiag.jl index 972ed266a453d..ccd739aeebd1b 100644 --- a/test/linalg/tridiag.jl +++ b/test/linalg/tridiag.jl @@ -227,9 +227,13 @@ let n = 12 #Size of matrix problem to test debug && println("Rounding to Ints") if elty <: Real @test round(Int,A) == round(Int,fA) + @test isa(round(Int,A), SymTridiagonal) @test trunc(Int,A) == trunc(Int,fA) + @test isa(trunc(Int,A), SymTridiagonal) @test ceil(Int,A) == ceil(Int,fA) + @test isa(ceil(Int,A), SymTridiagonal) @test floor(Int,A) == floor(Int,fA) + @test isa(floor(Int,A), SymTridiagonal) end debug && println("Tridiagonal/SymTridiagonal mixing ops") @@ -341,9 +345,13 @@ let n = 12 #Size of matrix problem to test debug && println("Rounding to Ints") if elty <: Real @test round(Int,A) == round(Int,fA) + @test isa(round(Int,A), Tridiagonal) @test trunc(Int,A) == trunc(Int,fA) + @test isa(trunc(Int,A), Tridiagonal) @test ceil(Int,A) == ceil(Int,fA) + @test isa(ceil(Int,A), Tridiagonal) @test floor(Int,A) == floor(Int,fA) + @test isa(floor(Int,A), Tridiagonal) end debug && println("Binary operations") diff --git a/test/readdlm.jl b/test/readdlm.jl index 3e0674f664b2e..8cfd34609ea86 100644 --- a/test/readdlm.jl +++ b/test/readdlm.jl @@ -233,3 +233,7 @@ end let data = "1 2 3" readdlm(IOBuffer(data), ' ', BigInt) == BigInt[1 2 3] end + +# test writemime +@test sprint(io -> writemime(io, "text/csv", [1 2; 3 4])) == "1,2\n3,4\n" + From c2934b5c67fbeecb025b032425aae1829070fa0a Mon Sep 17 00:00:00 2001 From: Art Kuo Date: Tue, 13 Oct 2015 19:29:41 -0400 Subject: [PATCH 0551/1938] speed up print_matrix by avoiding unnecessary alignment() --- base/show.jl | 103 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 61 insertions(+), 42 deletions(-) diff --git a/base/show.jl b/base/show.jl index 68279acb7756d..8393d5d1e67f3 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1062,79 +1062,98 @@ function print_matrix_vdots(io::IO, end end +""" +`print_matrix(io, X)` composes an entire matrix, taking into account the screen size. +If X is too big, it will be nine-sliced with vertical, horizontal, or diagonal +ellipsis inserted as appropriate. +Optional parameters are screen size tuple sz such as (24,80), +string pre prior to the matrix (e.g. opening bracket), which will cause +a corresponding same-size indent on following rows, +string post on the end of the last row of the matrix. +Also options to use different ellipsis characters hdots, +vdots, ddots. The ellipsis are repeated every hmod or vmod element. +""" function print_matrix(io::IO, X::AbstractVecOrMat, sz::Tuple{Integer, Integer} = (s = tty_size(); (s[1]-4, s[2])), - pre::AbstractString = " ", - sep::AbstractString = " ", - post::AbstractString = "", + pre::AbstractString = " ", # pre-matrix string + sep::AbstractString = " ", # separator between elements + post::AbstractString = "", # post-matrix string hdots::AbstractString = " \u2026 ", vdots::AbstractString = "\u22ee", ddots::AbstractString = " \u22f1 ", hmod::Integer = 5, vmod::Integer = 5) - rows, cols = sz - cols -= length(pre) + length(post) - presp = repeat(" ", length(pre)) + screenheight, screenwidth = sz + screenwidth -= length(pre) + length(post) + presp = repeat(" ", length(pre)) # indent each row to match pre string postsp = "" @assert strwidth(hdots) == strwidth(ddots) - ss = length(sep) + sepsize = length(sep) m, n = size(X,1), size(X,2) - if m <= rows # rows fit - A = alignment(X,1:m,1:n,cols,cols,ss) - if n <= length(A) # rows and cols fit - for i = 1:m + # To figure out alignments, only need to look at as many rows as could + # fit down screen. If screen has at least as many rows as A, look at A. + # If not, then we only need to look at the first and last chunks of A, + # each half a screen height in size. + halfheight = div(screenheight,2) + rowsA = m <= screenheight ? (1:m) : [1:halfheight; m-div(screenheight-1,2)+1:m] + # Similarly for columns, only necessary to get alignments for as many + # columns as could conceivably fit across the screen + maxpossiblecols = div(screenwidth, 1+sepsize) + colsA = n <= maxpossiblecols ? (1:n) : [1:maxpossiblecols; (n-maxpossiblecols+1):n] + A = alignment(X,rowsA,colsA,screenwidth,screenwidth,sepsize) + # Nine-slicing is accomplished using print_matrix_row repeatedly + if m <= screenheight # rows fit vertically on screen + if n <= length(A) # rows and cols fit so just print whole matrix in one piece + for i in rowsA print(io, i == 1 ? pre : presp) - print_matrix_row(io, X,A,i,1:n,sep) + print_matrix_row(io, X,A,i,colsA,sep) print(io, i == m ? post : postsp) if i != m; println(io, ); end end - else # rows fit, cols don't - c = div(cols-length(hdots)+1,2)+1 - R = reverse(alignment(X,1:m,n:-1:1,c,c,ss)) - c = cols - sum(map(sum,R)) - (length(R)-1)*ss - length(hdots) - L = alignment(X,1:m,1:n,c,c,ss) - for i = 1:m + else # rows fit down screen but cols don't, so need horizontal ellipsis + c = div(screenwidth-length(hdots)+1,2)+1 # what goes to right of ellipsis + Ralign = reverse(alignment(X,rowsA,reverse(colsA),c,c,sepsize)) # alignments for right + c = screenwidth - sum(map(sum,Ralign)) - (length(Ralign)-1)*sepsize - length(hdots) + Lalign = alignment(X,rowsA,colsA,c,c,sepsize) # alignments for left of ellipsis + for i in rowsA print(io, i == 1 ? pre : presp) - print_matrix_row(io, X,L,i,1:length(L),sep) + print_matrix_row(io, X,Lalign,i,1:length(Lalign),sep) print(io, i % hmod == 1 ? hdots : repeat(" ", length(hdots))) - print_matrix_row(io, X,R,i,n-length(R)+1:n,sep) + print_matrix_row(io, X,Ralign,i,n-length(Ralign)+colsA,sep) print(io, i == m ? post : postsp) if i != m; println(io, ); end end end - else # rows don't fit - t = div(rows,2) - I = [1:t; m-div(rows-1,2)+1:m] - A = alignment(X,I,1:n,cols,cols,ss) - if n <= length(A) # rows don't fit, cols do - for i in I + else # rows don't fit so will need vertical ellipsis + if n <= length(A) # rows don't fit, cols do, so only vertical ellipsis + for i in rowsA print(io, i == 1 ? pre : presp) - print_matrix_row(io, X,A,i,1:n,sep) + print_matrix_row(io, X,A,i,colsA,sep) print(io, i == m ? post : postsp) - if i != I[end]; println(io, ); end - if i == t + if i != rowsA[end]; println(io, ); end + if i == halfheight print(io, i == 1 ? pre : presp) print_matrix_vdots(io, vdots,A,sep,vmod,1) println(io, i == m ? post : postsp) end end - else # neither rows nor cols fit - c = div(cols-length(hdots)+1,2)+1 - R = reverse(alignment(X,I,n:-1:1,c,c,ss)) - c = cols - sum(map(sum,R)) - (length(R)-1)*ss - length(hdots) - L = alignment(X,I,1:n,c,c,ss) - r = mod((length(R)-n+1),vmod) - for i in I + else # neither rows nor cols fit, so use all 3 kinds of ellipsis + c = div(screenwidth-length(hdots)+1,2)+1 + Ralign = reverse(alignment(X,rowsA,reverse(colsA),c,c,sepsize)) + c = screenwidth - sum(map(sum,Ralign)) - (length(Ralign)-1)*sepsize - length(hdots) + Lalign = alignment(X,rowsA,colsA,c,c,sepsize) + r = mod((length(Ralign)-n+1),vmod) # where to put dots on right half + for i in rowsA print(io, i == 1 ? pre : presp) - print_matrix_row(io, X,L,i,1:length(L),sep) + print_matrix_row(io, X,Lalign,i,1:length(Lalign),sep) print(io, i % hmod == 1 ? hdots : repeat(" ", length(hdots))) - print_matrix_row(io, X,R,i,n-length(R)+1:n,sep) + print_matrix_row(io, X,Ralign,i,n-length(Ralign)+colsA,sep) print(io, i == m ? post : postsp) - if i != I[end]; println(io, ); end - if i == t + if i != rowsA[end]; println(io, ); end + if i == halfheight print(io, i == 1 ? pre : presp) - print_matrix_vdots(io, vdots,L,sep,vmod,1) + print_matrix_vdots(io, vdots,Lalign,sep,vmod,1) print(io, ddots) - print_matrix_vdots(io, vdots,R,sep,vmod,r) + print_matrix_vdots(io, vdots,Ralign,sep,vmod,r) println(io, i == m ? post : postsp) end end From aa5d843aa17e271fb229c0cee70d9b2812a2f9b3 Mon Sep 17 00:00:00 2001 From: Art Kuo Date: Tue, 13 Oct 2015 19:31:55 -0400 Subject: [PATCH 0552/1938] tests for print_matrix with various sizes of arrays --- test/show.jl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/show.jl b/test/show.jl index 0fb185acbc365..846c57c5ce79b 100644 --- a/test/show.jl +++ b/test/show.jl @@ -302,3 +302,14 @@ function f13127() takebuf_string(buf) end @test f13127() == "f" + +# print_matrix should be able to handle small and large objects easily, test by +# calling writemime. This also indirectly tests print_matrix_row, which +# is used repeatedly by print_matrix. +# This fits on screen: +@test replstr(eye(10)) == "10x10 Array{Float64,2}:\n 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0\n 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0\n 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0\n 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0\n 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0\n 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0\n 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0" +# an array too long vertically to fit on screen, and too long horizontally: +@test replstr(collect(1.:100.)) == "100-element Array{Float64,1}:\n 1.0\n 2.0\n 3.0\n 4.0\n 5.0\n 6.0\n 7.0\n 8.0\n 9.0\n 10.0\n ⋮ \n 92.0\n 93.0\n 94.0\n 95.0\n 96.0\n 97.0\n 98.0\n 99.0\n 100.0" +@test replstr(collect(1.:100.)') == "1x100 Array{Float64,2}:\n 1.0 2.0 3.0 4.0 5.0 6.0 7.0 … 95.0 96.0 97.0 98.0 99.0 100.0" +# too big in both directions to fit on screen: +@test replstr((1.:100.)*(1:100)') == "100x100 Array{Float64,2}:\n 1.0 2.0 3.0 4.0 5.0 6.0 … 97.0 98.0 99.0 100.0\n 2.0 4.0 6.0 8.0 10.0 12.0 194.0 196.0 198.0 200.0\n 3.0 6.0 9.0 12.0 15.0 18.0 291.0 294.0 297.0 300.0\n 4.0 8.0 12.0 16.0 20.0 24.0 388.0 392.0 396.0 400.0\n 5.0 10.0 15.0 20.0 25.0 30.0 485.0 490.0 495.0 500.0\n 6.0 12.0 18.0 24.0 30.0 36.0 … 582.0 588.0 594.0 600.0\n 7.0 14.0 21.0 28.0 35.0 42.0 679.0 686.0 693.0 700.0\n 8.0 16.0 24.0 32.0 40.0 48.0 776.0 784.0 792.0 800.0\n 9.0 18.0 27.0 36.0 45.0 54.0 873.0 882.0 891.0 900.0\n 10.0 20.0 30.0 40.0 50.0 60.0 970.0 980.0 990.0 1000.0\n ⋮ ⋮ ⋱ \n 92.0 184.0 276.0 368.0 460.0 552.0 8924.0 9016.0 9108.0 9200.0\n 93.0 186.0 279.0 372.0 465.0 558.0 9021.0 9114.0 9207.0 9300.0\n 94.0 188.0 282.0 376.0 470.0 564.0 9118.0 9212.0 9306.0 9400.0\n 95.0 190.0 285.0 380.0 475.0 570.0 9215.0 9310.0 9405.0 9500.0\n 96.0 192.0 288.0 384.0 480.0 576.0 … 9312.0 9408.0 9504.0 9600.0\n 97.0 194.0 291.0 388.0 485.0 582.0 9409.0 9506.0 9603.0 9700.0\n 98.0 196.0 294.0 392.0 490.0 588.0 9506.0 9604.0 9702.0 9800.0\n 99.0 198.0 297.0 396.0 495.0 594.0 9603.0 9702.0 9801.0 9900.0\n 100.0 200.0 300.0 400.0 500.0 600.0 9700.0 9800.0 9900.0 10000.0" From d8b6a24f98551d3ad00c2fc31f1f5a02579fbd48 Mon Sep 17 00:00:00 2001 From: Michael Hatherly Date: Wed, 14 Oct 2015 08:04:42 +0200 Subject: [PATCH 0553/1938] Fix doc bootstrap method error. Docstrings defined before `==` is available for `Symbol` comparisons causes a build error. Fix by comparing with `===`. Ref: https://github.com/JuliaLang/julia/pull/13534#issuecomment-147906630 --- base/docs/bootstrap.jl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/base/docs/bootstrap.jl b/base/docs/bootstrap.jl index 9fbe4b4bfafe9..62f6e7cfc01d0 100644 --- a/base/docs/bootstrap.jl +++ b/base/docs/bootstrap.jl @@ -21,8 +21,8 @@ setexpand!(f) = global _expand_ = f function __bootexpand(str, obj) global docs = List((ccall(:jl_get_current_module, Any, ()), str, obj), docs) - (isa(obj, Expr) && obj.head == :call) && return nothing - (isa(obj, Expr) && obj.head == :module) && return esc(Expr(:toplevel, obj)) + (isa(obj, Expr) && obj.head === :call) && return nothing + (isa(obj, Expr) && obj.head === :module) && return esc(Expr(:toplevel, obj)) esc(obj) end @@ -45,6 +45,11 @@ that were stored in `DocBootstrap.docs` are migrated to their correct modules us """ DocBootstrap +""" + loaddocs() + +Move all docstrings from `DocBootstrap.docs` to their module's `__META__` dict. +""" function loaddocs() node = docs while node ≠ nothing From 065021373b1da17781eaca1ac0cd53787896a12b Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Tue, 13 Oct 2015 23:27:50 +0200 Subject: [PATCH 0554/1938] Replace a few one-liners with clearer blocks and clean spacing --- base/datafmt.jl | 94 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 32 deletions(-) diff --git a/base/datafmt.jl b/base/datafmt.jl index 595ec572216c0..9b8a5573a9419 100644 --- a/base/datafmt.jl +++ b/base/datafmt.jl @@ -15,7 +15,7 @@ invalid_dlm(::Type{UInt32}) = 0xfffffffe const offs_chunk_size = 5000 -countlines(f::AbstractString,eol::Char='\n') = open(io->countlines(io,eol),f)::Int +countlines(f::AbstractString, eol::Char='\n') = open(io->countlines(io,eol), f)::Int function countlines(io::IO, eol::Char='\n') isascii(eol) || throw(ArgumentError("only ASCII line terminators are supported")) a = Array(UInt8, 8192) @@ -41,7 +41,14 @@ readdlm(input, dlm::Char, T::Type, eol::Char; opts...) = readdlm_auto(input, dlm function readdlm_auto(input, dlm::Char, T::Type, eol::Char, auto::Bool; opts...) optsd = val_opts(opts) use_mmap = get(optsd, :use_mmap, @windows ? false : true) - isa(input, AbstractString) && (fsz = filesize(input); input = use_mmap && (fsz > 0) && fsz < typemax(Int) ? as_mmap(input,fsz) : readall(input)) + if isa(input, AbstractString) + fsz = filesize(input) + if use_mmap && fsz > 0 && fsz < typemax(Int) + input = as_mmap(input, fsz) + else + input = readall(input) + end + end sinp = isa(input, Vector{UInt8}) ? bytestring(input) : isa(input, IO) ? readall(input) : input @@ -55,7 +62,7 @@ function as_mmap(fname::AbstractString, fsz::Int64) end function ascii_if_possible(sbuff::AbstractString) - isascii(sbuff) ? convert(ASCIIString,sbuff) : sbuff + isascii(sbuff) ? convert(ASCIIString, sbuff) : sbuff end # @@ -75,7 +82,7 @@ type DLMOffsets <: DLMHandler function DLMOffsets(sbuff::ByteString) offsets = Array(Array{Int,1}, 1) offsets[1] = Array(Int, offs_chunk_size) - thresh = ceil(min(typemax(UInt),Base.Sys.total_memory()) / sizeof(Int) / 5) + thresh = ceil(min(typemax(UInt), Base.Sys.total_memory()) / sizeof(Int) / 5) new(offsets, 1, thresh, length(sbuff.data)) end end @@ -111,7 +118,7 @@ function store_cell(dlmoffsets::DLMOffsets, row::Int, col::Int, quoted::Bool, st end function result(dlmoffsets::DLMOffsets) - trimsz = (dlmoffsets.offidx-1)%offs_chunk_size + trimsz = (dlmoffsets.offidx-1) % offs_chunk_size ((trimsz > 0) || (dlmoffsets.offidx == 1)) && resize!(dlmoffsets.oarr[end], trimsz) dlmoffsets.oarr end @@ -165,7 +172,7 @@ function store_cell{T,S<:ByteString}(dlmstore::DLMStore{T,S}, row::Int, col::Int end for cidx in (lastcol+1):ncols if (T <: AbstractString) || (T == Any) - cells[lastrow,cidx] = SubString(sbuff, 1, 0) + cells[lastrow, cidx] = SubString(sbuff, 1, 0) elseif ((T <: Number) || (T <: Char)) && dlmstore.auto throw(TypeError(:store_cell, "", Any, T)) else @@ -177,13 +184,13 @@ function store_cell{T,S<:ByteString}(dlmstore::DLMStore{T,S}, row::Int, col::Int # fill data if quoted && _chrinstr(sbuff, UInt8('"'), startpos, endpos) - unescaped = replace(SubString(sbuff,startpos,endpos), r"\"\"", "\"") + unescaped = replace(SubString(sbuff, startpos, endpos), r"\"\"", "\"") fail = colval(unescaped, 1, length(unescaped), cells, drow, col) else fail = colval(sbuff, startpos, endpos, cells, drow, col) end if fail - sval = SubString(sbuff,startpos,endpos) + sval = SubString(sbuff, startpos, endpos) ((T <: Number) && dlmstore.auto) ? throw(TypeError(:store_cell, "", Any, T)) : error("file entry \"$(sval)\" cannot be converted to $T") end @@ -192,10 +199,10 @@ function store_cell{T,S<:ByteString}(dlmstore::DLMStore{T,S}, row::Int, col::Int else # fill header if quoted && _chrinstr(sbuff, UInt8('"'), startpos, endpos) - unescaped = replace(SubString(sbuff,startpos,endpos), r"\"\"", "\"") + unescaped = replace(SubString(sbuff, startpos, endpos), r"\"\"", "\"") colval(unescaped, 1, length(unescaped), dlmstore.hdr, 1, col) else - colval(sbuff, startpos,endpos, dlmstore.hdr, 1, col) + colval(sbuff, startpos, endpos, dlmstore.hdr, 1, col) end end @@ -215,7 +222,7 @@ function result{T}(dlmstore::DLMStore{T}) (lastcol == ncols) && (lastcol = 0; lastrow += 1) for cidx in (lastcol+1):ncols if (T <: AbstractString) || (T == Any) - cells[lastrow,cidx] = SubString(sbuff, 1, 0) + cells[lastrow, cidx] = SubString(sbuff, 1, 0) elseif ((T <: Number) || (T <: Char)) && dlmstore.auto throw(TypeError(:store_cell, "", Any, T)) else @@ -317,59 +324,82 @@ end function colval{S<:ByteString}(sbuff::S, startpos::Int, endpos::Int, cells::Array{Bool,2}, row::Int, col::Int) n = tryparse_internal(Bool, sbuff, startpos, endpos, false) - isnull(n) || (cells[row,col] = get(n)) + isnull(n) || (cells[row, col] = get(n)) isnull(n) end function colval{T<:Integer, S<:ByteString}(sbuff::S, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int) n = tryparse_internal(T, sbuff, startpos, endpos, 0, false) - isnull(n) || (cells[row,col] = get(n)) + isnull(n) || (cells[row, col] = get(n)) isnull(n) end function colval(sbuff::ByteString, startpos::Int, endpos::Int, cells::Array{Float64,2}, row::Int, col::Int) n = ccall(:jl_try_substrtod, Nullable{Float64}, (Ptr{UInt8},Csize_t,Csize_t), sbuff, startpos-1, endpos-startpos+1) - isnull(n) || (cells[row,col] = get(n)) + isnull(n) || (cells[row, col] = get(n)) isnull(n) end function colval(sbuff::ByteString, startpos::Int, endpos::Int, cells::Array{Float32,2}, row::Int, col::Int) - n = ccall(:jl_try_substrtof, Nullable{Float32}, (Ptr{UInt8},Csize_t,Csize_t), sbuff, startpos-1, endpos-startpos+1) - isnull(n) || (cells[row,col] = get(n)) + n = ccall(:jl_try_substrtof, Nullable{Float32}, (Ptr{UInt8}, Csize_t, Csize_t), sbuff, startpos-1, endpos-startpos+1) + isnull(n) || (cells[row, col] = get(n)) isnull(n) end -colval{T<:AbstractString, S<:ByteString}(sbuff::S, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int) = ((cells[row,col] = SubString(sbuff,startpos,endpos)); false) +function colval{T<:AbstractString, S<:ByteString}(sbuff::S, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int) + cells[row, col] = SubString(sbuff, startpos, endpos) + return false +end function colval{S<:ByteString}(sbuff::S, startpos::Int, endpos::Int, cells::Array{Any,2}, row::Int, col::Int) # if array is of Any type, attempt parsing only the most common types: Int, Bool, Float64 and fallback to SubString len = endpos-startpos+1 if len > 0 # check Inteter ni64 = tryparse_internal(Int, sbuff, startpos, endpos, 0, false) - isnull(ni64) || (cells[row,col] = get(ni64); return false) + isnull(ni64) || (cells[row, col] = get(ni64); return false) # check Bool nb = tryparse_internal(Bool, sbuff, startpos, endpos, false) - isnull(nb) || (cells[row,col] = get(nb); return false) + isnull(nb) || (cells[row, col] = get(nb); return false) # check float64 - nf64 = ccall(:jl_try_substrtod, Nullable{Float64}, (Ptr{UInt8},Csize_t,Csize_t), sbuff, startpos-1, endpos-startpos+1) - isnull(nf64) || (cells[row,col] = get(nf64); return false) + nf64 = ccall(:jl_try_substrtod, Nullable{Float64}, (Ptr{UInt8}, Csize_t, Csize_t), sbuff, startpos-1, endpos-startpos+1) + isnull(nf64) || (cells[row, col] = get(nf64); return false) end - cells[row,col] = SubString(sbuff, startpos, endpos) + cells[row, col] = SubString(sbuff, startpos, endpos) false end -colval{T<:Char, S<:ByteString}(sbuff::S, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int) = ((startpos==endpos) ? ((cells[row,col] = next(sbuff,startpos)[1]); false) : true) +function colval{T<:Char, S<:ByteString}(sbuff::S, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int) + if startpos == endpos + cells[row, col] = next(sbuff, startpos)[1] + return false + else + return true + end +end colval{S<:ByteString}(sbuff::S, startpos::Int, endpos::Int, cells::Array, row::Int, col::Int) = true -dlm_parse(s::ASCIIString, eol::Char, dlm::Char, qchar::Char, cchar::Char, ign_adj_dlm::Bool, allow_quote::Bool, allow_comments::Bool, skipstart::Int, skipblanks::Bool, dh::DLMHandler) = begin - dlm_parse(s.data, reinterpret(UInt32,eol)%UInt8, reinterpret(UInt32,dlm)%UInt8, reinterpret(UInt32,qchar)%UInt8, reinterpret(UInt32,cchar)%UInt8, +function dlm_parse(s::ASCIIString, eol::Char, dlm::Char, qchar::Char, cchar::Char, + ign_adj_dlm::Bool, allow_quote::Bool, allow_comments::Bool, + skipstart::Int, skipblanks::Bool, dh::DLMHandler) + dlm_parse(s.data, reinterpret(UInt32, eol) % UInt8, reinterpret(UInt32, dlm) % UInt8, + reinterpret(UInt32, qchar) % UInt8, reinterpret(UInt32, cchar) % UInt8, ign_adj_dlm, allow_quote, allow_comments, skipstart, skipblanks, dh) end -function dlm_parse{T,D}(dbuff::T, eol::D, dlm::D, qchar::D, cchar::D, ign_adj_dlm::Bool, allow_quote::Bool, allow_comments::Bool, skipstart::Int, skipblanks::Bool, dh::DLMHandler) - all_ascii = (D <: UInt8) || (isascii(eol) && isascii(dlm) && (!allow_quote || isascii(qchar)) && (!allow_comments || isascii(cchar))) - (T <: UTF8String) && all_ascii && (return dlm_parse(dbuff.data, eol%UInt8, dlm%UInt8, qchar%UInt8, cchar%UInt8, ign_adj_dlm, allow_quote, allow_comments, skipstart, skipblanks, dh)) +function dlm_parse{T,D}(dbuff::T, eol::D, dlm::D, qchar::D, cchar::D, + ign_adj_dlm::Bool, allow_quote::Bool, allow_comments::Bool, + skipstart::Int, skipblanks::Bool, dh::DLMHandler) + all_ascii = (D <: UInt8) || (isascii(eol) && + isascii(dlm) && + (!allow_quote || isascii(qchar)) && + (!allow_comments || isascii(cchar))) + if (T <: UTF8String) && all_ascii + return dlm_parse(dbuff.data, eol % UInt8, dlm % UInt8, qchar % UInt8, cchar % UInt8, + ign_adj_dlm, allow_quote, allow_comments, skipstart, skipblanks, dh) + end ncols = nrows = col = 0 is_default_dlm = (dlm == invalid_dlm(D)) error_str = "" - # 0: begin field, 1: quoted field, 2: unquoted field, 3: second quote (could either be end of field or escape character), 4: comment, 5: skipstart + # 0: begin field, 1: quoted field, 2: unquoted field, + # 3: second quote (could either be end of field or escape character), + # 4: comment, 5: skipstart state = (skipstart > 0) ? 5 : 0 is_eol = is_dlm = is_cr = is_quote = is_comment = expct_col = false idx = 1 @@ -537,11 +567,11 @@ function writedlm(io::IO, a::AbstractVecOrMat, dlm; opts...) optsd = val_opts(opts) quotes = get(optsd, :quotes, true) pb = PipeBuffer() - nr = size(a,1) - nc = size(a,2) + nr = size(a, 1) + nc = size(a, 2) for i = 1:nr for j = 1:nc - writedlm_cell(pb, a[i,j], dlm, quotes) + writedlm_cell(pb, a[i, j], dlm, quotes) j == nc ? write(pb,'\n') : print(pb,dlm) end (nb_available(pb) > (16*1024)) && write(io, takebuf_array(pb)) From 2707db32112f3fb8d6d92bb79e0efb1b2eb22db3 Mon Sep 17 00:00:00 2001 From: Guru Devanla Date: Wed, 14 Oct 2015 04:17:25 -0700 Subject: [PATCH 0555/1938] Update types.rst --- doc/manual/types.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/types.rst b/doc/manual/types.rst index 09572088b8616..84e3aab036411 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -1302,7 +1302,7 @@ are safe. At present, the interface consists of four possible interactions: - Access the value of a :obj:`Nullable` object with a guarantee that a :exc:`NullException` will be thrown if the object's value is missing. - Access the value of a :obj:`Nullable` object with a guarantee that a default - value of type ``T`` will be returned if the object's value is missing. + value of type ``T`` will be returned if the object's value is not missing. Constructing :obj:`Nullable` objects ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 87131f399eda8e8e35adfce16ce3e34b988a523e Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Thu, 8 Oct 2015 10:02:58 -0400 Subject: [PATCH 0556/1938] Deprecate A_ldiv_B!(SparseMatrixCSC, StrideVecOrMat) thereby fixing #10787 --- NEWS.md | 3 +++ base/deprecated.jl | 4 ++++ base/sparse/linalg.jl | 14 -------------- test/sparsedir/sparse.jl | 3 --- 4 files changed, 7 insertions(+), 17 deletions(-) diff --git a/NEWS.md b/NEWS.md index 75a5cc9246855..8adc4a08e24ae 100644 --- a/NEWS.md +++ b/NEWS.md @@ -45,6 +45,9 @@ Library improvements Deprecated or removed --------------------- + * The method `A_ldiv_B!(SparseMatrixCSC, StrideVecOrMat)` has been deprecated in favor + of versions that require the matrix to in factored form ([#13496]). + Julia v0.4.0 Release Notes ========================== diff --git a/base/deprecated.jl b/base/deprecated.jl index 687ff736c9316..555bcc2197013 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -843,6 +843,7 @@ for f in (:remotecall, :remotecall_fetch, :remotecall_wait) end end +#13465 @deprecate cov(x::AbstractVector; corrected=true, mean=Base.mean(x)) covm(x, mean, corrected) @deprecate cov(X::AbstractMatrix; vardim=1, corrected=true, mean=Base.mean(X, vardim)) covm(X, mean, vardim, corrected) @deprecate cov(x::AbstractVector, y::AbstractVector; corrected=true, mean=(Base.mean(x), Base.mean(y))) covm(x, mean[1], y, mean[2], corrected) @@ -854,3 +855,6 @@ end @deprecate cor(X::AbstractVecOrMat, Y::AbstractVecOrMat; vardim=1, mean=(Base.mean(X, vardim), Base.mean(Y, vardim))) corm(X, mean[1], Y, mean[2], vardim) @deprecate_binding SparseMatrix SparseArrays + +#13496 +@deprecate A_ldiv_B!(A::SparseMatrixCSC, B::StridedVecOrMat) A_ldiv_B!(factorize(A), B) diff --git a/base/sparse/linalg.jl b/base/sparse/linalg.jl index 9475575073df7..526365ab55b0f 100644 --- a/base/sparse/linalg.jl +++ b/base/sparse/linalg.jl @@ -165,20 +165,6 @@ function spmatmul{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, B::SparseMatrixCSC{Tv,Ti}; end ## solvers -function A_ldiv_B!(A::SparseMatrixCSC, b::AbstractVecOrMat) - if eltype(b)<:Complex; A = complex(A); end - - if istril(A) - # TODO: Fix diagonal case. Diagonal(A.nzval) needs to handle - # the case where there are zeros on the diagonal and error out. - # It also does not work in the complex case. VBS. - #if istriu(A); return A_ldiv_B!(Diagonal(A.nzval), b); end - return fwdTriSolve!(A, b) - end - if istriu(A); return bwdTriSolve!(A, b); end - return A_ldiv_B!(lufact(A),b) -end - function fwdTriSolve!(A::SparseMatrixCSC, B::AbstractVecOrMat) # forward substitution for CSC matrices n = length(B) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 27e363354888b..1fa66634cec7a 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -126,7 +126,6 @@ for i = 1:5 @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) - @test (maximum(abs(A_ldiv_B!(a,copy(b)) - full(a)\b)) < 1000*eps()) a = speye(5) + tril(0.1*sprandn(5, 5, 0.2) + 0.1*im*sprandn(5, 5, 0.2)) b = randn(5,3) @@ -145,7 +144,6 @@ for i = 1:5 @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) - @test (maximum(abs(A_ldiv_B!(a,copy(b)) - full(a)\b)) < 1000*eps()) a = speye(5) + triu(0.1*sprandn(5, 5, 0.2) + 0.1*im*sprandn(5, 5, 0.2)) b = randn(5,3) @@ -164,7 +162,6 @@ for i = 1:5 @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) - @test (maximum(abs(A_ldiv_B!(a,copy(b)) - full(a)\b)) < 1000*eps()) a = spdiagm(randn(5)) + im*spdiagm(randn(5)) b = randn(5,3) From 7e4edbbc0eee5a3e0066c9c318e1a7a7e02c441b Mon Sep 17 00:00:00 2001 From: John Myles White Date: Wed, 14 Oct 2015 08:19:57 -0700 Subject: [PATCH 0557/1938] Revert "Update types.rst" --- doc/manual/types.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/types.rst b/doc/manual/types.rst index 84e3aab036411..09572088b8616 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -1302,7 +1302,7 @@ are safe. At present, the interface consists of four possible interactions: - Access the value of a :obj:`Nullable` object with a guarantee that a :exc:`NullException` will be thrown if the object's value is missing. - Access the value of a :obj:`Nullable` object with a guarantee that a default - value of type ``T`` will be returned if the object's value is not missing. + value of type ``T`` will be returned if the object's value is missing. Constructing :obj:`Nullable` objects ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From dcb44c5f7d0907e3080bafd83b8725c940965b1e Mon Sep 17 00:00:00 2001 From: Art Kuo Date: Wed, 14 Oct 2015 11:32:36 -0400 Subject: [PATCH 0558/1938] Made sure ellipsis was used consistently as singular; avoided plural --- base/show.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/show.jl b/base/show.jl index 8393d5d1e67f3..9d25110fb7fd5 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1071,7 +1071,7 @@ string pre prior to the matrix (e.g. opening bracket), which will cause a corresponding same-size indent on following rows, string post on the end of the last row of the matrix. Also options to use different ellipsis characters hdots, -vdots, ddots. The ellipsis are repeated every hmod or vmod element. +vdots, ddots. These are repeated every hmod or vmod elements. """ function print_matrix(io::IO, X::AbstractVecOrMat, sz::Tuple{Integer, Integer} = (s = tty_size(); (s[1]-4, s[2])), @@ -1136,7 +1136,7 @@ function print_matrix(io::IO, X::AbstractVecOrMat, println(io, i == m ? post : postsp) end end - else # neither rows nor cols fit, so use all 3 kinds of ellipsis + else # neither rows nor cols fit, so use all 3 kinds of dots c = div(screenwidth-length(hdots)+1,2)+1 Ralign = reverse(alignment(X,rowsA,reverse(colsA),c,c,sepsize)) c = screenwidth - sum(map(sum,Ralign)) - (length(Ralign)-1)*sepsize - length(hdots) From 92e32e78248d638b3e63331d6134a34209a39cd1 Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Wed, 14 Oct 2015 13:11:42 -0400 Subject: [PATCH 0559/1938] Fix #13610, long compilation time for Date(::Day, ::Month, ::Year) --- base/dates/types.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/base/dates/types.jl b/base/dates/types.jl index 36b92c20808d3..7a889c994a2b6 100644 --- a/base/dates/types.jl +++ b/base/dates/types.jl @@ -109,22 +109,22 @@ function DateTime(periods::Period...) y = Year(1); m = Month(1); d = Day(1) h = Hour(0); mi = Minute(0); s = Second(0); ms = Millisecond(0) for p in periods - typeof(p) <: Year && (y = p) - typeof(p) <: Month && (m = p) - typeof(p) <: Day && (d = p) - typeof(p) <: Hour && (h = p) - typeof(p) <: Minute && (mi = p) - typeof(p) <: Second && (s = p) - typeof(p) <: Millisecond && (ms = p) + isa(p, Year) && (y = p::Year) + isa(p, Month) && (m = p::Month) + isa(p, Day) && (d = p::Day) + isa(p, Hour) && (h = p::Hour) + isa(p, Minute) && (mi = p::Minute) + isa(p, Second) && (s = p::Second) + isa(p, Millisecond) && (ms = p::Millisecond) end return DateTime(y,m,d,h,mi,s,ms) end function Date(periods::Period...) y = Year(1); m = Month(1); d = Day(1) for p in periods - typeof(p) <: Year && (y = p) - typeof(p) <: Month && (m = p) - typeof(p) <: Day && (d = p) + isa(p, Year) && (y = p::Year) + isa(p, Month) && (m = p::Month) + isa(p, Day) && (d = p::Day) end return Date(y,m,d) end From 9575c2db6819609a6d4e1d9cbc2221f58ce59467 Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Wed, 14 Oct 2015 17:49:56 -0400 Subject: [PATCH 0560/1938] :whale: Avoid unnecessary recompilation in Docker images --- base/loading.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/loading.jl b/base/loading.jl index 89584055f4a6e..d6db483572119 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -508,7 +508,8 @@ function stale_cachefile(modpath, cachefile) return true # cache file was compiled from a different path end for (f,ftime) in files - if mtime(f) != ftime + # Issue #13606: compensate for Docker images rounding mtimes + if mtime(f) ∉ (ftime, floor(ftime)) return true end end From 17e2e1c42855578ab6c558248365c49bc4d2f55f Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Tue, 13 Oct 2015 15:35:16 +0530 Subject: [PATCH 0561/1938] restart worker during tests depending on resident memory size --- base/multi.jl | 56 ++-- base/sysinfo.jl | 2 + src/sys.c | 28 +- test/choosetests.jl | 5 +- test/examples.jl | 9 +- test/parallel.jl | 577 +----------------------------------------- test/parallel_exec.jl | 568 +++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 40 ++- test/testdefs.jl | 15 +- 9 files changed, 690 insertions(+), 610 deletions(-) create mode 100644 test/parallel_exec.jl diff --git a/base/multi.jl b/base/multi.jl index 5f977eccd0f03..bce8d7b26e79f 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -336,30 +336,35 @@ function rmprocs(args...; waitfor = 0.0) error("only process 1 can add and remove processes") end - rmprocset = [] - for i in vcat(args...) - if i == 1 - warn("rmprocs: process 1 not removed") - else - if haskey(map_pid_wrkr, i) - w = map_pid_wrkr[i] - set_worker_state(w, W_TERMINATING) - kill(w.manager, i, w.config) - push!(rmprocset, w) + lock(worker_lock) + try + rmprocset = [] + for i in vcat(args...) + if i == 1 + warn("rmprocs: process 1 not removed") + else + if haskey(map_pid_wrkr, i) + w = map_pid_wrkr[i] + set_worker_state(w, W_TERMINATING) + kill(w.manager, i, w.config) + push!(rmprocset, w) + end end end - end - start = time() - while (time() - start) < waitfor - if all(w -> w.state == W_TERMINATED, rmprocset) - break; - else - sleep(0.1) + start = time() + while (time() - start) < waitfor + if all(w -> w.state == W_TERMINATED, rmprocset) + break; + else + sleep(0.1) + end end - end - ((waitfor > 0) && any(w -> w.state != W_TERMINATED, rmprocset)) ? :timed_out : :ok + ((waitfor > 0) && any(w -> w.state != W_TERMINATED, rmprocset)) ? :timed_out : :ok + finally + unlock(worker_lock) + end end @@ -1075,7 +1080,20 @@ end # `manager` is of type ClusterManager. The respective managers are responsible # for launching the workers. All keyword arguments (plus a few default values) # are available as a dictionary to the `launch` methods +# +# Only one addprocs can be in progress at any time +# +const worker_lock = ReentrantLock() function addprocs(manager::ClusterManager; kwargs...) + lock(worker_lock) + try + addprocs_locked(manager::ClusterManager; kwargs...) + finally + unlock(worker_lock) + end +end + +function addprocs_locked(manager::ClusterManager; kwargs...) params = merge(default_addprocs_params(), AnyDict(kwargs)) topology(symbol(params[:topology])) diff --git a/base/sysinfo.jl b/base/sysinfo.jl index f02950920c46b..030abfbacca8c 100644 --- a/base/sysinfo.jl +++ b/base/sysinfo.jl @@ -146,4 +146,6 @@ function set_process_title(title::AbstractString) uv_error("set_process_title", err) end +maxrss() = ccall(:jl_maxrss, Csize_t, ()) + end # module Sys diff --git a/src/sys.c b/src/sys.c index 4d88301b8aed3..98fe7ef3bc85a 100644 --- a/src/sys.c +++ b/src/sys.c @@ -10,7 +10,9 @@ #include #include #include -#ifndef _OS_WINDOWS_ +#ifdef _OS_WINDOWS_ +#include +#else #include #include #include @@ -742,6 +744,30 @@ DLLEXPORT jl_sym_t* jl_get_ARCH() return ARCH; } +DLLEXPORT size_t jl_maxrss() +{ +#if defined(_OS_WINDOWS_) + PROCESS_MEMORY_COUNTERS counter; + GetProcessMemoryInfo( GetCurrentProcess( ), &counter, sizeof(counter) ); + return (size_t)counter.PeakWorkingSetSize; + +#elif defined(_OS_LINUX_) || defined(_OS_DARWIN_) || defined (_OS_FREEBSD_) + struct rusage rusage; + getrusage( RUSAGE_SELF, &rusage ); + +#if defined(_OS_LINUX_) + return (size_t)(rusage.ru_maxrss * 1024); +#else + return (size_t)rusage.ru_maxrss; +#endif + +#else + return (size_t)0; +#endif +} + + + #ifdef __cplusplus } #endif diff --git a/test/choosetests.jl b/test/choosetests.jl index 2a2eecfcbe248..ad3027f08d9a8 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -32,7 +32,7 @@ function choosetests(choices = []) "nullable", "meta", "profile", "libgit2", "docs", "markdown", "base64", "serialize", "functors", "misc", "enums", "cmdlineargs", "i18n", "workspace", "libdl", "int", - "intset", "floatfuncs", "compile" + "intset", "floatfuncs", "compile", "parallel" ] if Base.USE_GPL_LIBS @@ -43,9 +43,6 @@ function choosetests(choices = []) push!(testnames, "examples") end - # parallel tests depend on other workers - do them last - push!(testnames, "parallel") - tests = [] skip_tests = [] diff --git a/test/examples.jl b/test/examples.jl index 764d0e1d82666..fc8e76ecb2a88 100644 --- a/test/examples.jl +++ b/test/examples.jl @@ -52,14 +52,13 @@ end dc_path = joinpath(dir, "dictchannel.jl") include(dc_path) -w_set=filter!(x->x != myid(), workers()) -pid = length(w_set) > 0 ? w_set[1] : myid() - -remotecall_fetch(pid, dc_path) do f +# Run the remote on pid 1, since runtests may terminate workers +# at any time depending on memory usage +remotecall_fetch(1, dc_path) do f include(f) nothing end -dc=RemoteRef(()->DictChannel(), pid) +dc=RemoteRef(()->DictChannel(), 1) @test typeof(dc) == RemoteRef{DictChannel} @test isready(dc) == false diff --git a/test/parallel.jl b/test/parallel.jl index 7c4e523b14fcf..777cec5bfc300 100644 --- a/test/parallel.jl +++ b/test/parallel.jl @@ -1,573 +1,14 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -# NOTE: worker processes cannot add more workers, only the client process can. -using Base.Test +# Run the parallel test outside of the main driver, since it runs off its own +# set of workers. -if nworkers() < 3 - remotecall_fetch(1) do - addprocs(3 - nworkers()) - end -end - -id_me = myid() -id_other = filter(x -> x != id_me, procs())[rand(1:(nprocs()-1))] - -@test fetch(@spawnat id_other myid()) == id_other -@test @fetchfrom id_other begin myid() end == id_other -@fetch begin myid() end - -rr=RemoteRef() -@test typeof(rr) == RemoteRef{Channel{Any}} -a = rand(5,5) -put!(rr, a) -@test rr[2,3] == a[2,3] -@test rr[] == a - -rr=RemoteRef(workers()[1]) -@test typeof(rr) == RemoteRef{Channel{Any}} -a = rand(5,5) -put!(rr, a) -@test rr[1,5] == a[1,5] -@test rr[] == a - -dims = (20,20,20) - -@linux_only begin - S = SharedArray(Int64, dims) - @test startswith(S.segname, "/jl") - @test !ispath("/dev/shm" * S.segname) - - S = SharedArray(Int64, dims; pids=[id_other]) - @test startswith(S.segname, "/jl") - @test !ispath("/dev/shm" * S.segname) -end - -# TODO : Need a similar test of shmem cleanup for OSX - -##### SharedArray tests - -function check_pids_all(S::SharedArray) - pidtested = falses(size(S)) - for p in procs(S) - idxes_in_p = remotecall_fetch(p, S) do D - parentindexes(D.loc_subarr_1d)[1] - end - @test all(sdata(S)[idxes_in_p] .== p) - pidtested[idxes_in_p] = true - end - @test all(pidtested) -end - -d = Base.shmem_rand(1:100, dims) -a = convert(Array, d) - -partsums = Array(Int, length(procs(d))) -@sync begin - for (i, p) in enumerate(procs(d)) - @async partsums[i] = remotecall_fetch(p, d) do D - sum(D.loc_subarr_1d) - end - end -end -@test sum(a) == sum(partsums) - -d = Base.shmem_rand(dims) -for p in procs(d) - idxes_in_p = remotecall_fetch(p, d) do D - parentindexes(D.loc_subarr_1d)[1] - end - idxf = first(idxes_in_p) - idxl = last(idxes_in_p) - d[idxf] = Float64(idxf) - rv = remotecall_fetch(p, d,idxf,idxl) do D,idxf,idxl - assert(D[idxf] == Float64(idxf)) - D[idxl] = Float64(idxl) - D[idxl] - end - @test d[idxl] == rv -end - -@test ones(10, 10, 10) == Base.shmem_fill(1.0, (10,10,10)) -@test zeros(Int32, 10, 10, 10) == Base.shmem_fill(0, (10,10,10)) - -d = Base.shmem_rand(dims) -s = Base.shmem_rand(dims) -copy!(s, d) -@test s == d -s = Base.shmem_rand(dims) -copy!(s, sdata(d)) -@test s == d -a = rand(dims) -@test sdata(a) == a - -d = SharedArray(Int, dims; init = D->fill!(D.loc_subarr_1d, myid())) -for p in procs(d) - idxes_in_p = remotecall_fetch(p, d) do D - parentindexes(D.loc_subarr_1d)[1] - end - idxf = first(idxes_in_p) - idxl = last(idxes_in_p) - @test d[idxf] == p - @test d[idxl] == p -end - -d = SharedArray(Float64, (2,3)) -@test isa(d[:,2], Vector{Float64}) - -### SharedArrays from a file - -# Mapping an existing file -fn = tempname() -open(fn, "w") do io - write(io, 1:30) -end -sz = (6,5) -Atrue = reshape(1:30, sz) - -S = SharedArray(fn, Int, sz) -@test S == Atrue -@test length(procs(S)) > 1 -@sync begin - for p in procs(S) - @async remotecall_wait(p, S) do D - fill!(D.loc_subarr_1d, myid()) - end - end -end -check_pids_all(S) - -filedata = similar(Atrue) -open(fn, "r") do io - read!(io, filedata) -end -@test filedata == sdata(S) - -# Error for write-only files -@test_throws ArgumentError SharedArray(fn, Int, sz, mode="w") - -# Error for file doesn't exist, but not allowed to create -@test_throws ArgumentError SharedArray(tempname(), Int, sz, mode="r") - -# Creating a new file -fn2 = tempname() -S = SharedArray(fn2, Int, sz, init=D->D[localindexes(D)] = myid()) -@test S == filedata -filedata2 = similar(Atrue) -open(fn2, "r") do io - read!(io, filedata2) -end -@test filedata == filedata2 - -# Appending to a file -fn3 = tempname() -open(fn3, "w") do io - write(io, ones(UInt8, 4)) -end -S = SharedArray(fn3, UInt8, sz, 4, mode="a+", init=D->D[localindexes(D)]=0x02) -len = prod(sz)+4 -@test filesize(fn3) == len -filedata = Array(UInt8, len) -open(fn3, "r") do io - read!(io, filedata) -end -@test all(filedata[1:4] .== 0x01) -@test all(filedata[5:end] .== 0x02) - -@unix_only begin # these give unlink: operation not permitted (EPERM) on Windows - rm(fn); rm(fn2); rm(fn3) -end - -### Utility functions - -# construct PR #13514 -S = SharedArray{Int}((1,2,3)) -@test size(S) == (1,2,3) -@test typeof(S) <: SharedArray{Int} -S = SharedArray{Int}(2) -@test size(S) == (2,) -@test typeof(S) <: SharedArray{Int} -S = SharedArray{Int}(1,2) -@test size(S) == (1,2) -@test typeof(S) <: SharedArray{Int} -S = SharedArray{Int}(1,2,3) -@test size(S) == (1,2,3) -@test typeof(S) <: SharedArray{Int} - -# reshape - -d = Base.shmem_fill(1.0, (10,10,10)) -@test ones(100, 10) == reshape(d,(100,10)) -d = Base.shmem_fill(1.0, (10,10,10)) -@test_throws DimensionMismatch reshape(d,(50,)) - -# rand, randn -d = Base.shmem_rand(dims) -@test size(rand!(d)) == dims -d = Base.shmem_fill(1.0, dims) -@test size(randn!(d)) == dims +cmd = `$(Base.julia_cmd()) --check-bounds=yes --depwarn=error parallel_exec.jl` -# similar -d = Base.shmem_rand(dims) -@test size(similar(d, Complex128)) == dims -@test size(similar(d, dims)) == dims - -# issue #6362 -d = Base.shmem_rand(dims) -s = copy(sdata(d)) -ds = deepcopy(d) -@test ds == d -pids_d = procs(d) -remotecall_fetch(setindex!, pids_d[findfirst(id->(id != myid()), pids_d)], d, 1.0, 1:10) -@test ds != d -@test s != d - - -# SharedArray as an array -# Since the data in d will depend on the nprocs, just test that these operations work -a = d[1:5] -@test_throws BoundsError d[-1:5] -a = d[1,1,1:3:end] -d[2:4] = 7 -d[5,1:2:4,8] = 19 - -AA = rand(4,2) -A = convert(SharedArray, AA) -B = convert(SharedArray, AA') -@test B*A == ctranspose(AA)*AA - -d=SharedArray(Int64, (10,10); init = D->fill!(D.loc_subarr_1d, myid()), pids=[id_me, id_other]) -d2 = map(x->1, d) -@test reduce(+, d2) == 100 - -@test reduce(+, d) == ((50*id_me) + (50*id_other)) -map!(x->1, d) -@test reduce(+, d) == 100 - -@test fill!(d, 1) == ones(10, 10) -@test fill!(d, 2.) == fill(2, 10, 10) -@test d[:] == fill(2, 100) -@test d[:,1] == fill(2, 10) -@test d[1,:] == fill(2, 1, 10) - -# Boundary cases where length(S) <= length(pids) -@test 2.0 == remotecall_fetch(D->D[2], id_other, Base.shmem_fill(2.0, 2; pids=[id_me, id_other])) -@test 3.0 == remotecall_fetch(D->D[1], id_other, Base.shmem_fill(3.0, 1; pids=[id_me, id_other])) - -# Test @parallel load balancing - all processors should get either M or M+1 -# iterations out of the loop range for some M. -workloads = hist(@parallel((a,b)->[a;b], for i=1:7; myid(); end), nprocs())[2] -@test maximum(workloads) - minimum(workloads) <= 1 - -# @parallel reduction should work even with very short ranges -@test @parallel(+, for i=1:2; i; end) == 3 - -# Testing timedwait on multiple channels -@sync begin - rr1 = Channel() - rr2 = Channel() - rr3 = Channel() - - @async begin sleep(0.5); put!(rr1, :ok) end - @async begin sleep(1.0); put!(rr2, :ok) end - @async begin sleep(2.0); put!(rr3, :ok) end - - tic() - timedwait(1.0) do - all(map(isready, [rr1, rr2, rr3])) - end - et=toq() - # assuming that 0.5 seconds is a good enough buffer on a typical modern CPU - try - @test (et >= 1.0) && (et <= 1.5) - @test !isready(rr3) - catch - warn("timedwait tests delayed. et=$et, isready(rr3)=$(isready(rr3))") - end - @test isready(rr1) -end - -@test_throws ArgumentError sleep(-1) -@test_throws ArgumentError timedwait(()->false, 0.1, pollint=-0.5) - -# specify pids for pmap -@test sort(workers()[1:2]) == sort(unique(pmap(x->(sleep(0.1);myid()), 1:10, pids = workers()[1:2]))) - -# Testing buffered and unbuffered reads -# This large array should write directly to the socket -a = ones(10^6) -@test a == remotecall_fetch((x)->x, id_other, a) - -# Not a bitstype, should be buffered -s = [randstring() for x in 1:10^5] -@test s == remotecall_fetch((x)->x, id_other, s) - -#large number of small requests -num_small_requests = 10000 -@test fill(id_other, num_small_requests) == [remotecall_fetch(myid, id_other) for i in 1:num_small_requests] - -# test parallel sends of large arrays from multiple tasks to the same remote worker -ntasks = 10 -rr_list = [Channel() for x in 1:ntasks] -a=ones(2*10^5); -for rr in rr_list - @async let rr=rr - try - for i in 1:10 - @test a == remotecall_fetch((x)->x, id_other, a) - yield() - end - put!(rr, :OK) - catch - put!(rr, :ERROR) - end - end -end - -@test [fetch(rr) for rr in rr_list] == [:OK for x in 1:ntasks] - -function test_channel(c) - put!(c, 1) - put!(c, "Hello") - put!(c, 5.0) - - @test isready(c) == true - @test fetch(c) == 1 - @test fetch(c) == 1 # Should not have been popped previously - @test take!(c) == 1 - @test take!(c) == "Hello" - @test fetch(c) == 5.0 - @test take!(c) == 5.0 - @test isready(c) == false - close(c) -end - -test_channel(Channel(10)) -test_channel(RemoteRef(()->Channel(10))) - -c=Channel{Int}(1) -@test_throws MethodError put!(c, "Hello") - -c=Channel(256) -# Test growth of channel -@test c.szp1 <= 33 -for x in 1:40 - put!(c, x) -end -@test c.szp1 <= 65 -for x in 1:39 - take!(c) -end -for x in 1:64 - put!(c, x) -end -@test (c.szp1 > 65) && (c.szp1 <= 129) -for x in 1:39 - take!(c) -end -@test fetch(c) == 39 -for x in 1:26 - take!(c) -end -@test isready(c) == false - -# test channel iterations -function test_iteration(in_c, out_c) - t=@schedule for v in in_c - put!(out_c, v) - end - - isa(in_c, Channel) && @test isopen(in_c) == true - put!(in_c, 1) - @test take!(out_c) == 1 - put!(in_c, "Hello") - close(in_c) - @test take!(out_c) == "Hello" - isa(in_c, Channel) && @test isopen(in_c) == false - @test_throws InvalidStateException put!(in_c, :foo) - yield() - @test istaskdone(t) == true -end - -test_iteration(Channel(10), Channel(10)) -# make sure exceptions propagate when waiting on Tasks -@test_throws CompositeException (@sync (@async error("oops"))) -try - @sync begin - for i in 1:5 - @async error(i) - end - end - error("unexpected") -catch ex - @test typeof(ex) == CompositeException - @test length(ex) == 5 - @test typeof(ex.exceptions[1]) == CapturedException - @test typeof(ex.exceptions[1].ex) == ErrorException - errors = map(x->x.ex.msg, ex.exceptions) - @test collect(1:5) == sort(map(x->parse(Int, x), errors)) -end - -try - remotecall_fetch(()->throw(ErrorException("foobar")), id_other) - error("unexpected") -catch ex - @test typeof(ex) == RemoteException - @test typeof(ex.captured) == CapturedException - @test typeof(ex.captured.ex) == ErrorException - @test ex.captured.ex.msg == "foobar" -end - -# The below block of tests are usually run only on local development systems, since: -# - tests which print errors -# - addprocs tests are memory intensive -# - ssh addprocs requires sshd to be running locally with passwordless login enabled. -# The test block is enabled by defining env JULIA_TESTFULL=1 - -DoFullTest = Bool(parse(Int,(get(ENV, "JULIA_TESTFULL", "0")))) - -if DoFullTest - # pmap tests - # needs at least 4 processors dedicated to the below tests - ppids = remotecall_fetch(()->addprocs(4), 1) - s = "abcdefghijklmnopqrstuvwxyz"; - ups = uppercase(s); - @test ups == bytestring(UInt8[UInt8(c) for c in pmap(x->uppercase(x), s)]) - @test ups == bytestring(UInt8[UInt8(c) for c in pmap(x->uppercase(Char(x)), s.data)]) - - # retry, on error exit - res = pmap(x->(x=='a') ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : (sleep(0.1);uppercase(x)), s; err_retry=true, err_stop=true, pids=ppids); - @test (length(res) < length(ups)) - @test isa(res[1], Exception) - - # no retry, on error exit - res = pmap(x->(x=='a') ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : (sleep(0.1);uppercase(x)), s; err_retry=false, err_stop=true, pids=ppids); - @test (length(res) < length(ups)) - @test isa(res[1], Exception) - - # retry, on error continue - res = pmap(x->iseven(myid()) ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : (sleep(0.1);uppercase(x)), s; err_retry=true, err_stop=false, pids=ppids); - @test length(res) == length(ups) - @test ups == bytestring(UInt8[UInt8(c) for c in res]) - - # no retry, on error continue - res = pmap(x->(x=='a') ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : (sleep(0.1);uppercase(x)), s; err_retry=false, err_stop=false, pids=ppids); - @test length(res) == length(ups) - @test isa(res[1], Exception) - - # Topology tests need to run externally since a given cluster at any - # time can only support a single topology and the current session - # is already running in parallel under the default topology. - script = joinpath(dirname(@__FILE__), "topology.jl") - cmd = `$(joinpath(JULIA_HOME,Base.julia_exename())) $script` - - (strm, proc) = open(cmd) - wait(proc) - if !success(proc) && ccall(:jl_running_on_valgrind,Cint,()) == 0 - println(readall(strm)) - error("Topology tests failed : $cmd") - end - - println("Testing exception printing on remote worker from a `remote_do` call") - println("Please ensure the remote error and backtrace is displayed on screen") - - Base.remote_do(id_other) do - throw(ErrorException("TESTING EXCEPTION ON REMOTE DO. PLEASE IGNORE")) - end - sleep(0.5) # Give some time for the above error to be printed - -@unix_only begin - function test_n_remove_pids(new_pids) - for p in new_pids - w_in_remote = sort(remotecall_fetch(workers, p)) - try - @test intersect(new_pids, w_in_remote) == new_pids - catch e - print("p : $p\n") - print("newpids : $new_pids\n") - print("w_in_remote : $w_in_remote\n") - print("intersect : $(intersect(new_pids, w_in_remote))\n\n\n") - rethrow(e) - end - end - - @test :ok == remotecall_fetch(1, new_pids) do p - rmprocs(p; waitfor=5.0) - end - end - - print("\n\nTesting SSHManager. A minimum of 4GB of RAM is recommended.\n") - print("Please ensure sshd is running locally with passwordless login enabled.\n") - - sshflags = `-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR ` - #Issue #9951 - hosts=[] - localhost_aliases = ["localhost", string(getipaddr()), "127.0.0.1"] - num_workers = parse(Int,(get(ENV, "JULIA_ADDPROCS_NUM", "9"))) - - for i in 1:(num_workers/length(localhost_aliases)) - append!(hosts, localhost_aliases) - end - - print("\nTesting SSH addprocs with $(length(hosts)) workers...\n") - new_pids = remotecall_fetch(1, hosts, sshflags) do h, sf - addprocs(h; sshflags=sf) - end - @test length(new_pids) == length(hosts) - test_n_remove_pids(new_pids) - - print("\nMixed ssh addprocs with :auto\n") - new_pids = sort(remotecall_fetch(1, ["localhost", ("127.0.0.1", :auto), "localhost"], sshflags) do h, sf - addprocs(h; sshflags=sf) - end) - @test length(new_pids) == (2 + Sys.CPU_CORES) - test_n_remove_pids(new_pids) - - print("\nMixed ssh addprocs with numeric counts\n") - new_pids = sort(remotecall_fetch(1, [("localhost", 2), ("127.0.0.1", 2), "localhost"], sshflags) do h, sf - addprocs(h; sshflags=sf) - end) - @test length(new_pids) == 5 - test_n_remove_pids(new_pids) - - print("\nssh addprocs with tunnel\n") - new_pids = sort(remotecall_fetch(1, [("localhost", num_workers)], sshflags) do h, sf - addprocs(h; tunnel=true, sshflags=sf) - end) - @test length(new_pids) == num_workers - test_n_remove_pids(new_pids) - -end -end - -# issue #7727 -let A = [], B = [] - t = @task produce(11) - @sync begin - @async for x in t; push!(A,x); end - @async for x in t; push!(B,x); end - end - @test (A == [11]) != (B == [11]) -end - -let t = @task 42 - schedule(t, ErrorException(""), error=true) - @test_throws ErrorException wait(t) -end - -# issue #8207 -let A = Any[] - @parallel (+) for i in (push!(A,1); 1:2) - i - end - @test length(A) == 1 -end - -# issue #13168 -function f13168(n) - val = 0 - for i=1:n val+=sum(rand(n,n)^2) end - val -end -let t = schedule(@task f13168(100)) - @test schedule(t) === t +(strm, proc) = open(cmd) +errors = readall(strm) +wait(proc) +if !success(proc) && ccall(:jl_running_on_valgrind,Cint,()) == 0 + println(errors); + error("Parallel test failed, cmd : $cmd") end diff --git a/test/parallel_exec.jl b/test/parallel_exec.jl new file mode 100644 index 0000000000000..b773428b2e807 --- /dev/null +++ b/test/parallel_exec.jl @@ -0,0 +1,568 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + +using Base.Test + +addprocs(3; exeflags=`--check-bounds=yes --depwarn=error`) + +id_me = myid() +id_other = filter(x -> x != id_me, procs())[rand(1:(nprocs()-1))] + +@test fetch(@spawnat id_other myid()) == id_other +@test @fetchfrom id_other begin myid() end == id_other +@fetch begin myid() end + +rr=RemoteRef() +@test typeof(rr) == RemoteRef{Channel{Any}} +a = rand(5,5) +put!(rr, a) +@test rr[2,3] == a[2,3] +@test rr[] == a + +rr=RemoteRef(workers()[1]) +@test typeof(rr) == RemoteRef{Channel{Any}} +a = rand(5,5) +put!(rr, a) +@test rr[1,5] == a[1,5] +@test rr[] == a + +dims = (20,20,20) + +@linux_only begin + S = SharedArray(Int64, dims) + @test startswith(S.segname, "/jl") + @test !ispath("/dev/shm" * S.segname) + + S = SharedArray(Int64, dims; pids=[id_other]) + @test startswith(S.segname, "/jl") + @test !ispath("/dev/shm" * S.segname) +end + +# TODO : Need a similar test of shmem cleanup for OSX + +##### SharedArray tests + +function check_pids_all(S::SharedArray) + pidtested = falses(size(S)) + for p in procs(S) + idxes_in_p = remotecall_fetch(p, S) do D + parentindexes(D.loc_subarr_1d)[1] + end + @test all(sdata(S)[idxes_in_p] .== p) + pidtested[idxes_in_p] = true + end + @test all(pidtested) +end + +d = Base.shmem_rand(1:100, dims) +a = convert(Array, d) + +partsums = Array(Int, length(procs(d))) +@sync begin + for (i, p) in enumerate(procs(d)) + @async partsums[i] = remotecall_fetch(p, d) do D + sum(D.loc_subarr_1d) + end + end +end +@test sum(a) == sum(partsums) + +d = Base.shmem_rand(dims) +for p in procs(d) + idxes_in_p = remotecall_fetch(p, d) do D + parentindexes(D.loc_subarr_1d)[1] + end + idxf = first(idxes_in_p) + idxl = last(idxes_in_p) + d[idxf] = Float64(idxf) + rv = remotecall_fetch(p, d,idxf,idxl) do D,idxf,idxl + assert(D[idxf] == Float64(idxf)) + D[idxl] = Float64(idxl) + D[idxl] + end + @test d[idxl] == rv +end + +@test ones(10, 10, 10) == Base.shmem_fill(1.0, (10,10,10)) +@test zeros(Int32, 10, 10, 10) == Base.shmem_fill(0, (10,10,10)) + +d = Base.shmem_rand(dims) +s = Base.shmem_rand(dims) +copy!(s, d) +@test s == d +s = Base.shmem_rand(dims) +copy!(s, sdata(d)) +@test s == d +a = rand(dims) +@test sdata(a) == a + +d = SharedArray(Int, dims; init = D->fill!(D.loc_subarr_1d, myid())) +for p in procs(d) + idxes_in_p = remotecall_fetch(p, d) do D + parentindexes(D.loc_subarr_1d)[1] + end + idxf = first(idxes_in_p) + idxl = last(idxes_in_p) + @test d[idxf] == p + @test d[idxl] == p +end + +d = SharedArray(Float64, (2,3)) +@test isa(d[:,2], Vector{Float64}) + +### SharedArrays from a file + +# Mapping an existing file +fn = tempname() +open(fn, "w") do io + write(io, 1:30) +end +sz = (6,5) +Atrue = reshape(1:30, sz) + +S = SharedArray(fn, Int, sz) +@test S == Atrue +@test length(procs(S)) > 1 +@sync begin + for p in procs(S) + @async remotecall_wait(p, S) do D + fill!(D.loc_subarr_1d, myid()) + end + end +end +check_pids_all(S) + +filedata = similar(Atrue) +open(fn, "r") do io + read!(io, filedata) +end +@test filedata == sdata(S) + +# Error for write-only files +@test_throws ArgumentError SharedArray(fn, Int, sz, mode="w") + +# Error for file doesn't exist, but not allowed to create +@test_throws ArgumentError SharedArray(tempname(), Int, sz, mode="r") + +# Creating a new file +fn2 = tempname() +S = SharedArray(fn2, Int, sz, init=D->D[localindexes(D)] = myid()) +@test S == filedata +filedata2 = similar(Atrue) +open(fn2, "r") do io + read!(io, filedata2) +end +@test filedata == filedata2 + +# Appending to a file +fn3 = tempname() +open(fn3, "w") do io + write(io, ones(UInt8, 4)) +end +S = SharedArray(fn3, UInt8, sz, 4, mode="a+", init=D->D[localindexes(D)]=0x02) +len = prod(sz)+4 +@test filesize(fn3) == len +filedata = Array(UInt8, len) +open(fn3, "r") do io + read!(io, filedata) +end +@test all(filedata[1:4] .== 0x01) +@test all(filedata[5:end] .== 0x02) + +@unix_only begin # these give unlink: operation not permitted (EPERM) on Windows + rm(fn); rm(fn2); rm(fn3) +end + +### Utility functions + +# construct PR #13514 +S = SharedArray{Int}((1,2,3)) +@test size(S) == (1,2,3) +@test typeof(S) <: SharedArray{Int} +S = SharedArray{Int}(2) +@test size(S) == (2,) +@test typeof(S) <: SharedArray{Int} +S = SharedArray{Int}(1,2) +@test size(S) == (1,2) +@test typeof(S) <: SharedArray{Int} +S = SharedArray{Int}(1,2,3) +@test size(S) == (1,2,3) +@test typeof(S) <: SharedArray{Int} + +# reshape + +d = Base.shmem_fill(1.0, (10,10,10)) +@test ones(100, 10) == reshape(d,(100,10)) +d = Base.shmem_fill(1.0, (10,10,10)) +@test_throws DimensionMismatch reshape(d,(50,)) + +# rand, randn +d = Base.shmem_rand(dims) +@test size(rand!(d)) == dims +d = Base.shmem_fill(1.0, dims) +@test size(randn!(d)) == dims + +# similar +d = Base.shmem_rand(dims) +@test size(similar(d, Complex128)) == dims +@test size(similar(d, dims)) == dims + +# issue #6362 +d = Base.shmem_rand(dims) +s = copy(sdata(d)) +ds = deepcopy(d) +@test ds == d +pids_d = procs(d) +remotecall_fetch(setindex!, pids_d[findfirst(id->(id != myid()), pids_d)], d, 1.0, 1:10) +@test ds != d +@test s != d + + +# SharedArray as an array +# Since the data in d will depend on the nprocs, just test that these operations work +a = d[1:5] +@test_throws BoundsError d[-1:5] +a = d[1,1,1:3:end] +d[2:4] = 7 +d[5,1:2:4,8] = 19 + +AA = rand(4,2) +A = convert(SharedArray, AA) +B = convert(SharedArray, AA') +@test B*A == ctranspose(AA)*AA + +d=SharedArray(Int64, (10,10); init = D->fill!(D.loc_subarr_1d, myid()), pids=[id_me, id_other]) +d2 = map(x->1, d) +@test reduce(+, d2) == 100 + +@test reduce(+, d) == ((50*id_me) + (50*id_other)) +map!(x->1, d) +@test reduce(+, d) == 100 + +@test fill!(d, 1) == ones(10, 10) +@test fill!(d, 2.) == fill(2, 10, 10) +@test d[:] == fill(2, 100) +@test d[:,1] == fill(2, 10) +@test d[1,:] == fill(2, 1, 10) + +# Boundary cases where length(S) <= length(pids) +@test 2.0 == remotecall_fetch(D->D[2], id_other, Base.shmem_fill(2.0, 2; pids=[id_me, id_other])) +@test 3.0 == remotecall_fetch(D->D[1], id_other, Base.shmem_fill(3.0, 1; pids=[id_me, id_other])) + +# Test @parallel load balancing - all processors should get either M or M+1 +# iterations out of the loop range for some M. +workloads = hist(@parallel((a,b)->[a;b], for i=1:7; myid(); end), nprocs())[2] +@test maximum(workloads) - minimum(workloads) <= 1 + +# @parallel reduction should work even with very short ranges +@test @parallel(+, for i=1:2; i; end) == 3 + +# Testing timedwait on multiple channels +@sync begin + rr1 = Channel() + rr2 = Channel() + rr3 = Channel() + + @async begin sleep(0.5); put!(rr1, :ok) end + @async begin sleep(1.0); put!(rr2, :ok) end + @async begin sleep(2.0); put!(rr3, :ok) end + + tic() + timedwait(1.0) do + all(map(isready, [rr1, rr2, rr3])) + end + et=toq() + # assuming that 0.5 seconds is a good enough buffer on a typical modern CPU + try + @test (et >= 1.0) && (et <= 1.5) + @test !isready(rr3) + catch + warn("timedwait tests delayed. et=$et, isready(rr3)=$(isready(rr3))") + end + @test isready(rr1) +end + +@test_throws ArgumentError sleep(-1) +@test_throws ArgumentError timedwait(()->false, 0.1, pollint=-0.5) + +# specify pids for pmap +@test sort(workers()[1:2]) == sort(unique(pmap(x->(sleep(0.1);myid()), 1:10, pids = workers()[1:2]))) + +# Testing buffered and unbuffered reads +# This large array should write directly to the socket +a = ones(10^6) +@test a == remotecall_fetch((x)->x, id_other, a) + +# Not a bitstype, should be buffered +s = [randstring() for x in 1:10^5] +@test s == remotecall_fetch((x)->x, id_other, s) + +#large number of small requests +num_small_requests = 10000 +@test fill(id_other, num_small_requests) == [remotecall_fetch(myid, id_other) for i in 1:num_small_requests] + +# test parallel sends of large arrays from multiple tasks to the same remote worker +ntasks = 10 +rr_list = [Channel() for x in 1:ntasks] +a=ones(2*10^5); +for rr in rr_list + @async let rr=rr + try + for i in 1:10 + @test a == remotecall_fetch((x)->x, id_other, a) + yield() + end + put!(rr, :OK) + catch + put!(rr, :ERROR) + end + end +end + +@test [fetch(rr) for rr in rr_list] == [:OK for x in 1:ntasks] + +function test_channel(c) + put!(c, 1) + put!(c, "Hello") + put!(c, 5.0) + + @test isready(c) == true + @test fetch(c) == 1 + @test fetch(c) == 1 # Should not have been popped previously + @test take!(c) == 1 + @test take!(c) == "Hello" + @test fetch(c) == 5.0 + @test take!(c) == 5.0 + @test isready(c) == false + close(c) +end + +test_channel(Channel(10)) +test_channel(RemoteRef(()->Channel(10))) + +c=Channel{Int}(1) +@test_throws MethodError put!(c, "Hello") + +c=Channel(256) +# Test growth of channel +@test c.szp1 <= 33 +for x in 1:40 + put!(c, x) +end +@test c.szp1 <= 65 +for x in 1:39 + take!(c) +end +for x in 1:64 + put!(c, x) +end +@test (c.szp1 > 65) && (c.szp1 <= 129) +for x in 1:39 + take!(c) +end +@test fetch(c) == 39 +for x in 1:26 + take!(c) +end +@test isready(c) == false + +# test channel iterations +function test_iteration(in_c, out_c) + t=@schedule for v in in_c + put!(out_c, v) + end + + isa(in_c, Channel) && @test isopen(in_c) == true + put!(in_c, 1) + @test take!(out_c) == 1 + put!(in_c, "Hello") + close(in_c) + @test take!(out_c) == "Hello" + isa(in_c, Channel) && @test isopen(in_c) == false + @test_throws InvalidStateException put!(in_c, :foo) + yield() + @test istaskdone(t) == true +end + +test_iteration(Channel(10), Channel(10)) +# make sure exceptions propagate when waiting on Tasks +@test_throws CompositeException (@sync (@async error("oops"))) +try + @sync begin + for i in 1:5 + @async error(i) + end + end + error("unexpected") +catch ex + @test typeof(ex) == CompositeException + @test length(ex) == 5 + @test typeof(ex.exceptions[1]) == CapturedException + @test typeof(ex.exceptions[1].ex) == ErrorException + errors = map(x->x.ex.msg, ex.exceptions) + @test collect(1:5) == sort(map(x->parse(Int, x), errors)) +end + +try + remotecall_fetch(()->throw(ErrorException("foobar")), id_other) + error("unexpected") +catch ex + @test typeof(ex) == RemoteException + @test typeof(ex.captured) == CapturedException + @test typeof(ex.captured.ex) == ErrorException + @test ex.captured.ex.msg == "foobar" +end + +# The below block of tests are usually run only on local development systems, since: +# - tests which print errors +# - addprocs tests are memory intensive +# - ssh addprocs requires sshd to be running locally with passwordless login enabled. +# The test block is enabled by defining env JULIA_TESTFULL=1 + +DoFullTest = Bool(parse(Int,(get(ENV, "JULIA_TESTFULL", "0")))) + +if DoFullTest + # pmap tests + # needs at least 4 processors dedicated to the below tests + ppids = remotecall_fetch(()->addprocs(4), 1) + s = "abcdefghijklmnopqrstuvwxyz"; + ups = uppercase(s); + @test ups == bytestring(UInt8[UInt8(c) for c in pmap(x->uppercase(x), s)]) + @test ups == bytestring(UInt8[UInt8(c) for c in pmap(x->uppercase(Char(x)), s.data)]) + + # retry, on error exit + res = pmap(x->(x=='a') ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : (sleep(0.1);uppercase(x)), s; err_retry=true, err_stop=true, pids=ppids); + @test (length(res) < length(ups)) + @test isa(res[1], Exception) + + # no retry, on error exit + res = pmap(x->(x=='a') ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : (sleep(0.1);uppercase(x)), s; err_retry=false, err_stop=true, pids=ppids); + @test (length(res) < length(ups)) + @test isa(res[1], Exception) + + # retry, on error continue + res = pmap(x->iseven(myid()) ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : (sleep(0.1);uppercase(x)), s; err_retry=true, err_stop=false, pids=ppids); + @test length(res) == length(ups) + @test ups == bytestring(UInt8[UInt8(c) for c in res]) + + # no retry, on error continue + res = pmap(x->(x=='a') ? error("EXPECTED TEST ERROR. TO BE IGNORED.") : (sleep(0.1);uppercase(x)), s; err_retry=false, err_stop=false, pids=ppids); + @test length(res) == length(ups) + @test isa(res[1], Exception) + + # Topology tests need to run externally since a given cluster at any + # time can only support a single topology and the current session + # is already running in parallel under the default topology. + script = joinpath(dirname(@__FILE__), "topology.jl") + cmd = `$(joinpath(JULIA_HOME,Base.julia_exename())) $script` + + (strm, proc) = open(cmd) + wait(proc) + if !success(proc) && ccall(:jl_running_on_valgrind,Cint,()) == 0 + println(readall(strm)) + error("Topology tests failed : $cmd") + end + + println("Testing exception printing on remote worker from a `remote_do` call") + println("Please ensure the remote error and backtrace is displayed on screen") + + Base.remote_do(id_other) do + throw(ErrorException("TESTING EXCEPTION ON REMOTE DO. PLEASE IGNORE")) + end + sleep(0.5) # Give some time for the above error to be printed + +@unix_only begin + function test_n_remove_pids(new_pids) + for p in new_pids + w_in_remote = sort(remotecall_fetch(workers, p)) + try + @test intersect(new_pids, w_in_remote) == new_pids + catch e + print("p : $p\n") + print("newpids : $new_pids\n") + print("w_in_remote : $w_in_remote\n") + print("intersect : $(intersect(new_pids, w_in_remote))\n\n\n") + rethrow(e) + end + end + + @test :ok == remotecall_fetch(1, new_pids) do p + rmprocs(p; waitfor=5.0) + end + end + + print("\n\nTesting SSHManager. A minimum of 4GB of RAM is recommended.\n") + print("Please ensure sshd is running locally with passwordless login enabled.\n") + + sshflags = `-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR ` + #Issue #9951 + hosts=[] + localhost_aliases = ["localhost", string(getipaddr()), "127.0.0.1"] + num_workers = parse(Int,(get(ENV, "JULIA_ADDPROCS_NUM", "9"))) + + for i in 1:(num_workers/length(localhost_aliases)) + append!(hosts, localhost_aliases) + end + + print("\nTesting SSH addprocs with $(length(hosts)) workers...\n") + new_pids = remotecall_fetch(1, hosts, sshflags) do h, sf + addprocs(h; sshflags=sf) + end + @test length(new_pids) == length(hosts) + test_n_remove_pids(new_pids) + + print("\nMixed ssh addprocs with :auto\n") + new_pids = sort(remotecall_fetch(1, ["localhost", ("127.0.0.1", :auto), "localhost"], sshflags) do h, sf + addprocs(h; sshflags=sf) + end) + @test length(new_pids) == (2 + Sys.CPU_CORES) + test_n_remove_pids(new_pids) + + print("\nMixed ssh addprocs with numeric counts\n") + new_pids = sort(remotecall_fetch(1, [("localhost", 2), ("127.0.0.1", 2), "localhost"], sshflags) do h, sf + addprocs(h; sshflags=sf) + end) + @test length(new_pids) == 5 + test_n_remove_pids(new_pids) + + print("\nssh addprocs with tunnel\n") + new_pids = sort(remotecall_fetch(1, [("localhost", num_workers)], sshflags) do h, sf + addprocs(h; tunnel=true, sshflags=sf) + end) + @test length(new_pids) == num_workers + test_n_remove_pids(new_pids) + +end +end + +# issue #7727 +let A = [], B = [] + t = @task produce(11) + @sync begin + @async for x in t; push!(A,x); end + @async for x in t; push!(B,x); end + end + @test (A == [11]) != (B == [11]) +end + +let t = @task 42 + schedule(t, ErrorException(""), error=true) + @test_throws ErrorException wait(t) +end + +# issue #8207 +let A = Any[] + @parallel (+) for i in (push!(A,1); 1:2) + i + end + @test length(A) == 1 +end + +# issue #13168 +function f13168(n) + val = 0 + for i=1:n val+=sum(rand(n,n)^2) end + val +end +let t = schedule(@task f13168(100)) + @test schedule(t) === t +end diff --git a/test/runtests.jl b/test/runtests.jl index 349faf18c64f9..4a42df0c36590 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -20,7 +20,45 @@ cd(dirname(@__FILE__)) do @everywhere include("testdefs.jl") - reduce(propagate_errors, nothing, pmap(runtests, tests; err_retry=false, err_stop=true)) + results=[] + max_worker_rss = parse(Int, get(ENV, "JULIA_TEST_MAXRSS_MB", "500")) + @sync begin + for p in workers() + @async begin + while length(tests) > 0 + test = shift!(tests) + local resp + try + resp = remotecall_fetch(t -> runtests(t), p, test) + catch e + resp = e + end + push!(results, (test, resp)) + + if (isa(resp, Integer) && (resp > max_worker_rss * 2^20)) || isa(resp, Exception) + if n > 1 + rmprocs(p, waitfor=0.5) + p = addprocs(1; exeflags=`--check-bounds=yes --depwarn=error`)[1] + remotecall_fetch(()->include("testdefs.jl"), p) + else + # single process testing, bail if mem limit reached, or on an exception. + isa(resp, Exception) ? rethrow(resp) : error("Halting tests. memory limit reached : $(resp) > $(max_worker_rss * 2^20)") + end + end + end + end + end + end + + errors = filter(x->isa(x[2], Exception), results) + if length(errors) > 0 + for err in errors + println("Exception running test $(err[1]) :") + showerror(STDERR, err[2]) + println() + end + error("Some tests exited with errors.") + end if compile_test n > 1 && print("\tFrom worker 1:\t") diff --git a/test/testdefs.jl b/test/testdefs.jl index 944b86d16c781..0c9a7cb4f002b 100644 --- a/test/testdefs.jl +++ b/test/testdefs.jl @@ -5,18 +5,9 @@ using Base.Test function runtests(name) @printf(" \033[1m*\033[0m \033[31m%-21s\033[0m", name) tt = @elapsed include("$name.jl") - @printf(" in %6.2f seconds\n", tt) - nothing -end - -function propagate_errors(a,b) - if isa(a,Exception) - rethrow(a) - end - if isa(b,Exception) - rethrow(b) - end - nothing + rss = Sys.maxrss() + @printf(" in %6.2f seconds, maxrss %7.2f MB\n", tt, rss / 2^20) + rss end # looking in . messes things up badly From e958e8ee0cf07965957884ee6d3637156a270512 Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Thu, 15 Oct 2015 12:23:30 +0530 Subject: [PATCH 0562/1938] limit maxrss check only to travis linux 64-bit --- .travis.yml | 1 + test/runtests.jl | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9f073f494552b..4fd4620b1cebe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,7 @@ before_install: sudo apt-get install binutils:i386 -y; sudo apt-get install gcc:i386 g++:i386 make:i386 cpp:i386 g++-4.6:i386 gcc-4.6:i386 libssl-dev:i386 patchelf:i386 gfortran:i386 llvm-3.3-dev:i386 libsuitesparse-dev:i386 libopenblas-dev:i386 libopenblas-base:i386 libblas-dev:i386 liblapack-dev:i386 liblapack3:i386 libarpack2-dev:i386 libarpack2:i386 libfftw3-dev:i386 libgmp-dev:i386 libpcre3-dev:i386 libunwind7-dev:i386 libopenlibm-dev:i386 libmpfr-dev:i386 -y; else + export JULIA_TEST_MAXRSS_MB="500"; sudo apt-get install patchelf gfortran llvm-3.3-dev libsuitesparse-dev libopenblas-dev liblapack-dev libarpack2-dev libfftw3-dev libgmp-dev libpcre3-dev libunwind7-dev libopenlibm-dev libmpfr-dev -y; fi; elif [ `uname` = "Darwin" ]; then diff --git a/test/runtests.jl b/test/runtests.jl index 4a42df0c36590..3a75c875e5a6c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -21,7 +21,12 @@ cd(dirname(@__FILE__)) do @everywhere include("testdefs.jl") results=[] - max_worker_rss = parse(Int, get(ENV, "JULIA_TEST_MAXRSS_MB", "500")) + if haskey(ENV, "JULIA_TEST_MAXRSS_MB") + max_worker_rss = parse(Int, ENV["JULIA_TEST_MAXRSS_MB"]) * 2^20 + else + max_worker_rss = typemax(Csize_t) + end + println(max_worker_rss) @sync begin for p in workers() @async begin @@ -35,14 +40,14 @@ cd(dirname(@__FILE__)) do end push!(results, (test, resp)) - if (isa(resp, Integer) && (resp > max_worker_rss * 2^20)) || isa(resp, Exception) + if (isa(resp, Integer) && (resp > max_worker_rss)) || isa(resp, Exception) if n > 1 rmprocs(p, waitfor=0.5) p = addprocs(1; exeflags=`--check-bounds=yes --depwarn=error`)[1] remotecall_fetch(()->include("testdefs.jl"), p) else - # single process testing, bail if mem limit reached, or on an exception. - isa(resp, Exception) ? rethrow(resp) : error("Halting tests. memory limit reached : $(resp) > $(max_worker_rss * 2^20)") + # single process testing, bail if mem limit reached, or, on an exception. + isa(resp, Exception) ? rethrow(resp) : error("Halting tests. Memory limit reached : $resp > $max_worker_rss") end end end From b30d6231fdd61f9f38782ca4ce5b06d69f3582b7 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Thu, 15 Oct 2015 11:45:32 +0530 Subject: [PATCH 0563/1938] implement digits(T, n, base, pad); allow base == typemax(T) + 1 --- base/docs/helpdb.jl | 4 ++-- base/intfuncs.jl | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index bac08307aab65..c3253589d63ea 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -4256,9 +4256,9 @@ Return a list of immediate subtypes of DataType `T`. Note that all currently loa subtypes doc""" - digits(n, [base], [pad]) + digits([T], n, [base], [pad]) -Returns an array of the digits of `n` in the given base, optionally padded with zeros to a specified size. More significant digits are at higher indexes, such that `n == sum([digits[k]*base^(k-1) for k=1:length(digits)])`. +Returns an array with element type `T` (default `Int`) of the digits of `n` in the given base, optionally padded with zeros to a specified size. More significant digits are at higher indexes, such that `n == sum([digits[k]*base^(k-1) for k=1:length(digits)])`. """ digits diff --git a/base/intfuncs.jl b/base/intfuncs.jl index 1825215f65ab6..ad7ce992d801d 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -308,16 +308,16 @@ bits(x::Union{Char,Int32,UInt32,Float32}) = bin(reinterpret(UInt32,x),32) bits(x::Union{Int64,UInt64,Float64}) = bin(reinterpret(UInt64,x),64) bits(x::Union{Int128,UInt128}) = bin(reinterpret(UInt128,x),128) -function digits{T<:Integer}(n::Integer, base::T=10, pad::Integer=1) +digits{T<:Integer}(n::Integer, base::T=10, pad::Integer=1) = digits(T, n, base, pad) + +function digits{T<:Integer}(::Type{T}, n::Integer, base::Integer=10, pad::Integer=1) 2 <= base || throw(ArgumentError("base must be ≥ 2, got $base")) - m = max(pad,ndigits0z(n,base)) - a = zeros(T,m) - digits!(a, n, base) - return a + digits!(zeros(T, max(pad, ndigits0z(n,base))), n, base) end -function digits!{T<:Integer}(a::AbstractArray{T,1}, n::Integer, base::T=10) +function digits!{T<:Integer}(a::AbstractArray{T,1}, n::Integer, base::Integer=10) 2 <= base || throw(ArgumentError("base must be ≥ 2, got $base")) + base - 1 <= typemax(T) || throw(ArgumentError("type $T too small for base $base")) for i = 1:length(a) a[i] = rem(n, base) n = div(n, base) From 7ff33c7a9d36cd794cb63e069a965f803e4818e9 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Thu, 15 Oct 2015 12:07:38 +0530 Subject: [PATCH 0564/1938] unique: add unique(f, itr) method that uniques by a transformation --- base/docs/helpdb.jl | 4 ++++ base/set.jl | 15 ++++++++++++++- test/sets.jl | 2 ++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index c3253589d63ea..7d8b882d83b3a 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -6394,6 +6394,10 @@ doc""" unique(itr[, dim]) Returns an array containing only the unique elements of the iterable `itr`, in the order that the first of each set of equivalent elements originally appears. If `dim` is specified, returns unique regions of the array `itr` along `dim`. + + unique(f, itr) + +Returns an array containing one value from `itr` for each unique value produced by `f` applied to elements of `itr`. """ unique diff --git a/base/set.jl b/base/set.jl index 857a36d9ea192..9015ac1293e09 100644 --- a/base/set.jl +++ b/base/set.jl @@ -104,7 +104,7 @@ const ⊆ = issubset ⊈(l::Set, r::Set) = !⊆(l, r) function unique(C) - out = Array(eltype(C),0) + out = Vector{eltype(C)}() seen = Set{eltype(C)}() for x in C if !in(x, seen) @@ -115,6 +115,19 @@ function unique(C) out end +function unique(f::Callable, C) + out = Vector{eltype(C)}() + seen = Set() + for x in C + y = f(x) + if !in(y, seen) + push!(seen, y) + push!(out, x) + end + end + out +end + function filter(f, s::Set) u = similar(s) for x in s diff --git a/test/sets.jl b/test/sets.jl index 8570042f7f98a..12beb835f32e3 100644 --- a/test/sets.jl +++ b/test/sets.jl @@ -201,6 +201,8 @@ u = unique([1,1,2]) @test in(1,u) @test in(2,u) @test length(u) == 2 +@test unique(iseven, [5,1,8,9,3,4,10,7,2,6]) == [5,8] +@test unique(n->n % 3, [5,1,8,9,3,4,10,7,2,6]) == [5,1,9] # filter s = Set([1,2,3,4]) From a0802b31c0902d0fc0025be7b983afe9efe7fb04 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 15 Oct 2015 09:45:13 -0400 Subject: [PATCH 0565/1938] Fix compilation on Linux (and Possibly other *nix) --- src/sys.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/sys.c b/src/sys.c index 98fe7ef3bc85a..0e9fe0f133a60 100644 --- a/src/sys.c +++ b/src/sys.c @@ -24,6 +24,13 @@ #include #include +#ifndef _OS_WINDOWS_ +// for getrusage +#include +#include +#include +#endif + #ifdef __APPLE__ #include #include From ceffe2dfbfaaa9ad69499e99b7c1e3f6b1d19e52 Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Thu, 15 Oct 2015 20:25:38 +0530 Subject: [PATCH 0566/1938] Remove debug print Remove debug print left behind --- test/runtests.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 3a75c875e5a6c..969c72ee53182 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -26,7 +26,6 @@ cd(dirname(@__FILE__)) do else max_worker_rss = typemax(Csize_t) end - println(max_worker_rss) @sync begin for p in workers() @async begin From 4f612dab0880eaceb7f334f2c1babd593fa2ea7e Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Thu, 15 Oct 2015 12:07:26 -0400 Subject: [PATCH 0567/1938] Better error msg when calling Pkg.test() on pkg(s) that are not installed closes #13583 --- base/pkg/entry.jl | 31 +++++++++++++++++++++++-------- test/pkg.jl | 17 +++++++++++++++++ 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index b471424c392fe..73fd8f598eefe 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -574,7 +574,10 @@ function updatehook(pkgs::Vector) """) end -function test!(pkg::AbstractString, errs::Vector{AbstractString}, notests::Vector{AbstractString}; coverage::Bool=false) +function test!(pkg::AbstractString, + errs::Vector{AbstractString}, + nopkgs::Vector{AbstractString}, + notests::Vector{AbstractString}; coverage::Bool=false) reqs_path = abspath(pkg,"test","REQUIRE") if isfile(reqs_path) tests_require = Reqs.parse(reqs_path) @@ -584,7 +587,11 @@ function test!(pkg::AbstractString, errs::Vector{AbstractString}, notests::Vecto end end test_path = abspath(pkg,"test","runtests.jl") - if isfile(test_path) + if !isdir(pkg) + push!(nopkgs, pkg) + elseif !isfile(test_path) + push!(notests, pkg) + else info("Testing $pkg") cd(dirname(test_path)) do try @@ -598,22 +605,30 @@ function test!(pkg::AbstractString, errs::Vector{AbstractString}, notests::Vecto push!(errs,pkg) end end - else - push!(notests,pkg) end isfile(reqs_path) && resolve() end function test(pkgs::Vector{AbstractString}; coverage::Bool=false) errs = AbstractString[] + nopkgs = AbstractString[] notests = AbstractString[] for pkg in pkgs - test!(pkg,errs,notests; coverage=coverage) + test!(pkg,errs,nopkgs,notests; coverage=coverage) end - if !isempty(errs) || !isempty(notests) + if !all(isempty, (errs, nopkgs, notests)) messages = AbstractString[] - isempty(errs) || push!(messages, "$(join(errs,", "," and ")) had test errors") - isempty(notests) || push!(messages, "$(join(notests,", "," and ")) did not provide a test/runtests.jl file") + if !isempty(errs) + push!(messages, "$(join(errs,", "," and ")) had test errors") + end + if !isempty(nopkgs) + msg = length(nopkgs) > 1 ? " are not installed packages" : + " is not an installed package" + push!(messages, string(join(nopkgs,", ", " and "), msg)) + end + if !isempty(notests) + push!(messages, "$(join(notests,", "," and ")) did not provide a test/runtests.jl file") + end throw(PkgError(join(messages, "and"))) end end diff --git a/test/pkg.jl b/test/pkg.jl index 2ea76ffed9044..e09b4a0861237 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -120,4 +120,21 @@ temp_pkg_dir() do Pkg.rm("Example") @test isempty(Pkg.installed()) + + # issue #13583 + begin + try + Pkg.test("IDoNotExist") + catch ex + @test isa(ex,Pkg.PkgError) + @test ex.msg == "IDoNotExist is not an installed package" + end + + try + Pkg.test("IDoNotExist1", "IDoNotExist2") + catch ex + @test isa(ex,Pkg.PkgError) + @test ex.msg == "IDoNotExist1 and IDoNotExist2 are not installed packages" + end + end end From 641df2e9666a5d1818fd0b0f2fd988bf5b883e6c Mon Sep 17 00:00:00 2001 From: Michael Klassen Date: Thu, 15 Oct 2015 11:49:47 -0500 Subject: [PATCH 0568/1938] Fix error messages with incorrect ranges --- base/dates/types.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/base/dates/types.jl b/base/dates/types.jl index 36b92c20808d3..5d0da97339b68 100644 --- a/base/dates/types.jl +++ b/base/dates/types.jl @@ -81,10 +81,10 @@ function DateTime(y::Int64,m::Int64=1,d::Int64=1, h::Int64=0,mi::Int64=0,s::Int64=0,ms::Int64=0) 0 < m < 13 || throw(ArgumentError("Month: $m out of range (1:12)")) 0 < d < daysinmonth(y,m)+1 || throw(ArgumentError("Day: $d out of range (1:$(daysinmonth(y,m)))")) - -1 < h < 24 || throw(ArgumentError("Hour: $h out of range (1:23)")) - -1 < mi < 60 || throw(ArgumentError("Minute: $mi out of range (1:59)")) - -1 < s < 60 || throw(ArgumentError("Second: $s out of range (1:59)")) - -1 < ms < 1000 || throw(ArgumentError("Millisecond: $ms out of range (1:999)")) + -1 < h < 24 || throw(ArgumentError("Hour: $h out of range (0:23)")) + -1 < mi < 60 || throw(ArgumentError("Minute: $mi out of range (0:59)")) + -1 < s < 60 || throw(ArgumentError("Second: $s out of range (0:59)")) + -1 < ms < 1000 || throw(ArgumentError("Millisecond: $ms out of range (0:999)")) rata = ms + 1000*(s + 60mi + 3600h + 86400*totaldays(y,m,d)) return DateTime(UTM(rata)) end From 990a14cf46d8b35620180c86f91246ba4953aa04 Mon Sep 17 00:00:00 2001 From: Eric Davies Date: Thu, 15 Oct 2015 13:14:51 -0500 Subject: [PATCH 0569/1938] New convert method for non-Nullables to Nullables --- base/nullable.jl | 2 ++ test/nullable.jl | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/base/nullable.jl b/base/nullable.jl index 7f059613e407e..11f472f627080 100644 --- a/base/nullable.jl +++ b/base/nullable.jl @@ -10,6 +10,8 @@ eltype{T}(::Type{Nullable{T}}) = T convert{T}(::Type{Nullable{T}}, x::Nullable{T}) = x +convert{T}(t::Type{Nullable{T}}, x::Any) = convert(t, convert(T, x)) + function convert{T}(::Type{Nullable{T}}, x::Nullable) return isnull(x) ? Nullable{T}() : Nullable{T}(convert(T, get(x))) end diff --git a/test/nullable.jl b/test/nullable.jl index 974b9bd15b5d8..958dc8a50825b 100644 --- a/test/nullable.jl +++ b/test/nullable.jl @@ -266,7 +266,16 @@ end # issue #9462 for T in types @test isa(convert(Nullable{Number}, Nullable(one(T))), Nullable{Number}) + @test isa(convert(Nullable{Number}, one(T)), Nullable{Number}) + @test isa(convert(Nullable{T}, one(T)), Nullable{T}) @test isa(convert(Nullable{Any}, Nullable(one(T))), Nullable{Any}) + @test isa(convert(Nullable{Any}, one(T)), Nullable{Any}) + + # one(T) is convertible to every type in types + # let's test that with Nullables + for S in types + @test isa(convert(Nullable{T}, one(S)), Nullable{T}) + end end @test isnull(convert(Nullable, nothing)) From e44938c6ffea5cc99cd8265fcf9cad790821c997 Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Thu, 15 Oct 2015 21:33:44 +0530 Subject: [PATCH 0570/1938] fix leak in RemoteRef --- base/multi.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/multi.jl b/base/multi.jl index bce8d7b26e79f..23c2e7ecae67c 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -599,6 +599,7 @@ function send_add_client(rr::RemoteRef, i) end end +channel_type{T}(rr::RemoteRef{T}) = T function serialize(s::SerializationState, rr::RemoteRef) i = worker_id_from_socket(s.io) #println("$(myid()) serializing $rr to $i") @@ -609,14 +610,14 @@ function serialize(s::SerializationState, rr::RemoteRef) invoke(serialize, Tuple{SerializationState, Any}, s, rr) end -function deserialize(s::SerializationState, t::Type{RemoteRef}) +function deserialize{T<:RemoteRef}(s::SerializationState, t::Type{T}) rr = invoke(deserialize, Tuple{SerializationState, DataType}, s, t) where = rr.where if where == myid() add_client(rr2id(rr), myid()) end # call ctor to make sure this rr gets added to the client_refs table - RemoteRef(where, rr.whence, rr.id) + RemoteRef{channel_type(rr)}(where, rr.whence, rr.id) end # data stored by the owner of a RemoteRef From 2bf78187ad150b9eb6c6c6fd3fbd10fad3368e74 Mon Sep 17 00:00:00 2001 From: Spencer Lyon Date: Thu, 15 Oct 2015 19:43:47 -0400 Subject: [PATCH 0571/1938] Fix bug in findmin! and findmax! Also added tests/docs for those functions and exported them --- base/exports.jl | 2 ++ base/reducedim.jl | 19 +++++++++++++++---- doc/stdlib/collections.rst | 12 ++++++++++++ test/reducedim.jl | 20 ++++++++++++++------ 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index e32a5ae247046..af5c8d856e038 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -521,6 +521,8 @@ export findin, findmax, findmin, + findmin!, + findmax!, findn, findnext, findprev, diff --git a/base/reducedim.jl b/base/reducedim.jl index 77d1d0af5e1cb..55fb8c04794a1 100644 --- a/base/reducedim.jl +++ b/base/reducedim.jl @@ -325,12 +325,18 @@ function findminmax!{T,N}(f, Rval, Rind, A::AbstractArray{T,N}) Rval, Rind end -# findmin + +""" + findmin!(rval, rind, A, [init=true]) -> (minval, index) + +Find the minimum of `A` and the corresponding linear index along singleton +dimensions of `rval` and `rind`, and store the results in `rval` and `rind`. +""" function findmin!{R}(rval::AbstractArray{R}, rind::AbstractArray, A::AbstractArray; init::Bool=true) - findminmax!(LessFun(), initarray!(rval, typemax(R), init), rind, A) + findminmax!(LessFun(), initarray!(rval, MinFun(), init), rind, A) end function findmin{T}(A::AbstractArray{T}, region) @@ -342,12 +348,17 @@ function findmin{T}(A::AbstractArray{T}, region) zeros(Int, reduced_dims0(A, region)), A) end -# findmax +""" + findmax!(rval, rind, A, [init=true]) -> (maxval, index) + +Find the maximum of `A` and the corresponding linear index along singleton +dimensions of `rval` and `rind`, and store the results in `rval` and `rind`. +""" function findmax!{R}(rval::AbstractArray{R}, rind::AbstractArray, A::AbstractArray; init::Bool=true) - findminmax!(MoreFun(), initarray!(rval, typemin(R), init), rind, A) + findminmax!(MoreFun(), initarray!(rval, MaxFun(), init), rind, A) end function findmax{T}(A::AbstractArray{T}, region) diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 906374baa2c96..253657fe81644 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -347,6 +347,18 @@ Iterable Collections For an array input, returns the value and index of the minimum over the given dimensions. +.. function:: findmax!(rval, rind, A, [init=true]) -> (maxval, index) + + .. Docstring generated from Julia source + + Find the maximum of ``A`` and the corresponding linear index along singleton dimensions of ``rval`` and ``rind``\ , and store the results in ``rval`` and ``rind``\ . + +.. function:: findmin!(rval, rind, A, [init=true]) -> (minval, index) + + .. Docstring generated from Julia source + + Find the minimum of ``A`` and the corresponding linear index along singleton dimensions of ``rval`` and ``rind``\ , and store the results in ``rval`` and ``rind``\ . + .. function:: maxabs(itr) .. Docstring generated from Julia source diff --git a/test/reducedim.jl b/test/reducedim.jl index da2d5192dccd4..1c74fa5b3959c 100644 --- a/test/reducedim.jl +++ b/test/reducedim.jl @@ -113,15 +113,23 @@ R = reducedim((a,b) -> a+b, [1 2; 3 4], 2, 0.0) rt = Base.return_types(reducedim, Tuple{Function, Array{Float64, 3}, Int, Float64}) @test length(rt) == 1 && rt[1] == Array{Float64, 3} + ## findmin/findmax A = [1.0 3.0 6.0; 5.0 2.0 4.0] -@test findmin(A, (1,)) == ([1.0 2.0 4.0], [1 4 6]) -@test findmin(A, (2,)) == (reshape([1.0,2.0], 2, 1), reshape([1,4], 2, 1)) -@test findmin(A, (1,2)) == (fill(1.0,1,1),fill(1,1,1)) -@test findmax(A, (1,)) == ([5.0 3.0 6.0], [2 3 5]) -@test findmax(A, (2,)) == (reshape([6.0,5.0], 2, 1), reshape([5,2], 2, 1)) -@test findmax(A, (1,2)) == (fill(6.0,1,1),fill(5,1,1)) +for (tup, rval, rind) in [((1,), [1.0 2.0 4.0], [1 4 6]), + ((2,), reshape([1.0,2.0], 2, 1), reshape([1,4], 2, 1)), + ((1,2), fill(1.0,1,1),fill(1,1,1))] + @test findmin(A, tup) == (rval, rind) + @test findmin!(similar(rval), similar(rind), A) == (rval, rind) +end + +for (tup, rval, rind) in [((1,), [5.0 3.0 6.0], [2 3 5]), + ((2,), reshape([6.0,5.0], 2, 1), reshape([5,2], 2, 1)), + ((1,2), fill(6.0,1,1),fill(5,1,1))] + @test findmax(A, tup) == (rval, rind) + @test findmax!(similar(rval), similar(rind), A) == (rval, rind) +end # issue #6672 @test sum(Real[1 2 3; 4 5.3 7.1], 2) == reshape([6, 16.4], 2, 1) From fe7bb298e61fa06414840ec28ec7fa0e2186f892 Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Thu, 15 Oct 2015 22:42:30 -0400 Subject: [PATCH 0572/1938] Fix generic axpy! for vector element types without commutative multiplication, e.g. quaternions. --- base/linalg/generic.jl | 10 +++++----- test/linalg/generic.jl | 9 +++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/base/linalg/generic.jl b/base/linalg/generic.jl index 9d15ce0f5bc92..4642bc51363f0 100644 --- a/base/linalg/generic.jl +++ b/base/linalg/generic.jl @@ -439,20 +439,20 @@ function peakflops(n::Integer=2000; parallel::Bool=false) parallel ? sum(pmap(peakflops, [ n for i in 1:nworkers()])) : (2*Float64(n)^3/t) end -# BLAS-like in-place y=alpha*x+y function (see also the version in blas.jl +# BLAS-like in-place y = x*α+y function (see also the version in blas.jl # for BlasFloat Arrays) -function axpy!(alpha, x::AbstractArray, y::AbstractArray) +function axpy!(α, x::AbstractArray, y::AbstractArray) n = length(x) if n != length(y) throw(DimensionMismatch("x has length $n, but y has length $(length(y))")) end for i = 1:n - @inbounds y[i] += alpha * x[i] + @inbounds y[i] += x[i]*α end y end -function axpy!{Ti<:Integer,Tj<:Integer}(alpha, x::AbstractArray, rx::AbstractArray{Ti}, y::AbstractArray, ry::AbstractArray{Tj}) +function axpy!{Ti<:Integer,Tj<:Integer}(α, x::AbstractArray, rx::AbstractArray{Ti}, y::AbstractArray, ry::AbstractArray{Tj}) if length(x) != length(y) throw(DimensionMismatch("x has length $(length(x)), but y has length $(length(y))")) elseif minimum(rx) < 1 || maximum(rx) > length(x) @@ -463,7 +463,7 @@ function axpy!{Ti<:Integer,Tj<:Integer}(alpha, x::AbstractArray, rx::AbstractArr throw(ArgumentError("rx has length $(length(rx)), but ry has length $(length(ry))")) end for i = 1:length(rx) - @inbounds y[ry[i]] += alpha * x[rx[i]] + @inbounds y[ry[i]] += x[rx[i]]*α end y end diff --git a/test/linalg/generic.jl b/test/linalg/generic.jl index a1ce1a5061419..7df2a235643f1 100644 --- a/test/linalg/generic.jl +++ b/test/linalg/generic.jl @@ -141,3 +141,12 @@ let x = Vector{Int}[[1,2], [3,4]] @test norm(x, 1) ≈ sqrt(5) + 5 @test norm(x, 3) ≈ cbrt(sqrt(125)+125) end + +# test that LinAlg.axpy! works for element type without commutative multiplication +let + α = ones(Int, 2, 2) + x = fill([1 0; 1 1], 3) + y = fill(zeros(Int, 2, 2), 3) + @test LinAlg.axpy!(α, x, deepcopy(y)) == x .* Matrix{Int}[α] + @test LinAlg.axpy!(α, x, deepcopy(y)) != Matrix{Int}[α] .* x +end From 64525aaa8fbeaa668cae5a5e5fae16c5bfe1dbee Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Wed, 14 Oct 2015 16:45:01 -0400 Subject: [PATCH 0573/1938] Detangle linear indexing from non-scalar indexing This removes the LinearFast special cases from non-scalar indexing. Previously, we were manually hoisting the div/rem sub2ind calculation along the indexed strides, but LLVM seems to be just as capable at performing this optimization in the cases I have tested. Even better, though, this creates a clean separation between the array indexing fallbacks: * Scalar fallbacks use `ind2sub` and `sub2ind` to compute the required number of indices that the custom type must implement. * Non-scalar fallbacks simply "unwrap" the elements from AbstractArrays and use scalar indexing with the indices that were provided. * (CartesianIndices are also expanded to individual integers, but that is a smaller detail.) In all cases that I've tried, I've been unable to measure a performance difference. Indeed, the LLVM IR looks identical in my spot-checks, too. --- base/multidimensional.jl | 52 +++++++--------------------------------- 1 file changed, 8 insertions(+), 44 deletions(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 56f882338ec36..185f8d7c66bd2 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -187,14 +187,14 @@ index_shape_dim(A, dim, ::Colon) = (trailingsize(A, dim),) checkbounds(A, I...) _unsafe_getindex(l, A, I...) end -@generated function _unsafe_getindex(l::LinearIndexing, A::AbstractArray, I::Union{Real, AbstractArray, Colon}...) +@generated function _unsafe_getindex(::LinearIndexing, A::AbstractArray, I::Union{Real, AbstractArray, Colon}...) N = length(I) quote # This is specifically *not* inlined. @nexprs $N d->(I_d = to_index(I[d])) dest = similar(A, @ncall $N index_shape A I) @ncall $N checksize dest I - @ncall $N _unsafe_getindex! dest l A I + @ncall $N _unsafe_getindex! dest A I end end @@ -221,7 +221,7 @@ end # Indexing with an array of indices is inherently linear in the source, but # might be able to be optimized with fast dividing integers -@inline function _unsafe_getindex!(dest::AbstractArray, ::LinearIndexing, src::AbstractArray, I::AbstractArray) +@inline function _unsafe_getindex!(dest::AbstractArray, src::AbstractArray, I::AbstractArray) D = eachindex(dest) Ds = start(D) for idx in I @@ -231,26 +231,8 @@ end dest end -# Fast source - compute the linear index -@generated function _unsafe_getindex!(dest::AbstractArray, ::LinearFast, src::AbstractArray, I::Union{Real, AbstractVector, Colon}...) - N = length(I) - quote - $(Expr(:meta, :inline)) - stride_1 = 1 - @nexprs $N d->(stride_{d+1} = stride_d*size(src, d)) - $(symbol(:offset_, N)) = 1 - D = eachindex(dest) - Ds = start(D) - @nloops $N i dest d->(offset_{d-1} = offset_d + (unsafe_getindex(I[d], i_d)-1)*stride_d) begin - d, Ds = next(D, Ds) - unsafe_setindex!(dest, unsafe_getindex(src, offset_0), d) - end - dest - end -end -# Slow source - index with the indices provided. -# TODO: this may not be the full dimensionality; that case could be optimized -@generated function _unsafe_getindex!(dest::AbstractArray, ::LinearSlow, src::AbstractArray, I::Union{Real, AbstractVector, Colon}...) +# Always index with the exactly indices provided. +@generated function _unsafe_getindex!(dest::AbstractArray, src::AbstractArray, I::Union{Real, AbstractVector, Colon}...) N = length(I) quote $(Expr(:meta, :inline)) @@ -292,8 +274,8 @@ _iterable(v) = repeated(v) checkbounds(A, J...) _unsafe_setindex!(l, A, x, J...) end -@inline function _unsafe_setindex!(l::LinearIndexing, A::AbstractArray, x, J::Union{Real,AbstractArray,Colon}...) - _unsafe_batchsetindex!(l, A, _iterable(x), to_indexes(J...)...) +@inline function _unsafe_setindex!(::LinearIndexing, A::AbstractArray, x, J::Union{Real,AbstractArray,Colon}...) + _unsafe_batchsetindex!(A, _iterable(x), to_indexes(J...)...) end # 1-d logical indexing: override the above to avoid calling find (in to_index) @@ -315,25 +297,7 @@ function _unsafe_setindex!(::LinearIndexing, A::AbstractArray, x, I::AbstractArr A end -# Use iteration over X so we don't need to worry about its storage -@generated function _unsafe_batchsetindex!(::LinearFast, A::AbstractArray, X, I::Union{Real,AbstractArray,Colon}...) - N = length(I) - quote - @nexprs $N d->(I_d = I[d]) - idxlens = @ncall $N index_lengths A I - @ncall $N setindex_shape_check X (d->idxlens[d]) - Xs = start(X) - stride_1 = 1 - @nexprs $N d->(stride_{d+1} = stride_d*size(A,d)) - $(symbol(:offset_, N)) = 1 - @nloops $N i d->(1:idxlens[d]) d->(offset_{d-1} = offset_d + (unsafe_getindex(I_d, i_d)-1)*stride_d) begin - v, Xs = next(X, Xs) - unsafe_setindex!(A, v, offset_0) - end - A - end -end -@generated function _unsafe_batchsetindex!(::LinearSlow, A::AbstractArray, X, I::Union{Real,AbstractArray,Colon}...) +@generated function _unsafe_batchsetindex!(A::AbstractArray, X, I::Union{Real,AbstractArray,Colon}...) N = length(I) quote @nexprs $N d->(I_d = I[d]) From b2fc048652b24141896ddb8cf030d77ed6db33a2 Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Thu, 15 Oct 2015 15:41:59 -0400 Subject: [PATCH 0574/1938] fix some issues with Pkg.pin, add some tests closes #13520 --- base/libgit2.jl | 21 +++++++++++------- base/libgit2/blob.jl | 4 ---- base/libgit2/merge.jl | 44 ++++++++++++++++++-------------------- base/libgit2/reference.jl | 10 ++++++--- base/libgit2/repository.jl | 8 ++++--- base/pkg/entry.jl | 23 ++++++++++++++------ test/pkg.jl | 17 ++++++++++++--- 7 files changed, 77 insertions(+), 50 deletions(-) diff --git a/base/libgit2.jl b/base/libgit2.jl index c64daba7cb06a..0e7281c540e0b 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -213,10 +213,9 @@ function branch!(repo::GitRepo, branch_name::AbstractString, Oid(commit) end iszero(commit_id) && return - cmt = get(GitCommit, repo, commit_id) try - branch_ref = create_branch(repo, cmt, branch_name, force=force) + branch_ref = create_branch(repo, branch_name, cmt, force=force) finally finalize(cmt) end @@ -372,7 +371,10 @@ function merge!(repo::GitRepo; if committish == Consts.FETCH_HEAD # merge FETCH_HEAD fheads = fetchheads(repo) filter!(fh->fh.ismerge, fheads) - length(fheads) == 0 && throw(Error.GitError(Error.Merge, Error.ERROR, "There is no fetch reference for this branch.")) + if length(fheads) == 0 + throw(Error.GitError(Error.Merge,Error.ERROR, + "There is no fetch reference for this branch.")) + end map(fh->GitAnnotated(repo,fh), fheads) else # merge commitish [GitAnnotated(repo, committish)] @@ -383,7 +385,10 @@ function merge!(repo::GitRepo; [GitAnnotated(repo, brn_ref)] end else # try to get tracking remote branch for the head - !isattached(repo) && throw(Error.GitError(Error.Merge, Error.ERROR, "There is no tracking information for the current branch.")) + if !isattached(repo) + throw(Error.GitError(Error.Merge, Error.ERROR, + "There is no tracking information for the current branch.")) + end with(upstream(head_ref)) do tr_brn_ref [GitAnnotated(repo, tr_brn_ref)] end @@ -391,11 +396,11 @@ function merge!(repo::GitRepo; end try - merge!(repo, upst_anns, fastforward, merge_opts, checkout_opts=checkout_opts) + merge!(repo, upst_anns, fastforward, + merge_opts=merge_opts, + checkout_opts=checkout_opts) finally - for ann in upst_anns - finalize(ann) - end + map(finalize, upst_anns) end end end diff --git a/base/libgit2/blob.jl b/base/libgit2/blob.jl index 8f8a25bf87c78..06f39f0cca433 100644 --- a/base/libgit2/blob.jl +++ b/base/libgit2/blob.jl @@ -4,10 +4,6 @@ function content(blob::GitBlob) return ccall((:git_blob_rawcontent, :libgit2), Ptr{Void}, (Ptr{Void},), blob.ptr) end -function isbinary(blob::GitBlob) - return ccall((:git_blob_is_binary, :libgit2), CInt, (Ptr{Void},), blob.ptr) == 1 -end - function Base.length(blob::GitBlob) return ccall((:git_blob_rawsize, :libgit2), Coff_t, (Ptr{Void},), blob.ptr) end diff --git a/base/libgit2/merge.jl b/base/libgit2/merge.jl index d779bbe41c411..db64522ba55e4 100644 --- a/base/libgit2/merge.jl +++ b/base/libgit2/merge.jl @@ -74,13 +74,13 @@ function ffmerge!(repo::GitRepo, ann::GitAnnotated) end """ Merge changes into current head """ -function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, - merge_opts::MergeOptions = MergeOptions(); +function merge!(repo::GitRepo, anns::Vector{GitAnnotated}; + merge_opts::MergeOptions = MergeOptions(), checkout_opts::CheckoutOptions = CheckoutOptions()) anns_size = Csize_t(length(anns)) @check ccall((:git_merge, :libgit2), Cint, (Ptr{Void}, Ptr{Ptr{Void}}, Csize_t, - Ptr{MergeOptionsStruct}, Ptr{CheckoutOptions}), + Ptr{MergeOptions}, Ptr{CheckoutOptions}), repo.ptr, anns, anns_size, Ref(merge_opts), Ref(checkout_opts)) info("Review and commit merged changes.") @@ -90,15 +90,15 @@ end """Internal implementation of merge. Returns `true` if merge was successful, otherwise `false` """ -function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, fastforward::Bool, - merge_opts::MergeOptions = MergeOptions(); +function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, fastforward::Bool; + merge_opts::MergeOptions = MergeOptions(), checkout_opts::CheckoutOptions = CheckoutOptions()) ma, mp = merge_analysis(repo, anns) if isset(ma, Cint(Consts.MERGE_ANALYSIS_UP_TO_DATE)) return true # no merge - everything is up to date end - ffPref = if fastforward + ffpref = if fastforward Consts.MERGE_PREFERENCE_FASTFORWARD_ONLY elseif isset(mp, Cint(Consts.MERGE_PREFERENCE_NONE)) Consts.MERGE_PREFERENCE_NONE @@ -106,13 +106,11 @@ function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, fastforward::Bool, Consts.MERGE_PREFERENCE_NO_FASTFORWARD elseif isset(mp, Cint(Consts.MERGE_PREFERENCE_FASTFORWARD_ONLY)) Consts.MERGE_PREFERENCE_FASTFORWARD_ONLY - end - if ffPref === nothing - warn("Unknown merge preference: $(mp).") - return false + else + throw(ArgumentError("unknown merge preference: $(mp).")) end - mergeResult = if ffPref == Consts.MERGE_PREFERENCE_NONE + merge_result = if ffpref == Consts.MERGE_PREFERENCE_NONE if isset(ma, Cint(Consts.MERGE_ANALYSIS_FASTFORWARD)) if length(anns) > 1 warn("Unable to perform Fast-Forward merge with mith multiple merge heads.") @@ -121,9 +119,11 @@ function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, fastforward::Bool, ffmerge!(repo, anns[1]) end elseif isset(ma, Cint(Consts.MERGE_ANALYSIS_NORMAL)) - merge!(repo, anns, merge_opts, checkout_opts=checkout_opts) + merge!(repo, anns, + merge_opts=merge_opts, + checkout_opts=checkout_opts) end - elseif ffPref == Consts.MERGE_PREFERENCE_FASTFORWARD_ONLY + elseif ffpref == Consts.MERGE_PREFERENCE_FASTFORWARD_ONLY if isset(ma, Cint(Consts.MERGE_ANALYSIS_FASTFORWARD)) if length(anns) > 1 warn("Unable to perform Fast-Forward merge with mith multiple merge heads.") @@ -135,18 +135,16 @@ function merge!(repo::GitRepo, anns::Vector{GitAnnotated}, fastforward::Bool, warn("Cannot perform fast-forward merge.") false end - elseif ffPref == Consts.MERGE_PREFERENCE_NO_FASTFORWARD + elseif ffpref == Consts.MERGE_PREFERENCE_NO_FASTFORWARD if isset(ma, Cint(Consts.MERGE_ANALYSIS_NORMAL)) - merge!(repo, anns, merge_opts, checkout_opts=checkout_opts) + merge!(repo, anns, + merge_opts=merge_opts, + checkout_opts=checkout_opts) end + else + throw(ArgumentError("unknown merge analysis result: $(ma)")) end - - if mergeResult === nothing - warn("Unknown merge analysis result: $(ma). Merging is not possible.") - return false - end - - return mergeResult + return merge_result end function merge_base(repo::GitRepo, one::AbstractString, two::AbstractString) @@ -163,4 +161,4 @@ function merge_base(repo::GitRepo, one::AbstractString, two::AbstractString) Oid() end return moid -end \ No newline at end of file +end diff --git a/base/libgit2/reference.jl b/base/libgit2/reference.jl index 0298a334c0548..e1a3b9916cc4b 100644 --- a/base/libgit2/reference.jl +++ b/base/libgit2/reference.jl @@ -9,7 +9,7 @@ function GitReference(repo::GitRepo, refname::AbstractString) end function GitReference(repo::GitRepo, obj_oid::Oid, refname::AbstractString = Consts.HEAD_FILE; - force::Bool=false, msg::AbstractString="") + force::Bool=false, msg::AbstractString="") ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_reference_create, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{UInt8}, Ptr{Oid}, Cint, Cstring), @@ -89,7 +89,9 @@ function peel{T <: GitObject}(::Type{T}, ref::GitReference) return T(obj_ptr_ptr[]) end -function create_branch(repo::GitRepo, commit_obj::GitCommit, bname::AbstractString; +function create_branch(repo::GitRepo, + bname::AbstractString, + commit_obj::GitCommit; force::Bool=false) ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_branch_create, :libgit2), Cint, @@ -109,7 +111,9 @@ function head!(repo::GitRepo, ref::GitReference) return ref end -function lookup_branch(repo::GitRepo, branch_name::AbstractString, remote::Bool=false) +function lookup_branch(repo::GitRepo, + branch_name::AbstractString, + remote::Bool=false) ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) branch_type = remote ? Consts.BRANCH_REMOTE : Consts.BRANCH_LOCAL err = ccall((:git_branch_lookup, :libgit2), Cint, diff --git a/base/libgit2/repository.jl b/base/libgit2/repository.jl index 51929b1e423ac..0ce63bb2f921f 100644 --- a/base/libgit2/repository.jl +++ b/base/libgit2/repository.jl @@ -43,9 +43,11 @@ end function head_oid(repo::GitRepo) head_ref = head(repo) - oid = Oid(head_ref) - finalize(head_ref) - return oid + try + return Oid(head_ref) + finally + finalize(head_ref) + end end function headname(repo::GitRepo) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 73fd8f598eefe..67076308885d9 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -293,16 +293,27 @@ end function pin(pkg::AbstractString, head::AbstractString) ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) - rslv = !isempty(head) # no need to resolve, branch will be from HEAD + should_resolve = true with(GitRepo, pkg) do repo - if isempty(head) # get HEAD oid - head = head_oid(repo) + id = if isempty(head) # get HEAD commit + LibGit2.head_oid(repo) + else + # no need to resolve, branch will be from HEAD + should_resolve = false + LibGit2.revparseid(repo, head) end - branch = "pinned.$(head[1:8]).tmp" + commit = LibGit2.get(LibGit2.GitCommit, repo, id) + branch = "pinned.$(string(id)[1:8]).tmp" info("Creating $pkg branch $branch") - LibGit2.create_branch(repo, branch, head) + try + ref = LibGit2.create_branch(repo, branch, commit) + finalize(ref) + finally + finalize(commit) + end end - rslv ? resolve() : nothing + should_resolve && resolve() + nothing end pin(pkg::AbstractString) = pin(pkg, "") diff --git a/test/pkg.jl b/test/pkg.jl index e09b4a0861237..3c725ae451f96 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -118,9 +118,6 @@ temp_pkg_dir() do end @test Pkg.installed()["Example"] > v"0.0.0" - Pkg.rm("Example") - @test isempty(Pkg.installed()) - # issue #13583 begin try @@ -137,4 +134,18 @@ temp_pkg_dir() do @test ex.msg == "IDoNotExist1 and IDoNotExist2 are not installed packages" end end + + begin + Pkg.pin("Example") + Pkg.free("Example") + + Pkg.pin("Example", v"0.4.0") + Pkg.update() + Pkg.installed()["Example"] == v"0.4.0" + end + + begin + Pkg.rm("Example") + @test isempty(Pkg.installed()) + end end From 481bafe42dd7983167ef38374b04d1918e4c6d70 Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Fri, 16 Oct 2015 15:42:08 -0400 Subject: [PATCH 0575/1938] remove Char type parameter to rand(::AbstractRNG, ::UnitRange{T}) closes #13600 --- base/random.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/random.jl b/base/random.jl index b100aa6101138..02e850d5ebf7d 100644 --- a/base/random.jl +++ b/base/random.jl @@ -566,7 +566,7 @@ else end end -rand{T<:Union{Signed,Unsigned,BigInt,Bool,Char}}(rng::AbstractRNG, r::UnitRange{T}) = rand(rng, RangeGenerator(r)) +rand{T<:Union{Signed,Unsigned,BigInt,Bool}}(rng::AbstractRNG, r::UnitRange{T}) = rand(rng, RangeGenerator(r)) # Randomly draw a sample from an AbstractArray r From 13a851c5aa3d530c61ea3b408250a1444d4a15de Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Fri, 16 Oct 2015 16:40:25 -0400 Subject: [PATCH 0576/1938] get rid of rand(Oid) which is useless --- base/libgit2/oid.jl | 2 -- test/libgit2.jl | 11 +++++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/base/libgit2/oid.jl b/base/libgit2/oid.jl index 1af95d2edb296..c7ab4c668b22f 100644 --- a/base/libgit2/oid.jl +++ b/base/libgit2/oid.jl @@ -88,6 +88,4 @@ end Base.zero(::Type{Oid}) = Oid() -Base.rand(::Type{Oid}) = Oid(ntuple(x->rand(UInt8), OID_RAWSZ)) - Base.isequal(id1::Oid, id2::Oid) = id1.val == id2.val diff --git a/test/libgit2.jl b/test/libgit2.jl index 7e96689df85b9..161db390b18e3 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -32,14 +32,13 @@ end @testset "OID" begin z = LibGit2.Oid() - r = rand(LibGit2.Oid) @test LibGit2.iszero(z) @test z == zero(LibGit2.Oid) - rs = string(r) - rr = LibGit2.raw(r) - @test r == LibGit2.Oid(rr) - @test r == LibGit2.Oid(rs) - @test r == LibGit2.Oid(pointer(rr)) + rs = string(z) + rr = LibGit2.raw(z) + @test z == LibGit2.Oid(rr) + @test z == LibGit2.Oid(rs) + @test z == LibGit2.Oid(pointer(rr)) for i in 11:length(rr); rr[i] = 0; end @test LibGit2.Oid(rr) == LibGit2.Oid(rs[1:20]) end From b09a4bd364f9323eaa07b4a35bb21ab139adbbe4 Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Fri, 16 Oct 2015 17:21:17 -0400 Subject: [PATCH 0577/1938] Fix #13647, comparing boxed isbits immutables --- src/codegen.cpp | 4 ++-- test/core.jl | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 2894f650058df..02cbf786433ef 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2071,10 +2071,10 @@ static Value *emit_bits_compare(const jl_cgval_t &arg1, const jl_cgval_t &arg2, Type *atp = at->getPointerTo(); Value *varg1 = arg1.V; if (varg1->getType() != atp) - builder.CreatePointerCast(varg1, atp); + varg1 = builder.CreatePointerCast(varg1, atp); Value *varg2 = arg2.V; if (varg2->getType() != atp) - builder.CreatePointerCast(varg2, atp); + varg2 = builder.CreatePointerCast(varg2, atp); jl_svec_t *types = ((jl_datatype_t*)arg1.typ)->types; Value *answer = ConstantInt::get(T_int1, 1); size_t l = jl_svec_len(types); diff --git a/test/core.jl b/test/core.jl index dea37dfa7d483..666e87321d482 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3436,3 +3436,17 @@ end type IO13433 <: IO end Base.read(::IO13433, ::Type{UInt8}) = 0x01 @test read!(IO13433(), Array(UInt8, 4)) == [0x01, 0x01, 0x01, 0x01] + +# issue #13647, comparing boxed isbits immutables +immutable X13647 + a::Int + b::Bool +end +function f13647(x, y) + z = false + z = y + x === z +end +@test f13647(X13647(1, false), X13647(1, false)) +@test !f13647(X13647(1, false), X13647(1, true)) +@test !f13647(X13647(2, false), X13647(1, false)) From 848d30526ade6fe00d5bb795587f28f8b2b5e407 Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Fri, 16 Oct 2015 17:25:58 -0400 Subject: [PATCH 0578/1938] suggest to install a missing package in loading, closes #13482 --- base/loading.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/base/loading.jl b/base/loading.jl index d6db483572119..f2a90cb224acb 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -308,7 +308,9 @@ function require(mod::Symbol) name = string(mod) path = find_in_node_path(name, nothing, 1) - path === nothing && throw(ArgumentError("$name not found in path")) + if path === nothing + throw(ArgumentError("$name not found in path.\nRun Pkg.add(\"$name\") to install the $name package")) + end try if last && myid() == 1 && nprocs() > 1 # include on node 1 first to check for PrecompilableErrors From 0d457dd23961861a88c2b3b003fa5367831fab4a Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Fri, 16 Oct 2015 17:42:40 -0400 Subject: [PATCH 0579/1938] update url to point to github's token's page --- doc/manual/packages.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/packages.rst b/doc/manual/packages.rst index 49d77e52e338a..994532a50a575 100644 --- a/doc/manual/packages.rst +++ b/doc/manual/packages.rst @@ -800,7 +800,7 @@ on GitHub, push your changes to your fork, and open a pull request:: then you may have encountered an issue from using the GitHub API on multiple systems. The solution is to delete the "Julia Package Manager" personal access token `from your Github account - `_ and try again. + `_ and try again. Other failures may require you to circumvent :func:`Pkg.publish` by `creating a pull request on GitHub From 897886ce84dd063aad394d339aa30304dd5fb89f Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 16 Oct 2015 19:03:22 -0400 Subject: [PATCH 0580/1938] remove gcc-isms from runtime-intrinsics code --- src/intrinsics.cpp | 2 +- src/runtime_intrinsics.c | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index fe6a4baf2caa0..29f85674a447b 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -920,7 +920,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, case ccall: return emit_ccall(args, nargs, ctx); case cglobal: return emit_cglobal(args, nargs, ctx); case llvmcall: return emit_llvmcall(args, nargs, ctx); -#if 0 +#if 0 // this section enables runtime-intrinsics (e.g. for testing), and disables their llvm counterparts default: int ldepth = ctx->gc.argDepth; Value *r; diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index a7910152bcd23..6b7d3edfe09ba 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -86,7 +86,7 @@ static inline unsigned int next_power_of_two(unsigned int val) { static inline char signbitbyte(void *a, unsigned bytes) { // sign bit of an signed number of n bytes, as a byte - return signbit(((signed char*)a)[bytes-1]) ? ~0 : 0; + return (((signed char*)a)[bytes - 1] < 0) ? ~0 : 0; } static inline char usignbitbyte(void *a, unsigned bytes) { @@ -111,7 +111,9 @@ static inline unsigned select_by_size(unsigned sz) typedef intrinsic##_t select_##intrinsic##_t[6]; \ static inline intrinsic##_t select_##intrinsic(unsigned sz, select_##intrinsic##_t list) \ { \ - return list[select_by_size(sz)] ?: list[0]; \ + intrinsic##_t thunk = list[select_by_size(sz)]; \ + if (!thunk) thunk = list[0]; \ + return thunk; \ } #define fp_select(a, func) \ @@ -289,7 +291,7 @@ static inline jl_value_t *jl_iintrinsic_1(jl_value_t *ty, jl_value_t *a, const c /* TODO: this memcpy assumes little-endian, * for big-endian, need to align the copy to the other end */ \ memcpy(pa2, pa, isize); - memset(pa2 + isize, getsign(pa, isize), osize2 - isize); + memset((char*)pa2 + isize, getsign(pa, isize), osize2 - isize); pa = pa2; } jl_value_t *newv = lambda1(ty, pa, osize, osize2, list); @@ -837,8 +839,9 @@ DLLEXPORT jl_value_t *jl_check_top_bit(jl_value_t *a) // checked arithmetic #define check_sadd(a,b) \ - /* this test is a reduction of (b > 0) ? (a + b >= typemin(a)) : (a + b < typemin(a)) ==> overflow */ \ - (b > 0) == (a >= (((typeof(a))1) << (8 * sizeof(a) - 1)) - b) + /* this test is a reduction of (b > 0) ? (a + b >= typemin(a)) : (a + b < typemin(a)) ==> overflow \ + * where (a - a) == (typeof(a))0 */ \ + (b > 0) == (a >= ((a - a + 1) << (8 * sizeof(a) - 1)) - b) checked_iintrinsic_fast(LLVMAdd_sov, check_sadd, add, checked_sadd, ) #define check_uadd(a,b) \ /* this test checks for (a + b) > typemax(a) ==> overflow */ \ From 3edb24c2e5aecb30c9ab06c440e06a372dc13241 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 15 Oct 2015 18:03:42 -0400 Subject: [PATCH 0581/1938] keep TypeVar out of typeinf results, and add assertion --- base/inference.jl | 22 ++++++++++++++++------ src/dump.c | 1 + 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index fc190395ebb30..3c2836a8968da 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -903,6 +903,9 @@ function abstract_call(f, fargs, argtypes::Vector{Any}, vtypes, sv::StaticVarInf end end rt = builtin_tfunction(f, fargs, Tuple{argtypes...}, vtypes, sv) + if isa(rt, TypeVar) + rt = rt.ub + end #print("=> ", rt, "\n") return rt end @@ -1324,14 +1327,14 @@ function typeinf(linfo::LambdaStaticData,atypes::ANY,sparams::SimpleVector, def, break end if isa(code,Type) - curtype = code + curtype = code::Type # sometimes just a return type is stored here. if a full AST # is not needed, we can return it. if !needtree return (nothing, code) end else - curtype = ccall(:jl_ast_rettype, Any, (Any,Any), def, code) + curtype = ccall(:jl_ast_rettype, Any, (Any,Any), def, code)::Type return (code, curtype) end end @@ -1344,7 +1347,7 @@ function typeinf(linfo::LambdaStaticData,atypes::ANY,sparams::SimpleVector, def, (fulltree, result, rec) = typeinf_uncached(linfo, atypes, sparams, def, curtype, cop, true) if fulltree === () - return (fulltree,result) + return (fulltree, result::Type) end if !redo @@ -1373,7 +1376,7 @@ function typeinf(linfo::LambdaStaticData,atypes::ANY,sparams::SimpleVector, def, def.tfunc[tfunc_idx+1] = rec end - return (fulltree, result) + return (fulltree, result::Type) end typeinf_uncached(linfo, atypes::ANY, sparams::ANY; optimize=true) = @@ -1487,14 +1490,21 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV lastatype = lastatype.parameters[1] laty -= 1 end + if isa(lastatype, TypeVar) + lastatype = lastatype.ub + end if laty > la laty = la end for i=1:laty - s[1][args[i]] = VarState(atypes.parameters[i],false) + atyp = atypes.parameters[i] + if isa(atyp, TypeVar) + atyp = atyp.ub + end + s[1][args[i]] = VarState(atyp, false) end for i=laty+1:la - s[1][args[i]] = VarState(lastatype,false) + s[1][args[i]] = VarState(lastatype, false) end elseif la != 0 return ((), Bottom, false) # wrong number of arguments diff --git a/src/dump.c b/src/dump.c index da91aa1f86e95..84d57ea926d2d 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1942,6 +1942,7 @@ DLLEXPORT jl_value_t *jl_ast_rettype(jl_lambda_info_t *li, jl_value_t *ast) { if (jl_is_expr(ast)) return jl_lam_body((jl_expr_t*)ast)->etype; + assert(jl_is_array(ast)); JL_SIGATOMIC_BEGIN(); DUMP_MODES last_mode = mode; mode = MODE_AST; From 6e100101742d7ba20a11e5d306fe9f1842336d21 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 13 Oct 2015 21:15:24 -0400 Subject: [PATCH 0582/1938] avoid running and clearing the __init__ list when building output --- src/init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/init.c b/src/init.c index 05479077a80cb..71d5eb156eef6 100644 --- a/src/init.c +++ b/src/init.c @@ -604,7 +604,7 @@ void _julia_init(JL_IMAGE_SEARCH rel) jl_gc_enable(1); - if (jl_options.image_file) { + if (jl_options.image_file && (!jl_generating_output() || jl_options.incremental)) { jl_array_t *temp = jl_module_init_order; JL_GC_PUSH1(&temp); jl_module_init_order = NULL; From 040cf46f99f863fb201881813ee92a36ec69bb3e Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 15 Oct 2015 18:25:42 -0400 Subject: [PATCH 0583/1938] store a minimal list for jl_module_init_order reduces runtime lookup effort. but the real reason is that it is a workaround because of the replacing of Core.Inference --- src/init.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/init.c b/src/init.c index 71d5eb156eef6..aea44e293045e 100644 --- a/src/init.c +++ b/src/init.c @@ -627,15 +627,29 @@ void jl_compile_all(void); static void julia_save() { + if (!jl_generating_output()) + return; + if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL) jl_compile_all(); - if (jl_options.incremental) { - jl_array_t *worklist = jl_module_init_order; - if (!worklist) { - jl_printf(JL_STDERR, "WARNING: incremental output requested, but no modules defined during run\n"); - return; + if (!jl_module_init_order) { + jl_printf(JL_STDERR, "WARNING: --output requested, but no modules defined during run\n"); + return; + } + + jl_array_t *worklist = jl_module_init_order; + JL_GC_PUSH1(&worklist); + jl_module_init_order = jl_alloc_cell_1d(0); + int i, l = jl_array_len(worklist); + for (i = 0; i < l; i++) { + jl_value_t *m = jl_arrayref(worklist, i); + if (jl_module_get_initializer((jl_module_t*)m)) { + jl_cell_1d_push(jl_module_init_order, m); } + } + + if (jl_options.incremental) { if (jl_options.outputji) if (jl_save_incremental(jl_options.outputji, worklist)) jl_exit(1); @@ -668,6 +682,7 @@ static void julia_save() if (jl_options.outputo) jl_dump_objfile((char*)jl_options.outputo, 0, (const char*)s->buf, s->size); } + JL_GC_POP(); } jl_function_t *jl_typeinf_func=NULL; From cda8b06d31466467d13d2a473de51a545d73781c Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 13 Oct 2015 16:56:16 -0400 Subject: [PATCH 0584/1938] simplify more functions signatures that use Intrinsics (for compile-all mode) --- base/c.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/base/c.jl b/base/c.jl index a32bdda4ae56d..adc159e65bf1f 100644 --- a/base/c.jl +++ b/base/c.jl @@ -2,7 +2,7 @@ # definitions related to C interface -import Core.Intrinsics: cglobal, box, unbox +import Core.Intrinsics: cglobal, box const OS_NAME = ccall(:jl_get_OS_NAME, Any, ()) @@ -58,10 +58,10 @@ else bitstype 32 Cwstring end -convert{T<:Union{Int8,UInt8}}(::Type{Cstring}, p::Ptr{T}) = box(Cstring, unbox(Ptr{T}, p)) -convert(::Type{Cwstring}, p::Ptr{Cwchar_t}) = box(Cwstring, unbox(Ptr{Cwchar_t}, p)) -convert{T<:Union{Int8,UInt8}}(::Type{Ptr{T}}, p::Cstring) = box(Ptr{T}, unbox(Cstring, p)) -convert(::Type{Ptr{Cwchar_t}}, p::Cwstring) = box(Ptr{Cwchar_t}, unbox(Cwstring, p)) +convert{T<:Union{Int8,UInt8}}(::Type{Cstring}, p::Ptr{T}) = box(Cstring, p) +convert(::Type{Cwstring}, p::Ptr{Cwchar_t}) = box(Cwstring, p) +convert{T<:Union{Int8,UInt8}}(::Type{Ptr{T}}, p::Cstring) = box(Ptr{T}, p) +convert(::Type{Ptr{Cwchar_t}}, p::Cwstring) = box(Ptr{Cwchar_t}, p) # here, not in pointer.jl, to avoid bootstrapping problems in coreimg.jl pointer_to_string(p::Cstring, own::Bool=false) = pointer_to_string(convert(Ptr{UInt8}, p), own) From f0c911d847bf7b3e130b8b705f2224621b33218e Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Fri, 16 Oct 2015 16:29:16 -0400 Subject: [PATCH 0585/1938] more verbose Pkg.update() for checked out packages --- base/libgit2.jl | 5 ++--- base/pkg/entry.jl | 44 +++++++++++++++++++++++++++++++++----------- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/base/libgit2.jl b/base/libgit2.jl index 0e7281c540e0b..103232ba6b2fc 100644 --- a/base/libgit2.jl +++ b/base/libgit2.jl @@ -70,7 +70,8 @@ function iscommit(id::AbstractString, repo::GitRepo) end """ git diff-index HEAD [-- ]""" -isdirty(repo::GitRepo, paths::AbstractString=""; cached::Bool=false) = isdiff(repo, Consts.HEAD_FILE, paths, cached=cached) +isdirty(repo::GitRepo, paths::AbstractString=""; cached::Bool=false) = + isdiff(repo, Consts.HEAD_FILE, paths, cached=cached) """ git diff-index [-- ]""" function isdiff(repo::GitRepo, treeish::AbstractString, paths::AbstractString=""; cached::Bool=false) @@ -186,8 +187,6 @@ function branch(repo::GitRepo) head_ref = head(repo) try branch(head_ref) - catch - "" finally finalize(head_ref) end diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 67076308885d9..461064e87358a 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -331,8 +331,12 @@ function update(branch::AbstractString) with(GitRepo, "METADATA") do repo with(LibGit2.head(repo)) do h if LibGit2.branch(h) != branch - LibGit2.isdirty(repo) && throw(PkgError("METADATA is dirty and not on $branch, bailing")) - LibGit2.isattached(repo) || throw(PkgError("METADATA is detached not on $branch, bailing")) + if LibGit2.isdirty(repo) + throw(PkgError("METADATA is dirty and not on $branch, bailing")) + end + if !LibGit2.isattached(repo) + throw(PkgError("METADATA is detached not on $branch, bailing")) + end LibGit2.fetch(repo) LibGit2.checkout_head(repo) LibGit2.branch!(repo, branch, track="refs/remotes/origin/$branch") @@ -340,11 +344,14 @@ function update(branch::AbstractString) end end LibGit2.fetch(repo) - LibGit2.merge!(repo, fastforward=true) || LibGit2.rebase!(repo, "origin/$branch") + ff_succeeded = LibGit2.merge!(repo, fastforward=true) + if !ff_succeeded + LibGit2.rebase!(repo, "origin/$branch") + end end avail = Read.available() # this has to happen before computing free/fixed - for pkg in filter!(Read.isinstalled,collect(keys(avail))) + for pkg in filter(Read.isinstalled, collect(keys(avail))) try Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail[pkg]]) catch err @@ -352,19 +359,34 @@ function update(branch::AbstractString) end end instd = Read.installed(avail) - free = Read.free(instd) + free = Read.free(instd) for (pkg,ver) in free - Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail[pkg]]) + Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a) in avail[pkg]]) end fixed = Read.fixed(avail,instd) for (pkg,ver) in fixed ispath(pkg,".git") || continue with(GitRepo, pkg) do repo - if LibGit2.isattached(repo) && !LibGit2.isdirty(repo) - info("Updating $pkg...") - @recover begin - LibGit2.fetch(repo) - LibGit2.merge!(repo, fastforward=true) + if LibGit2.isattached(repo) + if LibGit2.isdirty(repo) + warn("Package $pkg: skipping update (dirty)...") + else + prev_sha = string(LibGit2.head_oid(repo)) + success = true + try + LibGit2.fetch(repo) + LibGit2.merge!(repo, fastforward=true) + catch err + show(err) + print('\n') + success = false + end + if success + post_sha = string(LibGit2.head_oid(repo)) + branch = LibGit2.branch(repo) + info("Updating $pkg $branch...", + prev_sha != post_sha ? " $(prev_sha[1:8]) → $(post_sha[1:8])" : "") + end end end end From f0f5938f573c5d653b4083d5d7f3fad84cff2412 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 17 Oct 2015 03:10:12 -0400 Subject: [PATCH 0586/1938] Fix #13223, linear indexing of sparse matrices make getindex(A::SparseMatrixCSC, I::AbstractArray) faster by allocating at most the number of nonzeros in A, and add a specialized implementation for getindex(A::SparseMatrixCSC, I::UnitRange) --- base/sparse/sparsevector.jl | 64 ++++++++++++++++++++++++++-------- test/sparsedir/sparsevector.jl | 6 ++++ 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index 43ea1dbf5cbc5..eebbeaf2ad6f3 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -376,10 +376,11 @@ getindex{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractArray{Bool}) = _logical_index(A, function _logical_index{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractArray{Bool}) checkbounds(A, I) n = sum(I) + nnzB = min(n, nnz(A)) colptrA = A.colptr; rowvalA = A.rowval; nzvalA = A.nzval - rowvalB = Array(Int, n) - nzvalB = Array(Tv, n) + rowvalB = Array(Int, nnzB) + nzvalB = Array(Tv, nnzB) c = 1 rowB = 1 @@ -403,17 +404,18 @@ function _logical_index{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractArray{Bool}) end (rowB > n) && break end - n = length(nzvalB) - if n > (c-1) - deleteat!(nzvalB, c:n) - deleteat!(rowvalB, c:n) + if nnzB > (c-1) + deleteat!(nzvalB, c:nnzB) + deleteat!(rowvalB, c:nnzB) end SparseVector(n, rowvalB, nzvalB) end -# TODO: huge optimizations are available for I::Range and ::Colon +# TODO: further optimizations are available for ::Colon and other types of Range getindex(A::SparseMatrixCSC, ::Colon) = A[1:end] -function getindex{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractVector) + +function getindex{Tv}(A::SparseMatrixCSC{Tv}, I::UnitRange) + checkbounds(A, I) szA = size(A) nA = szA[1]*szA[2] colptrA = A.colptr @@ -421,12 +423,46 @@ function getindex{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractVector) nzvalA = A.nzval n = length(I) - rowvalB = Array(Int, n) - nzvalB = Array(Tv, n) + nnzB = min(n, nnz(A)) + rowvalB = Array(Int, nnzB) + nzvalB = Array(Tv, nnzB) + + rowstart,colstart = ind2sub(szA, first(I)) + rowend,colend = ind2sub(szA, last(I)) - rowB = 1 idxB = 1 + @inbounds for col in colstart:colend + minrow = (col == colstart ? rowstart : 1) + maxrow = (col == colend ? rowend : szA[1]) + for r in colptrA[col]:(colptrA[col+1]-1) + rowA = rowvalA[r] + if minrow <= rowA <= maxrow + rowvalB[idxB] = sub2ind(szA, rowA, col) - first(I) + 1 + nzvalB[idxB] = nzvalA[r] + idxB += 1 + end + end + end + if nnzB > (idxB-1) + deleteat!(nzvalB, idxB:nnzB) + deleteat!(rowvalB, idxB:nnzB) + end + SparseVector(n, rowvalB, nzvalB) +end + +function getindex{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractVector) + szA = size(A) + nA = szA[1]*szA[2] + colptrA = A.colptr + rowvalA = A.rowval + nzvalA = A.nzval + n = length(I) + nnzB = min(n, nnz(A)) + rowvalB = Array(Int, nnzB) + nzvalB = Array(Tv, nnzB) + + idxB = 1 for i in 1:n ((I[i] < 1) | (I[i] > nA)) && throw(BoundsError(A, I)) row,col = ind2sub(szA, I[i]) @@ -439,9 +475,9 @@ function getindex{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractVector) end end end - if n > (idxB-1) - deleteat!(nzvalB, idxB:n) - deleteat!(rowvalB, idxB:n) + if nnzB > (idxB-1) + deleteat!(nzvalB, idxB:nnzB) + deleteat!(rowvalB, idxB:nnzB) end SparseVector(n, rowvalB, nzvalB) end diff --git a/test/sparsedir/sparsevector.jl b/test/sparsedir/sparsevector.jl index d7c1a1eba6177..a8186fc81d10d 100644 --- a/test/sparsedir/sparsevector.jl +++ b/test/sparsedir/sparsevector.jl @@ -328,6 +328,12 @@ let S = sprand(4, 8, 0.5) @test length(v) == length(S) - i + 1 @test full(v) == Sf[i:end] end + for i=0:div(length(S),2) + v = S[1+i:end-i] + @test isa(v, SparseVector{Float64,Int}) + @test length(v) == length(S) - 2i + @test full(v) == Sf[1+i:end-i] + end end ## math From 0fa069a5129e9d6dc94e6d5d6e105080b72efff0 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 17 Oct 2015 04:34:12 -0400 Subject: [PATCH 0587/1938] Fix compilation under MSVC after #13485 --- src/intrinsics.cpp | 2 +- src/runtime_ccall.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 29f85674a447b..ce923a37a5937 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -913,7 +913,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, f = fptosi_auto; unsigned expected_nargs = intrinsic_nargs[f]; if (expected_nargs && expected_nargs != nargs) { - jl_errorf("intrinsic #%d %s: wrong number of arguments", f, jl_intrinsic_name((int)f)); + jl_errorf("intrinsic #%d %s: wrong number of arguments", f, JL_I::jl_intrinsic_name((int)f)); } switch (f) { diff --git a/src/runtime_ccall.cpp b/src/runtime_ccall.cpp index 81e43164c64bd..eb7b9dec91ae1 100644 --- a/src/runtime_ccall.cpp +++ b/src/runtime_ccall.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "julia.h" #include "julia_internal.h" @@ -128,7 +129,6 @@ void *jl_load_and_lookup(char *f_lib, char *f_name, uv_lib_t **hnd) } // miscellany -#include extern "C" DLLEXPORT jl_value_t *jl_get_cpu_name(void) { From 098bf8fa8699eff558ad2fffb6dfaa427cedb1b6 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 17 Oct 2015 05:23:19 -0400 Subject: [PATCH 0588/1938] Account for possibility of repeated indices in getindex(A::SparseMatrixCSC, I::AbstractVector) --- base/sparse/sparsevector.jl | 11 ++++++-- test/sparsedir/sparsevector.jl | 51 ++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index eebbeaf2ad6f3..49e1d5fa913e2 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -468,9 +468,14 @@ function getindex{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractVector) row,col = ind2sub(szA, I[i]) for r in colptrA[col]:(colptrA[col+1]-1) @inbounds if rowvalA[r] == row - rowvalB[idxB] = i - nzvalB[idxB] = nzvalA[r] - idxB += 1 + if idxB <= nnzB + rowvalB[idxB] = i + nzvalB[idxB] = nzvalA[r] + idxB += 1 + else # this can happen if there are repeated indices in I + push!(rowvalB, i) + push!(nzvalB, nzvalA[r]) + end break end end diff --git a/test/sparsedir/sparsevector.jl b/test/sparsedir/sparsevector.jl index a8186fc81d10d..ff40b3ed1a033 100644 --- a/test/sparsedir/sparsevector.jl +++ b/test/sparsedir/sparsevector.jl @@ -336,6 +336,57 @@ let S = sprand(4, 8, 0.5) end end +let r = [1,10], S = sparse(r, r, r) + Sf = full(S) + @assert isa(Sf, Matrix{Int}) + + inds = [1,1,1,1,1,1] + v = S[inds] + @test isa(v, SparseVector{Int,Int}) + @test length(v) == length(inds) + @test full(v) == Sf[inds] + + inds = [2,2,2,2,2,2] + v = S[inds] + @test isa(v, SparseVector{Int,Int}) + @test length(v) == length(inds) + @test full(v) == Sf[inds] + + # get a single column + for j = 1:size(S,2) + col = S[:, j] + @test isa(col, SparseVector{Int,Int}) + @test length(col) == size(S,1) + @test full(col) == Sf[:,j] + end + + # Get a reshaped vector + v = S[:] + @test isa(v, SparseVector{Int,Int}) + @test length(v) == length(S) + @test full(v) == Sf[:] + + # Get a linear subset + for i=0:length(S) + v = S[1:i] + @test isa(v, SparseVector{Int,Int}) + @test length(v) == i + @test full(v) == Sf[1:i] + end + for i=1:length(S)+1 + v = S[i:end] + @test isa(v, SparseVector{Int,Int}) + @test length(v) == length(S) - i + 1 + @test full(v) == Sf[i:end] + end + for i=0:div(length(S),2) + v = S[1+i:end-i] + @test isa(v, SparseVector{Int,Int}) + @test length(v) == length(S) - 2i + @test full(v) == Sf[1+i:end-i] + end +end + ## math ### Data From ad7a263440fbe27496183eb97bb8d0cfcdc62424 Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Sat, 17 Oct 2015 10:16:44 -0400 Subject: [PATCH 0589/1938] fix Pkg.installed() that asummed every dir in Pkg was a git repo --- base/pkg/read.jl | 15 +++++++-------- test/pkg.jl | 9 +++++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/base/pkg/read.jl b/base/pkg/read.jl index c24019cc4b928..352d23cd19075 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -178,15 +178,14 @@ function installed(avail::Dict=available()) for pkg in readdir() isinstalled(pkg) || continue ap = get(avail,pkg,Dict{VersionNumber,Available}()) - prepo = LibGit2.GitRepo(pkg) - try - ver = installed_version(pkg, prepo, ap) - fixed = isfixed(pkg, prepo, ap) - pkgs[pkg] = (ver, fixed) - catch + if ispath(pkg,".git") + LibGit2.with(LibGit2.GitRepo, pkg) do repo + ver = installed_version(pkg, repo, ap) + fixed = isfixed(pkg, repo, ap) + pkgs[pkg] = (ver, fixed) + end + else pkgs[pkg] = (typemin(VersionNumber), true) - finally - finalize(prepo) end end return pkgs diff --git a/test/pkg.jl b/test/pkg.jl index 3c725ae451f96..39dfb5bdae81d 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -10,7 +10,6 @@ function temp_pkg_dir(fn::Function, remove_tmp_dir::Bool=true) Pkg.init() @test isdir(Pkg.dir()) Pkg.resolve() - fn() finally remove_tmp_dir && rm(tmpdir, recursive=true) @@ -144,8 +143,14 @@ temp_pkg_dir() do Pkg.installed()["Example"] == v"0.4.0" end + # add a directory that is not a git repository + begin + mkdir(joinpath(Pkg.dir(), "NOTGIT")) + Pkg.installed()["NOTGIT"] == typemin(VersionNumber) + end + begin Pkg.rm("Example") - @test isempty(Pkg.installed()) + @test !in("Example", keys(Pkg.installed())) end end From af5e483cff0f074018051133f275cb51d13a5998 Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Fri, 16 Oct 2015 19:23:25 -0400 Subject: [PATCH 0590/1938] don't swallow pkg read errors, better output for Pkg.status() --- base/pkg/dir.jl | 6 +++--- base/pkg/entry.jl | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/base/pkg/dir.jl b/base/pkg/dir.jl index 7566f6ec2c9ad..7f6733d111fd0 100644 --- a/base/pkg/dir.jl +++ b/base/pkg/dir.jl @@ -61,10 +61,10 @@ function init(meta::AbstractString=DEFAULT_META, branch::AbstractString=META_BRA Base.mv(joinpath(temp_dir,"REQUIRE"), joinpath(dir,"REQUIRE")) Base.mv(joinpath(temp_dir,"META_BRANCH"), joinpath(dir,"META_BRANCH")) rm(temp_dir) - catch e + catch err ispath(metadata_dir) && rm(metadata_dir, recursive=true) ispath(temp_dir) && rm(temp_dir, recursive=true) - rethrow(e) + rethrow(err) end end @@ -73,7 +73,7 @@ function getmetabranch() open(joinpath(path(),"META_BRANCH")) do io chomp(readuntil(io, "/n")) end - catch + catch err META_BRANCH end end diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 461064e87358a..522a33e0fd8bf 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -174,13 +174,13 @@ function status(io::IO, pkg::AbstractString, ver::VersionNumber, fix::Bool) isfile("METADATA",pkg,"url") || push!(attrs,"unregistered") LibGit2.isdirty(prepo) && push!(attrs,"dirty") isempty(attrs) || print(io, " (",join(attrs,", "),")") - catch - print(io, "broken-repo (unregistered)") + catch err + print_with_color(:red, io, " broken-repo (unregistered)") finally finalize(prepo) end else - print(io, "non-repo (unregistered)") + print_with_color(:yellow, io, "non-repo (unregistered)") end println(io) end @@ -196,9 +196,9 @@ function clone(url::AbstractString, pkg::AbstractString) LibGit2.with(LibGit2.clone(url, pkg)) do repo LibGit2.set_remote_url(repo, url) end - catch + catch err isdir(pkg) && Base.rm(pkg, recursive=true) - rethrow() + rethrow(err) end info("Computing changes...") if !edit(Reqs.add, pkg) @@ -471,7 +471,7 @@ function resolve( end push!(changed,(pkg,(ver1,ver2))) end - catch + catch err for (pkg,(ver1,ver2)) in reverse!(changed) if ver1 === nothing info("Rolling back install of $pkg") @@ -484,7 +484,7 @@ function resolve( @recover Write.update(pkg, Read.sha1(pkg,ver1)) end end - rethrow() + rethrow(err) end # re/build all updated/installed packages build(map(x->x[1], filter(x -> x[2][2] !== nothing, changes))) @@ -553,10 +553,10 @@ function build!(pkgs::Vector, errs::Dict, seen::Set=Set()) errs[pkg] = err end end - catch + catch err kill(pobj) close(io) - rethrow() + rethrow(err) finally isfile(errfile) && Base.rm(errfile) end From cb211a076401f1f1aa70cd1a485d0def0b0d2916 Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Sat, 17 Oct 2015 17:38:16 -0400 Subject: [PATCH 0591/1938] fix another case where Pkg was not checking if the folder was a git repo --- base/pkg/cache.jl | 3 ++- base/pkg/entry.jl | 9 ++++----- base/pkg/resolve.jl | 3 ++- test/pkg.jl | 7 ++----- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/base/pkg/cache.jl b/base/pkg/cache.jl index 1f8f9bf0f0925..5ad11cf503c15 100644 --- a/base/pkg/cache.jl +++ b/base/pkg/cache.jl @@ -58,6 +58,7 @@ function prefetch(pkg::AbstractString, url::AbstractString, sha1s::Vector) finalize(repo) # closing repo opened/created above end end -prefetch(pkg::AbstractString, url::AbstractString, sha1::AbstractString...) = prefetch(pkg, url, AbstractString[sha1...]) +prefetch(pkg::AbstractString, url::AbstractString, sha1::AbstractString...) = + prefetch(pkg, url, AbstractString[sha1...]) end # module diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 522a33e0fd8bf..f3532523ce920 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -113,11 +113,10 @@ function installed(pkg::AbstractString) avail = Read.available(pkg) if Read.isinstalled(pkg) res = typemin(VersionNumber) - repo = GitRepo(pkg) - try - res = Read.installed_version(pkg, repo, avail) - finally - finalize(repo) + if ispath(joinpath(pkg,".git")) + LibGit2.with(GitRepo, pkg) do repo + res = Read.installed_version(pkg, repo, avail) + end end return res end diff --git a/base/pkg/resolve.jl b/base/pkg/resolve.jl index d0b2005fece0f..0f943c59aa05d 100644 --- a/base/pkg/resolve.jl +++ b/base/pkg/resolve.jl @@ -50,7 +50,8 @@ function resolve(reqs::Requires, deps::Dict{ByteString,Dict{VersionNumber,Availa end # Scan dependencies for (explicit or implicit) contradictions -function sanity_check(deps::Dict{ByteString,Dict{VersionNumber,Available}}, pkgs::Set{ByteString} = Set{ByteString}()) +function sanity_check(deps::Dict{ByteString,Dict{VersionNumber,Available}}, + pkgs::Set{ByteString} = Set{ByteString}()) isempty(pkgs) || (deps = Query.undirected_dependencies_subset(deps, pkgs)) diff --git a/test/pkg.jl b/test/pkg.jl index 39dfb5bdae81d..155063a99c1e7 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -43,6 +43,7 @@ temp_pkg_dir() do Pkg.rm("Example") @test isempty(Pkg.installed()) @test !isempty(Pkg.available("Example")) + @test !in("Example", keys(Pkg.installed())) Pkg.clone("https://github.com/JuliaLang/Example.jl.git") @test [keys(Pkg.installed())...] == ["Example"] Pkg.status("Example", iob) @@ -146,11 +147,7 @@ temp_pkg_dir() do # add a directory that is not a git repository begin mkdir(joinpath(Pkg.dir(), "NOTGIT")) + Pkg.installed("NOTGIT") == typemin(VersionNumber) Pkg.installed()["NOTGIT"] == typemin(VersionNumber) end - - begin - Pkg.rm("Example") - @test !in("Example", keys(Pkg.installed())) - end end From e85fe40da43015d6a3f6231ec6cd5b8dd22ef1b7 Mon Sep 17 00:00:00 2001 From: Iain Dunning Date: Sat, 17 Oct 2015 18:05:34 -0400 Subject: [PATCH 0592/1938] BaseTestNext/Deprecated added to NEWS [ci skip] --- NEWS.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/NEWS.md b/NEWS.md index 8adc4a08e24ae..bd50a0f5c70e8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -30,6 +30,14 @@ Library improvements * The `Base.Test` module now has a `@testset` feature to bundle tests together and delay throwing an error until the end ([#13062]). + * The new features are mirrored in the + [BaseTestNext](https://github.com/IainNZ/BaseTestNext.jl) + package for users who would like to use the new functionality on Julia v0.4. + + * The [BaseTestDeprecated](https://github.com/IainNZ/BaseTestDeprecated.jl) + package provides the old-style `handler` functionality, for compatibility + with code that needs to support both Julia v0.4 and v0.5. + * The functions `remotecall`, `remotecall_fetch`, and `remotecall_wait` now have the function argument as the first argument to allow for do-block syntax ([#13338]). From 5e9d02f228ffb2b2e0a4db6a2eb810eb9f12246e Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Sun, 4 Oct 2015 00:22:43 -0400 Subject: [PATCH 0593/1938] Improve parse_eval_all line number reporting --- src/interpreter.c | 2 ++ src/toplevel.c | 35 ++++++++++++++++++++++++++--------- test/cmdlineargs.jl | 2 +- test/core.jl | 2 +- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/interpreter.c b/src/interpreter.c index 509cd6e50b319..c6f09d441e0e9 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -15,6 +15,7 @@ extern "C" { #endif extern int jl_lineno; +extern const char *jl_filename; static jl_value_t *eval(jl_value_t *e, jl_value_t **locals, size_t nl, size_t ngensym); static jl_value_t *eval_body(jl_array_t *stmts, jl_value_t **locals, size_t nl, size_t ngensym, @@ -159,6 +160,7 @@ static jl_value_t *eval(jl_value_t *e, jl_value_t **locals, size_t nl, size_t ng } if (jl_is_linenode(e)) { jl_lineno = jl_linenode_line(e); + jl_filename = jl_linenode_file(e)->name; } if (jl_is_newvarnode(e)) { jl_value_t *var = jl_fieldref(e,0); diff --git a/src/toplevel.c b/src/toplevel.c index 1e5f34aef1214..7a899fb282ee1 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -555,11 +555,12 @@ jl_value_t *jl_parse_eval_all(const char *fname, size_t len) { //jl_printf(JL_STDERR, "***** loading %s\n", fname); int last_lineno = jl_lineno; + int top_lineno = -1; const char *last_filename = jl_filename; jl_lineno = 0; jl_filename = fname; - jl_value_t *fn=NULL, *ln=NULL, *form=NULL, *result=jl_nothing; - JL_GC_PUSH4(&fn, &ln, &form, &result); + jl_value_t *fn=NULL, *ln=NULL, *form=NULL, *nest_exc=NULL, *result=jl_nothing; + JL_GC_PUSH5(&fn, &ln, &form, &result, &nest_exc); JL_TRY { // handle syntax error while (1) { @@ -574,22 +575,38 @@ jl_value_t *jl_parse_eval_all(const char *fname, size_t len) jl_interpret_toplevel_expr(form); } } + top_lineno = jl_lineno; // jl_parse_next sets lineno. result = jl_toplevel_eval_flex(form, 1); } } JL_CATCH { jl_stop_parsing(); - fn = jl_pchar_to_string(fname, len); - ln = jl_box_long(jl_lineno); - jl_lineno = last_lineno; - jl_filename = last_filename; + if (jl_loaderror_type == NULL) { + // reset line and filename before throwing + jl_lineno = last_lineno; + jl_filename = last_filename; jl_rethrow(); } - else { - jl_rethrow_other(jl_new_struct(jl_loaderror_type, fn, ln, - jl_exception_in_transit)); + + fn = jl_pchar_to_string(jl_filename, strlen(jl_filename)); + ln = jl_box_long(jl_lineno); + nest_exc = jl_new_struct(jl_loaderror_type, fn, ln, + jl_exception_in_transit); + + if ((strcmp(jl_filename, fname) == 0) && + jl_lineno == top_lineno) { + jl_lineno = last_lineno; + jl_filename = last_filename; + jl_rethrow_other(nest_exc); } + + jl_lineno = last_lineno; + jl_filename = last_filename; + fn = jl_pchar_to_string(fname, len); + ln = jl_box_long(top_lineno); + jl_rethrow_other(jl_new_struct(jl_loaderror_type, fn, ln, + nest_exc)); } jl_stop_parsing(); jl_lineno = last_lineno; diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 9a8e9396aed1a..e5192e6edeb3e 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -173,7 +173,7 @@ let exename = `$(joinpath(JULIA_HOME, Base.julia_exename())) --precompiled=yes` wait(proc) close(out.in) @test success(proc) - @test readchomp(out) == "WARNING: Foo.Deprecated is deprecated.\n likely near no file:5" + @test readchomp(out) == "WARNING: Foo.Deprecated is deprecated.\n likely near none:5" end @unix_only let out = Pipe(), diff --git a/test/core.jl b/test/core.jl index 666e87321d482..9bb929f9e89f7 100644 --- a/test/core.jl +++ b/test/core.jl @@ -704,7 +704,7 @@ let didthrow = false catch ex @test isa(ex, LoadError) - @test isa(ex.error, InitError) + @test isa(ex.error.error, InitError) true end @test didthrow From d418cbab0617faaed132ac99c40b8c8f58c6d568 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sat, 17 Oct 2015 21:36:32 -0400 Subject: [PATCH 0594/1938] Fix compilation with LLVM37 --- src/runtime_ccall.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/runtime_ccall.cpp b/src/runtime_ccall.cpp index eb7b9dec91ae1..f4e2152a00f04 100644 --- a/src/runtime_ccall.cpp +++ b/src/runtime_ccall.cpp @@ -6,6 +6,7 @@ #include #include "julia.h" #include "julia_internal.h" +using namespace llvm; // --- library symbol lookup --- From f244fb697ed335816f3b20f8c1d3aae013e2a2d6 Mon Sep 17 00:00:00 2001 From: Graham Inggs Date: Mon, 12 Oct 2015 10:18:41 +0200 Subject: [PATCH 0595/1938] Define all other architectures as "not x86" This allows the build to proceed for AArch64 (mentioned in #10791) and others. --- Make.inc | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Make.inc b/Make.inc index 4c1804c8fc707..e929c21632763 100644 --- a/Make.inc +++ b/Make.inc @@ -591,14 +591,9 @@ ISX86:=1 else ifeq ($(ARCH),x86_64) BINARY:=64 ISX86:=1 -else ifneq (,$(findstring arm,$(ARCH))) -ISX86:=0 -else ifneq (,$(findstring powerpc,$(ARCH))) -ISX86:=0 -else ifneq (,$(findstring ppc,$(ARCH))) -ISX86:=0 else -$(error "unknown word-size for arch: $(ARCH)") +# For all other architectures (ARM, PPC, AArch64, etc.) +ISX86:=0 endif # If we are running on ARM, set certain options automatically From c0eb6f1096d76ae67f557cda9485449a3babcde1 Mon Sep 17 00:00:00 2001 From: lucasb-eyer Date: Sun, 18 Oct 2015 16:50:15 +0200 Subject: [PATCH 0596/1938] Document noteworthy difference in += and co. --- doc/manual/noteworthy-differences.rst | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/manual/noteworthy-differences.rst b/doc/manual/noteworthy-differences.rst index 82ebc7973c380..f023327995f79 100644 --- a/doc/manual/noteworthy-differences.rst +++ b/doc/manual/noteworthy-differences.rst @@ -252,6 +252,11 @@ Noteworthy differences from Python major (C-ordered) by default. To get optimal performance when looping over arrays, the order of the loops should be reversed in Julia relative to NumPy (see relevant section of :ref:`man-performance-tips`). +- Julia's updating operators (e.g. ``+=``, ``-=``, ...) are *not in-place* + whereas NumPy's are. This means ``A = ones(4); B = A; B += 3`` doesn't change + values in ``A``, it rather rebinds the name ``B`` to the result of the right- + hand side ``B = B + 3``, which is a new array. Use ``B[:] += 3``, explicit loops, + or ``InplaceOps.jl``. - Julia evaluates default values of function arguments every time the method is invoked, unlike in Python where the default values are evaluated only once when the function is defined. For example, the function ``f(x=rand()) = x`` @@ -269,7 +274,9 @@ Noteworthy differences from C/C++ See the Julia documentation for the syntax for array construction (it has changed between versions). - In Julia, indexing of arrays, strings, etc. is 1-based not 0-based. - Julia arrays are assigned by reference. After ``A=B``, changing elements of - ``B`` will modify ``A`` as well. + ``B`` will modify ``A`` as well. Updating operators like ``+=`` do not operate + in-place, they are equivalent to ``A = A + B`` which rebinds the left-hand + side to the result of the right-hand side expression. - Julia arrays are column major (Fortran ordered) whereas C/C++ arrays are row major ordered by default. To get optimal performance when looping over arrays, the order of the loops should be reversed in Julia relative to C/C++ @@ -300,7 +307,7 @@ Noteworthy differences from C/C++ characters without quoting it like ``"\""`` String literals can have values of other variables or expressions interpolated into them, indicated by ``$variablename`` or ``$(expression)``, which evaluates the variable name or the expression in the context of the function. -- ``//`` indicates a ``Rational`` number, and not a single-line comment (which is # in Julia) +- ``//`` indicates a ``Rational`` number, and not a single-line comment (which is ``#`` in Julia) - ``#=`` indicates the start of a multiline comment, and ``=#`` ends it. - Functions in Julia return values from their last expression(s) or the ``return`` keyword. Multiple values can be returned from functions and assigned as tuples, e.g. From 4649409c26363921f8a600bb9a8756e0be2f9cfd Mon Sep 17 00:00:00 2001 From: Art Kuo Date: Fri, 9 Oct 2015 12:33:13 -0400 Subject: [PATCH 0597/1938] added print_range for Range objects, to show the elements of the Range in REPL. Previous commits squashed, docstring moved back to front, builds and passes tests --- base/range.jl | 53 +++++++++++++++++++++++++++++++++++++++++++++ base/replutil.jl | 25 ++++++++++++++------- base/show.jl | 56 ++++++++++++++++++++++++++++++++++++++++-------- test/ranges.jl | 11 ++++++++++ 4 files changed, 128 insertions(+), 17 deletions(-) diff --git a/base/range.jl b/base/range.jl index 52db2504d032b..af2f4dab0be4d 100644 --- a/base/range.jl +++ b/base/range.jl @@ -244,6 +244,59 @@ function show(io::IO, r::LinSpace) print(io, ')') end +""" +`print_range(io, r)` prints out a nice looking range r in terms of its elements +as if it were `collect(r)`, dependent on the size of the +terminal, and taking into account whether compact numbers should be shown. +It figures out the width in characters of each element, and if they +end up too wide, it shows the first and last elements separated by a +horizontal elipsis. Typical output will look like `1.0,2.0,3.0,…,4.0,5.0,6.0`. + +`print_range(io, r, sz, pre, sep, post, hdots)` uses optional +parameters `sz` for the (rows,cols) of the screen, +`pre` and `post` characters for each printed row, `sep` separator string between +printed elements, `hdots` string for the horizontal ellipsis. +""" +function print_range(io::IO, r::Range, + sz::Tuple{Integer, Integer} = (s = tty_size(); (s[1]-4, s[2])), + pre::AbstractString = " ", + sep::AbstractString = ",", + post::AbstractString = "", + hdots::AbstractString = ",\u2026,") # horiz ellipsis + # This function borrows from print_matrix() in show.jl + # and should be called by writemime (replutil.jl) and by display() + screenheight, screenwidth = sz + screenwidth -= length(pre) + length(post) + postsp = "" + sepsize = length(sep) + m = 1 # treat the range as a one-row matrix + n = length(r) + # Figure out spacing alignments for r, but only need to examine the + # left and right edge columns, as many as could conceivably fit on the + # screen, with the middle columns summarized by horz, vert, or diag ellipsis + maxpossiblecols = div(screenwidth, 1+sepsize) # assume each element is at least 1 char + 1 separator + colsr = n <= maxpossiblecols ? (1:n) : [1:div(maxpossiblecols,2)+1; (n-div(maxpossiblecols,2)):n] + rowmatrix = r[colsr]' # treat the range as a one-row matrix for print_matrix_row + A = alignment(rowmatrix,1:m,1:length(rowmatrix),screenwidth,screenwidth,sepsize) # how much space range takes + if n <= length(A) # cols fit screen, so print out all elements + print(io, pre) # put in pre chars + print_matrix_row(io,rowmatrix,A,1,1:n,sep) # the entire range + print(io, post) # add the post characters + else # cols don't fit so put horiz ellipsis in the middle + # how many chars left after dividing width of screen in half + # and accounting for the horiz ellipsis + c = div(screenwidth-length(hdots)+1,2)+1 # chars remaining for each side of rowmatrix + alignR = reverse(alignment(rowmatrix,1:m,length(rowmatrix):-1:1,c,c,sepsize)) # which cols of rowmatrix to put on the right + c = screenwidth - sum(map(sum,alignR)) - (length(alignR)-1)*sepsize - length(hdots) + alignL = alignment(rowmatrix,1:m,1:length(rowmatrix),c,c,sepsize) # which cols of rowmatrix to put on the left + print(io, pre) # put in pre chars + print_matrix_row(io, rowmatrix,alignL,1,1:length(alignL),sep) # left part of range + print(io, hdots) # horizontal ellipsis + print_matrix_row(io, rowmatrix,alignR,1,length(rowmatrix)-length(alignR)+1:length(rowmatrix),sep) # right part of range + print(io, post) # post chars + end +end + logspace(start::Real, stop::Real, n::Integer=50) = 10.^linspace(start, stop, n) ## interface implementations diff --git a/base/replutil.jl b/base/replutil.jl index 6e4412aaa5256..f666512c5d8a8 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -13,15 +13,24 @@ function writemime(io::IO, ::MIME"text/plain", f::Function) end end +# writemime for ranges, e.g. +# 3-element UnitRange{Int64,Int} +# 1,2,3 +# or for more elements than fit on screen: +# 1.0,2.0,3.0,…,6.0,7.0,8.0 +function writemime(io::IO, ::MIME"text/plain", r::Range) + print(io, summary(r)) + if !isempty(r) + println(io, ":") + with_output_limit(()->print_range(io, r)) + end +end + function writemime(io::IO, ::MIME"text/plain", v::AbstractVector) - if isa(v, Range) - show(io, v) - else - print(io, summary(v)) - if !isempty(v) - println(io, ":") - with_output_limit(()->print_matrix(io, v)) - end + print(io, summary(v)) + if !isempty(v) + println(io, ":") + with_output_limit(()->print_matrix(io, v)) end end diff --git a/base/show.jl b/base/show.jl index 9d25110fb7fd5..a318cae52e1af 100644 --- a/base/show.jl +++ b/base/show.jl @@ -972,14 +972,21 @@ dump(io::IO, x::DataType) = dump(io, x, 5, "") dump(io::IO, x::TypeVar, n::Int, indent) = println(io, x.name) +""" +`alignment(X)` returns a tuple (left,right) showing how many characters are +needed on either side of an alignment feature such as a decimal point. +""" alignment(x::Any) = (0, length(sprint(showcompact_lim, x))) alignment(x::Number) = (length(sprint(showcompact_lim, x)), 0) +"`alignment(42)` yields (2,0)" alignment(x::Integer) = (length(sprint(showcompact_lim, x)), 0) +"`alignment(4.23)` yields (1,3) for `4` and `.23`" function alignment(x::Real) m = match(r"^(.*?)((?:[\.eE].*)?)$", sprint(showcompact_lim, x)) m === nothing ? (length(sprint(showcompact_lim, x)), 0) : (length(m.captures[1]), length(m.captures[2])) end +"`alignment(1 + 10im)` yields (3,5) for `1 +` and `_10im` (plus sign on left, space on right)" function alignment(x::Complex) m = match(r"^(.*[\+\-])(.*)$", sprint(showcompact_lim, x)) m === nothing ? (length(sprint(showcompact_lim, x)), 0) : @@ -994,26 +1001,37 @@ end const undef_ref_str = "#undef" const undef_ref_alignment = (3,3) +""" +`alignment(X, rows, cols, cols_if_complete, cols_otherwise, sep)` returns the +alignment for specified parts of array `X`, returning the (left,right) info. +It will look in X's `rows`, `cols` (both lists of indices) +and figure out what's needed to be fully aligned, for example looking all +the way down a column and finding out the maximum size of each element. +Parameter `sep::Integer` is number of spaces to put between elements. +`cols_if_complete` and `cols_otherwise` indicate screen width to use. +Alignment is reported as a vector of (left,right) tuples, one for each +column going across the screen. +""" function alignment( X::AbstractVecOrMat, rows::AbstractVector, cols::AbstractVector, cols_if_complete::Integer, cols_otherwise::Integer, sep::Integer ) a = Tuple{Int, Int}[] - for j in cols + for j in cols # need to go down each column one at a time l = r = 0 - for i in rows + for i in rows # plumb down and see what largest element sizes are if isassigned(X,i,j) aij = alignment(X[i,j]) else aij = undef_ref_alignment end - l = max(l, aij[1]) - r = max(r, aij[2]) + l = max(l, aij[1]) # left characters + r = max(r, aij[2]) # right characters end - push!(a, (l, r)) + push!(a, (l, r)) # one tuple per column of X, pruned to screen width if length(a) > 1 && sum(map(sum,a)) + sep*length(a) >= cols_if_complete - pop!(a) + pop!(a) # remove this latest tuple if we're already beyond screen width break end end @@ -1025,6 +1043,13 @@ function alignment( return a end +""" +`print_matrix_row(io, X, A, i, cols, sep)` produces the aligned output for +a single matrix row X[i, cols] where the desired list of columns is given. +The corresponding alignment A is used, and the separation between elements +is specified as string sep. +`print_matrix_row` will also respect compact output for elements. +""" function print_matrix_row(io::IO, X::AbstractVecOrMat, A::Vector, i::Integer, cols::AbstractVector, sep::AbstractString @@ -1039,13 +1064,18 @@ function print_matrix_row(io::IO, a = undef_ref_alignment sx = undef_ref_str end - l = repeat(" ", A[k][1]-a[1]) + l = repeat(" ", A[k][1]-a[1]) # pad on left and right as needed r = repeat(" ", A[k][2]-a[2]) print(io, l, sx, r) if k < length(A); print(io, sep); end end end +""" +`print_matrix_vdots` is used to show a series of vertical ellipsis instead +of a bunch of rows for long matrices. Not only is the string vdots shown +but it also repeated every M elements if desired. +""" function print_matrix_vdots(io::IO, vdots::AbstractString, A::Vector, sep::AbstractString, M::Integer, m::Integer ) @@ -1161,15 +1191,20 @@ function print_matrix(io::IO, X::AbstractVecOrMat, end end -summary(x) = string(typeof(x)) +"`summary(x)` a string of type information, e.g. `Int64`" +summary(x) = string(typeof(x)) # e.g. Int64 +# sizes such as 0-dimensional, 4-dimensional, 2x3 dims2string(d) = length(d) == 0 ? "0-dimensional" : length(d) == 1 ? "$(d[1])-element" : join(map(string,d), 'x') +# anything array-like gets summarized e.g. 10-element Array{Int64,1} +"`summary(A)` for array is a string of size and type info, e.g. `10-element Array{Int64,1}`" summary(a::AbstractArray) = string(dims2string(size(a)), " ", typeof(a)) +# n-dimensional arrays function show_nd(io::IO, a::AbstractArray, limit, print_matrix, label_slices) if isempty(a) return @@ -1216,6 +1251,9 @@ end # for internal use in showing arrays. _limit_output = false +""" +`print_matrix_repr(io, X)` prints matrix X with opening and closing square brackets. +""" function print_matrix_repr(io, X::AbstractArray) compact, prefix = array_eltype_show_how(X) prefix *= "[" @@ -1282,7 +1320,7 @@ end show(io::IO, X::AbstractArray) = showarray(io, X, header=_limit_output, repr=!_limit_output) -function with_output_limit(thk, lim=true) +function with_output_limit(thk, lim=true) # thk is usually show() global _limit_output last = _limit_output _limit_output = lim diff --git a/test/ranges.jl b/test/ranges.jl index a4a53ed49cc02..a88f6c02ede77 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -544,6 +544,17 @@ for x in r end @test i == 7 +# stringmime/writemime should display the range or linspace nicely +# to test print_range in range.jl +replstr(x) = stringmime("text/plain", x) +@test replstr(1:4) == "4-element UnitRange{$Int}:\n 1,2,3,4" +@test replstr(linspace(1,5,7)) == "7-element LinSpace{Float64}:\n 1.0,1.66667,2.33333,3.0,3.66667,4.33333,5.0" +@test replstr(0:100.) == "101-element FloatRange{Float64}:\n 0.0,1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,…,94.0,95.0,96.0,97.0,98.0,99.0,100.0" +# next is to test a very large range, which should be fast because print_range +# only examines spacing of the left and right edges of the range, sufficient +# to cover the designated screen size. +@test replstr(0:10^9) == "1000000001-element UnitRange{$Int}:\n 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,…,999999998,999999999,1000000000" + # Issue 11049 and related @test promote(linspace(0f0, 1f0, 3), linspace(0., 5., 2)) === (linspace(0., 1., 3), linspace(0., 5., 2)) From 0eb4e441ffc8ddc9f7883c9afb01cd6f813686fe Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Mon, 19 Oct 2015 03:56:00 +0000 Subject: [PATCH 0598/1938] Add llvm 3.7 checksums --- deps/checksums/llvm-3.7.0.src.tar.xz/md5 | 1 + deps/checksums/llvm-3.7.0.src.tar.xz/sha512 | 1 + 2 files changed, 2 insertions(+) create mode 100644 deps/checksums/llvm-3.7.0.src.tar.xz/md5 create mode 100644 deps/checksums/llvm-3.7.0.src.tar.xz/sha512 diff --git a/deps/checksums/llvm-3.7.0.src.tar.xz/md5 b/deps/checksums/llvm-3.7.0.src.tar.xz/md5 new file mode 100644 index 0000000000000..af48058221db3 --- /dev/null +++ b/deps/checksums/llvm-3.7.0.src.tar.xz/md5 @@ -0,0 +1 @@ +b98b9495e5655a672d6cb83e1a180f8e diff --git a/deps/checksums/llvm-3.7.0.src.tar.xz/sha512 b/deps/checksums/llvm-3.7.0.src.tar.xz/sha512 new file mode 100644 index 0000000000000..668a1537f3948 --- /dev/null +++ b/deps/checksums/llvm-3.7.0.src.tar.xz/sha512 @@ -0,0 +1 @@ +fe3bfefd3d49423831ad12b89f944ac9f469aaaaafdea1dd94fc7fb6d5cc9fbe66b6463796aeaabfc145ae474ec3ad68da41c4a011e8aa4bc650f1835af75388 From e5342592f93fde4550aa71e8d951432d048b938c Mon Sep 17 00:00:00 2001 From: tan Date: Sat, 17 Oct 2015 21:48:17 +0530 Subject: [PATCH 0599/1938] adding scala micro benchmarks All benchmark code are in `perf.scala`. Uses `breeze` (https://github.com/scalanlp/breeze) for linear algebra. Requires `sbt` to be installed; `sbt` pulls in all other required dependencies. --- test/perf/micro/Makefile | 6 +- test/perf/micro/scala/.gitignore | 17 ++ test/perf/micro/scala/build.sbt | 18 ++ .../micro/scala/src/main/scala/perf.scala | 241 ++++++++++++++++++ 4 files changed, 281 insertions(+), 1 deletion(-) create mode 100644 test/perf/micro/scala/.gitignore create mode 100644 test/perf/micro/scala/build.sbt create mode 100644 test/perf/micro/scala/src/main/scala/perf.scala diff --git a/test/perf/micro/Makefile b/test/perf/micro/Makefile index bb4919427e36d..9e1c4dd14d591 100644 --- a/test/perf/micro/Makefile +++ b/test/perf/micro/Makefile @@ -116,6 +116,9 @@ benchmarks/lua.csv: perf.lua benchmarks/java.csv: java/src/main/java/PerfBLAS.java cd java; sh setup.sh; for t in 1 2 3 4 5; do mvn -q exec:java; done >../$@ +benchmarks/scala.csv: scala/src/main/scala/perf.scala scala/build.sbt + cd scala; for t in 1 2 3 4 5; do sbt run; done >../$@ + BENCHMARKS = \ benchmarks/c.csv \ benchmarks/fortran.csv \ @@ -128,7 +131,8 @@ BENCHMARKS = \ benchmarks/lua.csv \ benchmarks/javascript.csv \ benchmarks/mathematica.csv \ - benchmarks/java.csv + benchmarks/java.csv \ + benchmarks/scala.csv benchmarks.csv: bin/collect.pl $(BENCHMARKS) @$(call PRINT_PERL, $^ >$@) diff --git a/test/perf/micro/scala/.gitignore b/test/perf/micro/scala/.gitignore new file mode 100644 index 0000000000000..c58d83b318909 --- /dev/null +++ b/test/perf/micro/scala/.gitignore @@ -0,0 +1,17 @@ +*.class +*.log + +# sbt specific +.cache +.history +.lib/ +dist/* +target/ +lib_managed/ +src_managed/ +project/boot/ +project/plugins/project/ + +# Scala-IDE specific +.scala_dependencies +.worksheet diff --git a/test/perf/micro/scala/build.sbt b/test/perf/micro/scala/build.sbt new file mode 100644 index 0000000000000..c98322a85653e --- /dev/null +++ b/test/perf/micro/scala/build.sbt @@ -0,0 +1,18 @@ +// This file is a part of Julia. License is MIT: http://julialang.org/license + +libraryDependencies ++= Seq( + "org.scalanlp" %% "breeze" % "0.10", + "org.scalanlp" %% "breeze-natives" % "0.10" +) + +resolvers ++= Seq( + "Sonatype Releases" at "https://oss.sonatype.org/content/repositories/releases/" +) + +scalaVersion := "2.11.1" + +showSuccess := false + +onLoadMessage := "" + +logLevel := Level.Warn diff --git a/test/perf/micro/scala/src/main/scala/perf.scala b/test/perf/micro/scala/src/main/scala/perf.scala new file mode 100644 index 0000000000000..35613791e39c9 --- /dev/null +++ b/test/perf/micro/scala/src/main/scala/perf.scala @@ -0,0 +1,241 @@ +// This file is a part of Julia. License is MIT: http://julialang.org/license + +import scala.util._ +import java.io._ +import breeze.linalg._ +import breeze.numerics._ +import breeze.stats._ +import breeze.math._ +//import com.github.fommil.netlib.{BLAS} + +object PerfBreeze { + final val NITER = 5 + + // print results appropriately. times are in milliseconds + def print_perf(name:String, t:Double) = { + printf("scala,%s,%.9f\n", name, t/1e6) + } + + // time fib + def fib(n:Int):Int = { + if (n < 2) n else fib(n-1) + fib(n-2) + } + + def time_fib() = { + assert(fib(20) == 6765) + var tmin = Long.MaxValue + var f = 0 + + for(i <- 1 to NITER) { + val t1 = System.nanoTime() + for(j <- 1 to 1000) { + f += fib(20) + } + val t = System.nanoTime() - t1 + if(t < tmin) tmin = t + } + + tmin / 1000.0 + } + + // time parseint + def time_parseint() = { + val generator = scala.util.Random + var tmin = Long.MaxValue + + for(i <- 1 to NITER) { + var rand:Int = 0 + var rands:String = "0" + var parsed:Int = 0 + val t1 = System.nanoTime() + for(j <- 1 to 1000) { + rand = generator.nextInt() + rands = rand.toString + parsed = rands.toInt + } + val t = System.nanoTime() - t1 + assert(rand == parsed) + if(t < tmin) tmin = t + } + tmin / 1000.0 + } + + // time mandel + def mandel(zin:Complex):Int = { + val c = zin + var z = zin + val maxiter = 80 + for(n <- 0 to maxiter) { + if(z.abs > 2) return n + z = c + (z * z) + } + maxiter + } + + def mandelperf() = { + var mandel_sum = 0 + for(re <- -20 to 5) { + for(im <- -10 to 10) { + val m = mandel(re/10.0 + i * im/10.0) + mandel_sum += m + } + } + mandel_sum + } + + def time_mandel() = { + var mandel_sum = 0 + var mandel_sum2 = 0 + var tmin = Long.MaxValue + + for(i <- 1 to NITER) { + val t1 = System.nanoTime() + for(j <- 1 to 100) { + mandel_sum = mandelperf() + mandel_sum2 += mandel_sum + } + val t = System.nanoTime() - t1 + if(t < tmin) tmin = t + } + assert(mandel_sum == 14791) + assert(mandel_sum2 == mandel_sum * 100 * NITER) + tmin / 100.0 + } + + // time quicksort + def time_quicksort() = { + var tmin = Long.MaxValue + + for(i <- 1 to NITER) { + val t1 = System.nanoTime() + for(j <- 1 to 1000) { + val A = randomInt(5000, (1, Int.MaxValue)) + Sorting.quickSort(A.data) + } + val t = System.nanoTime() - t1 + if(t < tmin) tmin = t + } + tmin / 1000.0 + } + + // time pisum + def pisum() = { + var sum = 0.0 + for(j <- 1 to 500) { + sum = 0.0 + for(k <- 1 to 10000) { + sum += 1.0/(k*k) + } + } + sum + } + + def time_pisum() = { + var tmin = Long.MaxValue + var pi = 0:Double + for(i <- 1 to NITER) { + val t1 = System.nanoTime() + pi = pisum() + val t = System.nanoTime() - t1 + if(t < tmin) tmin = t + assert(abs(pi-1.644834071848065) < 1e-12) + } + tmin + } + + // time printfd + def printfd(n:Int) = { + var stream = None: Option[PrintStream] + try { + stream = Some(new PrintStream(new BufferedOutputStream(new FileOutputStream("/dev/null")))) + val valid_stream = stream.get + for (i <- 1 to n) { + valid_stream.printf(i + " " + i) + } + } catch { + case e: Exception => println("Exception caught: " + e) + } finally { + if(stream.isDefined) stream.get.close() + } + } + + def time_printfd() = { + var tmin = Long.MaxValue + for(i <- 1 to NITER) { + val t1 = System.nanoTime() + printfd(100000) + val t = System.nanoTime() - t1 + if(t < tmin) tmin = t + } + tmin + } + + // random matrix statistics + def randmatstat(t:Int):(Double,Double) = { + val n = 5 + val v = DenseVector.zeros[Double](t) + val w = DenseVector.zeros[Double](t) + + val g = breeze.stats.distributions.Gaussian(0, 1) + for(i <- 0 to t-1) { + val a = DenseMatrix.rand(n, n, g) + val b = DenseMatrix.rand(n, n, g) + val c = DenseMatrix.rand(n, n, g) + val d = DenseMatrix.rand(n, n, g) + val P = DenseMatrix.horzcat(a, b, c, d) + val Q = DenseMatrix.vertcat(DenseMatrix.horzcat(a, b), DenseMatrix.horzcat(c, d)) + val V = P.t * P + val W = Q.t * Q + + v(i) = trace(V * V * V * V) + w(i) = trace(W * W * W * W) + } + (stddev(v)/mean(v), stddev(w)/mean(w)) + } + + def time_randmatstat() = { + var tmin = Long.MaxValue + for(i <- 1 to NITER) { + val t1 = System.nanoTime() + val (s1, s2) = randmatstat(1000) + val t = System.nanoTime() - t1 + assert(0.5 < s1 && s1 < 1.0 && 0.5 < s2 && s2 < 1.0) + + if(t < tmin) tmin = t + } + tmin + } + + // random matrix multiplication + def randmatmul(t:Int):DenseMatrix[Double] = { + val m1 = randomDouble((t, t)) + val m2 = randomDouble((t, t)) + m1 * m2 + } + + def time_randmatmul() = { + var tmin = Long.MaxValue + for(i <- 1 to NITER) { + val t1 = System.nanoTime() + val m = randmatmul(1000) + val t = System.nanoTime() - t1 + assert(0 <= m(0,0)) + + if(t < tmin) tmin = t + } + tmin + } + + + def main(args: Array[String]) = { + //println("BLAS: " + BLAS.getInstance().getClass().getName()) + print_perf("fib", time_fib()) + print_perf("parse_int", time_parseint()) + print_perf("mandel", time_mandel()) + print_perf("quicksort", time_quicksort()) + print_perf("pi_sum", time_pisum()) + print_perf("rand_mat_stat", time_randmatstat()) + print_perf("rand_mat_mul", time_randmatmul()) + print_perf("printfd", time_printfd()) + } +} From 38606b271660a98cd66ff2fb0ac60ab4b6713491 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Mon, 19 Oct 2015 18:18:12 +0530 Subject: [PATCH 0600/1938] perf/micro/perf.c: allocate mandelbrot array and sum only as check This is what all the other benchmarks do, and therefore what the C version should do as a fairer comparison. Java and C, etc. should follow the same pattern as well. --- test/perf/micro/Makefile | 2 ++ test/perf/micro/perf.c | 47 +++++++++++++++++++++++++--------------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/test/perf/micro/Makefile b/test/perf/micro/Makefile index bb4919427e36d..1788fd63984b7 100644 --- a/test/perf/micro/Makefile +++ b/test/perf/micro/Makefile @@ -140,3 +140,5 @@ clean: @rm -rf perf.h bin/perf* bin/fperf* benchmarks/*.csv benchmarks.csv mods *~ octave-core perf.log gopath/* .PHONY: all perf clean + +.PRECIOUS: bin/perf0 bin/perf1 bin/perf2 bin/perf3 diff --git a/test/perf/micro/perf.c b/test/perf/micro/perf.c index 7d764fccd13ce..7bfa4199e67a7 100644 --- a/test/perf/micro/perf.c +++ b/test/perf/micro/perf.c @@ -59,7 +59,7 @@ double *matmul_aat(int n, double *b) { int mandel(double complex z) { int maxiter = 80; double complex c = z; - for (int n=0; n 2.0) { return n; } @@ -68,19 +68,14 @@ int mandel(double complex z) { return maxiter; } -int mandelperf() { - /* The initialization on the next two lines is deliberately written to - * prevent gcc from optimizing away the entire loop. - * (First observed in gcc 4.9.2) */ - static volatile int mandel_sum_init = 0; - int mandel_sum = mandel_sum_init; - for (int re=-20; re<=5; re+=1) { - for (int im=-10; im<=10; im+=1) { - int m = mandel(re/10.0+I*im/10.0); - mandel_sum += m; +int *mandelperf() { + int *M = (int*) malloc(21*26*sizeof(int)); + for (int i = 0; i < 21; i++) { + for (int j = 0; j < 26; j++) { + M[26*i + j] = mandel((j-20)/10.0 + ((i-10)/10.0)*I); } } - return mandel_sum; + return M; } void quicksort(double *a, int lo, int hi) { @@ -293,20 +288,38 @@ int main() { // print_perf("AtA", tmin); // mandel - int mandel_sum; - int mandel_sum2 = 0; + /* The initialization on the next line is deliberately volatile to + * prevent gcc from optimizing away the entire loop. + * (First observed in gcc 4.9.2) + */ + static volatile int mandel_sum_init = 0; + int mandel_sum2 = mandel_sum_init; tmin = INFINITY; for (int i=0; i Date: Mon, 19 Oct 2015 11:10:59 -0400 Subject: [PATCH 0601/1938] Revert "Improve parse_eval_all line number reporting" --- src/interpreter.c | 2 -- src/toplevel.c | 35 +++++++++-------------------------- test/cmdlineargs.jl | 2 +- test/core.jl | 2 +- 4 files changed, 11 insertions(+), 30 deletions(-) diff --git a/src/interpreter.c b/src/interpreter.c index c6f09d441e0e9..509cd6e50b319 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -15,7 +15,6 @@ extern "C" { #endif extern int jl_lineno; -extern const char *jl_filename; static jl_value_t *eval(jl_value_t *e, jl_value_t **locals, size_t nl, size_t ngensym); static jl_value_t *eval_body(jl_array_t *stmts, jl_value_t **locals, size_t nl, size_t ngensym, @@ -160,7 +159,6 @@ static jl_value_t *eval(jl_value_t *e, jl_value_t **locals, size_t nl, size_t ng } if (jl_is_linenode(e)) { jl_lineno = jl_linenode_line(e); - jl_filename = jl_linenode_file(e)->name; } if (jl_is_newvarnode(e)) { jl_value_t *var = jl_fieldref(e,0); diff --git a/src/toplevel.c b/src/toplevel.c index 7a899fb282ee1..1e5f34aef1214 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -555,12 +555,11 @@ jl_value_t *jl_parse_eval_all(const char *fname, size_t len) { //jl_printf(JL_STDERR, "***** loading %s\n", fname); int last_lineno = jl_lineno; - int top_lineno = -1; const char *last_filename = jl_filename; jl_lineno = 0; jl_filename = fname; - jl_value_t *fn=NULL, *ln=NULL, *form=NULL, *nest_exc=NULL, *result=jl_nothing; - JL_GC_PUSH5(&fn, &ln, &form, &result, &nest_exc); + jl_value_t *fn=NULL, *ln=NULL, *form=NULL, *result=jl_nothing; + JL_GC_PUSH4(&fn, &ln, &form, &result); JL_TRY { // handle syntax error while (1) { @@ -575,38 +574,22 @@ jl_value_t *jl_parse_eval_all(const char *fname, size_t len) jl_interpret_toplevel_expr(form); } } - top_lineno = jl_lineno; // jl_parse_next sets lineno. result = jl_toplevel_eval_flex(form, 1); } } JL_CATCH { jl_stop_parsing(); - + fn = jl_pchar_to_string(fname, len); + ln = jl_box_long(jl_lineno); + jl_lineno = last_lineno; + jl_filename = last_filename; if (jl_loaderror_type == NULL) { - // reset line and filename before throwing - jl_lineno = last_lineno; - jl_filename = last_filename; jl_rethrow(); } - - fn = jl_pchar_to_string(jl_filename, strlen(jl_filename)); - ln = jl_box_long(jl_lineno); - nest_exc = jl_new_struct(jl_loaderror_type, fn, ln, - jl_exception_in_transit); - - if ((strcmp(jl_filename, fname) == 0) && - jl_lineno == top_lineno) { - jl_lineno = last_lineno; - jl_filename = last_filename; - jl_rethrow_other(nest_exc); + else { + jl_rethrow_other(jl_new_struct(jl_loaderror_type, fn, ln, + jl_exception_in_transit)); } - - jl_lineno = last_lineno; - jl_filename = last_filename; - fn = jl_pchar_to_string(fname, len); - ln = jl_box_long(top_lineno); - jl_rethrow_other(jl_new_struct(jl_loaderror_type, fn, ln, - nest_exc)); } jl_stop_parsing(); jl_lineno = last_lineno; diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index e5192e6edeb3e..9a8e9396aed1a 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -173,7 +173,7 @@ let exename = `$(joinpath(JULIA_HOME, Base.julia_exename())) --precompiled=yes` wait(proc) close(out.in) @test success(proc) - @test readchomp(out) == "WARNING: Foo.Deprecated is deprecated.\n likely near none:5" + @test readchomp(out) == "WARNING: Foo.Deprecated is deprecated.\n likely near no file:5" end @unix_only let out = Pipe(), diff --git a/test/core.jl b/test/core.jl index 9bb929f9e89f7..666e87321d482 100644 --- a/test/core.jl +++ b/test/core.jl @@ -704,7 +704,7 @@ let didthrow = false catch ex @test isa(ex, LoadError) - @test isa(ex.error.error, InitError) + @test isa(ex.error, InitError) true end @test didthrow From 8f46fe65432561554b3f55702326673721c2e2e3 Mon Sep 17 00:00:00 2001 From: tan Date: Mon, 19 Oct 2015 23:17:45 +0530 Subject: [PATCH 0602/1938] make scala micro benchmark fns similar to others - parseint benchmark now converts to hex string like others - mandelbrot test now returns an array of the results, which is summed only in the first iteration and validated against a checksum ref #13673 --- .../micro/scala/src/main/scala/perf.scala | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/test/perf/micro/scala/src/main/scala/perf.scala b/test/perf/micro/scala/src/main/scala/perf.scala index 35613791e39c9..840eaa7778d9d 100644 --- a/test/perf/micro/scala/src/main/scala/perf.scala +++ b/test/perf/micro/scala/src/main/scala/perf.scala @@ -50,8 +50,8 @@ object PerfBreeze { val t1 = System.nanoTime() for(j <- 1 to 1000) { rand = generator.nextInt() - rands = rand.toString - parsed = rands.toInt + rands = if(rand < 0) "-" + abs(rand).toHexString else rand.toHexString + parsed = Integer.parseInt(rands, 16) } val t = System.nanoTime() - t1 assert(rand == parsed) @@ -73,14 +73,7 @@ object PerfBreeze { } def mandelperf() = { - var mandel_sum = 0 - for(re <- -20 to 5) { - for(im <- -10 to 10) { - val m = mandel(re/10.0 + i * im/10.0) - mandel_sum += m - } - } - mandel_sum + for(re <- -20 to 5; im <- -10 to 10) yield mandel(re/10.0 + i * im/10.0) } def time_mandel() = { @@ -91,14 +84,17 @@ object PerfBreeze { for(i <- 1 to NITER) { val t1 = System.nanoTime() for(j <- 1 to 100) { - mandel_sum = mandelperf() - mandel_sum2 += mandel_sum + val mandel_arr = mandelperf() + if(j == 1) { + mandel_sum = sum(mandel_arr) + mandel_sum2 += mandel_sum + } } val t = System.nanoTime() - t1 if(t < tmin) tmin = t } assert(mandel_sum == 14791) - assert(mandel_sum2 == mandel_sum * 100 * NITER) + assert(mandel_sum2 == mandel_sum * NITER) tmin / 100.0 } From 4282ba2c47e7077203fde476a528e34c039280dd Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 19 Oct 2015 14:24:42 -0400 Subject: [PATCH 0603/1938] use llvm-config directly to always determine the real llvm-version this reverts f5da3f8aba86d14e32577ea9dc6e7f0993c2a748 --- Make.inc | 3 +-- base/Makefile | 2 +- base/atomics.jl | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Make.inc b/Make.inc index eb1f25c44c0f9..5e982162c906f 100644 --- a/Make.inc +++ b/Make.inc @@ -598,7 +598,7 @@ JCFLAGS += -fsigned-char LLVM_ASSERTIONS=1 #LLVM_FLAGS+="--with-float=hard --with-abi=aapcs-vfp" -LLVM_VER:=3.6.1 +LLVM_VER:=3.7.0 override USE_BLAS64:=0 override OPENBLAS_DYNAMIC_ARCH:=0 @@ -650,7 +650,6 @@ endif # LLVM_CONFIG undefined ifeq ($(USE_SYSTEM_LLVM), 1) JCPPFLAGS+=-DSYSTEM_LLVM -LLVM_VER := $(shell $(LLVM_CONFIG) --version) endif # SYSTEM_LLVM ifeq ($(BUILD_OS),$(OS)) diff --git a/base/Makefile b/base/Makefile index a33a55b739847..0975ce0476458 100644 --- a/base/Makefile +++ b/base/Makefile @@ -59,7 +59,7 @@ else endif @echo "const libfftw_name = \"$(LIBFFTWNAME)\"" >> $@ @echo "const libfftwf_name = \"$(LIBFFTWFNAME)\"" >> $@ - @echo "const libllvm_version = \"$(LLVM_VER)\"" >> $@ + @echo "const libllvm_version = \"$$($(LLVM_CONFIG_HOST) --version)\"" >> $@ @echo "const VERSION_STRING = \"$(JULIA_VERSION)\"" >> $@ @echo "const TAGGED_RELEASE_BANNER = \"$(TAGGED_RELEASE_BANNER)\"" >> $@ @echo "const SYSCONFDIR = \"$(sysconfdir_rel)\"" >> $@ diff --git a/base/atomics.jl b/base/atomics.jl index d69a5931fdded..b211923a5dd6d 100644 --- a/base/atomics.jl +++ b/base/atomics.jl @@ -30,7 +30,7 @@ unsafe_convert{T}(::Type{Ptr{T}}, x::Atomic{T}) = convert(Ptr{T}, pointer_from_o setindex!{T}(x::Atomic{T}, v) = setindex!(x, convert(T, v)) for (typ, lt) in atomicintsmap - rt = (Base.libllvm_version == "svn" || VersionNumber(Base.libllvm_version) >= v"3.6") ? "$lt, $lt*" : "$lt*" + rt = VersionNumber(Base.libllvm_version) >= v"3.6" ? "$lt, $lt*" : "$lt*" @eval getindex(x::Atomic{$typ}) = llvmcall($""" %rv = load atomic volatile $rt %0 monotonic, align $WORD_SIZE From 7d2f7b50b35838cb06e31ccd9c369d15ecf00efb Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 19 Oct 2015 17:28:33 -0400 Subject: [PATCH 0604/1938] Adjust to LLVM 3.8 API change --- src/codegen.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 3417f2dc08f0a..fdb8999b7ffae 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4469,7 +4469,9 @@ static Function *emit_function(jl_lambda_info_t *lam) continue; ditypes.push_back(julia_type_to_di(jl_tparam(lam->specTypes,i),ctx.dbuilder,false)); } -#ifdef LLVM36 +#ifdef LLVM38 + subrty = ctx.dbuilder->createSubroutineType(ctx.dbuilder->getOrCreateTypeArray(ditypes)); +#elif defined(LLVM36) subrty = ctx.dbuilder->createSubroutineType(topfile,ctx.dbuilder->getOrCreateTypeArray(ditypes)); #else subrty = ctx.dbuilder->createSubroutineType(topfile,ctx.dbuilder->getOrCreateArray(ditypes)); @@ -5296,7 +5298,10 @@ static void init_julia_llvm_env(Module *m) // Third argument (length(argv)) diargs.push_back(julia_type_to_di((jl_value_t*)jl_int32_type,&dbuilder,false)); -#ifdef LLVM36 +#ifdef LLVM38 + jl_di_func_sig = dbuilder.createSubroutineType( + dbuilder.getOrCreateTypeArray(diargs)); +#elif defined(LLVM36) jl_di_func_sig = dbuilder.createSubroutineType(julia_h, dbuilder.getOrCreateTypeArray(diargs)); #else From b82ab6af1b1873070e09deab4d7913917091ad33 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 19 Oct 2015 19:20:47 -0400 Subject: [PATCH 0605/1938] Split out sanitize LDFLAGS from SANITIZE_OPTS Sometimes both CXXFLAGS and LDFLAGS are passed, in which case we need to avoid duplicating the `-mllvm` option, which may only be specified once. --- Make.inc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Make.inc b/Make.inc index e929c21632763..ca5341f7a79bf 100644 --- a/Make.inc +++ b/Make.inc @@ -453,12 +453,14 @@ endif ifeq ($(SANITIZE),1) ifeq ($(SANITIZE_MEMORY),1) SANITIZE_OPTS := -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer +SANITIZE_LDFLAGS := $(SANITIZE_OPTS) else SANITIZE_OPTS := -fsanitize=address -mllvm -asan-stack=0 +SANITIZE_LDFLAGS := -fsanitize=address endif JCXXFLAGS += $(SANITIZE_OPTS) JCFLAGS += $(SANITIZE_OPTS) -LDFLAGS += $(SANITIZE_OPTS) +LDFLAGS += $(SANITIZE_LDFLAGS) DEPS_CFLAGS += $(SANITIZE_OPTS) DEPS_CXXFLAGS += $(SANITIZE_OPTS) endif From 316da80ff3c147fa2e5a305a5a0832b46ab14761 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 15 Oct 2015 21:34:01 -0600 Subject: [PATCH 0606/1938] Export `jl_strtod_c`/`jl_strtof_c` from src/ --- src/builtins.c | 6 +++--- src/flisp/read.c | 4 ++-- src/support/strtod.c | 10 +++++----- src/support/strtod.h | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/builtins.c b/src/builtins.c index 6fd90d141a513..f1a3d6dd538e5 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -831,7 +831,7 @@ DLLEXPORT jl_nullable_float64_t jl_try_substrtod(char *str, size_t offset, size_ bstr = newstr; pend = bstr+len; } - double out = strtod_c(bstr, &p); + double out = jl_strtod_c(bstr, &p); if (errno==ERANGE && (out==0 || out==HUGE_VAL || out==-HUGE_VAL)) { err = 1; @@ -884,9 +884,9 @@ DLLEXPORT jl_nullable_float32_t jl_try_substrtof(char *str, size_t offset, size_ pend = bstr+len; } #if defined(_OS_WINDOWS_) && !defined(_COMPILER_MINGW_) - float out = (float)strtod_c(bstr, &p); + float out = (float)jl_strtod_c(bstr, &p); #else - float out = strtof_c(bstr, &p); + float out = jl_strtof_c(bstr, &p); #endif if (errno==ERANGE && (out==0 || out==HUGE_VALF || out==-HUGE_VALF)) { diff --git a/src/flisp/read.c b/src/flisp/read.c index 1764f7da5701c..c8240590c0c23 100644 --- a/src/flisp/read.c +++ b/src/flisp/read.c @@ -39,7 +39,7 @@ int isnumtok_base(char *tok, value_t *pval, int base) return 0; if (!((tok[0]=='0' && tok[1]=='x') || (base >= 15)) && strpbrk(tok, ".eEpP")) { - d = strtod_c(tok, &end); + d = jl_strtod_c(tok, &end); if (*end == '\0') { if (pval) *pval = mk_double(d); return 1; @@ -55,7 +55,7 @@ int isnumtok_base(char *tok, value_t *pval, int base) // hexadecimal float literals else if (((tok[0]=='0' && tok[1]=='x') || (base == 16)) && strpbrk(tok, "pP")) { - d = strtod_c(tok, &end); + d = jl_strtod_c(tok, &end); if (*end == '\0') { if (pval) *pval = mk_double(d); return 1; diff --git a/src/support/strtod.c b/src/support/strtod.c index 2d6b1f20a728e..38fd353018103 100644 --- a/src/support/strtod.c +++ b/src/support/strtod.c @@ -28,12 +28,12 @@ locale_t get_c_locale() return c_locale; } -double strtod_c(const char *nptr, char **endptr) +DLLEXPORT double jl_strtod_c(const char *nptr, char **endptr) { return strtod_l(nptr, endptr, get_c_locale()); } -float strtof_c(const char *nptr, char **endptr) +DLLEXPORT float jl_strtof_c(const char *nptr, char **endptr) { return strtof_l(nptr, endptr, get_c_locale()); } @@ -98,7 +98,7 @@ double parse_inf_or_nan(const char *p, char **endptr) } -double strtod_c(const char *nptr, char **endptr) +DLLEXPORT double jl_strtod_c(const char *nptr, char **endptr) { char *fail_pos; double val; @@ -285,9 +285,9 @@ double strtod_c(const char *nptr, char **endptr) } -float strtof_c(const char *nptr, char **endptr) +DLLEXPORT float jl_strtof_c(const char *nptr, char **endptr) { - return (float) strtod_c(nptr, endptr); + return (float) jl_strtod_c(nptr, endptr); } #endif diff --git a/src/support/strtod.h b/src/support/strtod.h index 1758d59b1506b..8dff983d5414b 100644 --- a/src/support/strtod.h +++ b/src/support/strtod.h @@ -7,8 +7,8 @@ extern "C" { #endif -double strtod_c(const char *nptr, char **endptr); -float strtof_c(const char *nptr, char **endptr); +DLLEXPORT double jl_strtod_c(const char *nptr, char **endptr); +DLLEXPORT float jl_strtof_c(const char *nptr, char **endptr); #ifdef __cplusplus } From 68442e2d7876ffdc42ca1a5e2106954aef4c8c70 Mon Sep 17 00:00:00 2001 From: Hessam Mehr Date: Fri, 16 Oct 2015 12:44:39 -0400 Subject: [PATCH 0607/1938] Fixing #13559 with more limited impact. Performance should be identical to baseline. Fixed infinite loop in previous fix for #13559. Do not throw EOFError if buffer already contains n bytes. #13559 --- src/sys.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/sys.c b/src/sys.c index 002be1a597af6..c76abbdef1e07 100644 --- a/src/sys.c +++ b/src/sys.c @@ -310,9 +310,13 @@ static void NORETURN throw_eof_error(void) DLLEXPORT uint64_t jl_ios_get_nbyte_int(ios_t *s, const size_t n) { assert(n <= 8); - size_t ret = ios_readprep(s, n); - if (ret < n) - throw_eof_error(); + size_t space, ret; + do { + space = s->size - s->bpos; + ret = ios_readprep(s, n); + if (space == ret && ret < n) + throw_eof_error(); + } while(ret < n); uint64_t x = 0; uint8_t *buf = (uint8_t*)&s->buf[s->bpos]; if (n == 8) { From 0a79189a1ea287586056dbafe55859bbe0a7ac97 Mon Sep 17 00:00:00 2001 From: Hessam Mehr Date: Fri, 16 Oct 2015 23:09:52 -0400 Subject: [PATCH 0608/1938] Unit test for #13559. Uses the qualified path to launch julia --- test/file.jl | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/file.jl b/test/file.jl index ed3a385229bbd..d359eb7ace627 100644 --- a/test/file.jl +++ b/test/file.jl @@ -922,3 +922,25 @@ test_12992() test2_12992() test2_12992() test2_12992() + +# issue 13559 + +function test_13559() + fn = tempname() + run(`mkfifo $fn`) + # use subprocess to write 127 bytes to FIFO + writer_cmds = "x=open(\"$fn\", \"w\"); for i=1:127 write(x,0xaa); flush(x); sleep(0.1) end; close(x); quit()" + open(`$(Base.julia_cmd()) -e $writer_cmds`) + #quickly read FIFO, draining it and blocking but not failing with EOFError yet + r = open(fn, "r") + # 15 proper reads + for i=1:15 + @test read(r, Int64) == -6148914691236517206 + end + # last read should throw EOFError when FIFO closes, since there are only 7 bytes available. + @test_throws EOFError read(r, Int64) + close(r) + rm(fn) +end + +@unix_only test_13559() From 9ef294921425b0dc6e730294e8ebfef421561ad7 Mon Sep 17 00:00:00 2001 From: Hessam Mehr Date: Sat, 17 Oct 2015 20:45:24 -0700 Subject: [PATCH 0609/1938] Cross-platform test coverage for n-byte reads and throwing EOF on insufficient bytes read. --- test/file.jl | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/test/file.jl b/test/file.jl index d359eb7ace627..bd08ef3215f49 100644 --- a/test/file.jl +++ b/test/file.jl @@ -942,5 +942,51 @@ function test_13559() close(r) rm(fn) end - @unix_only test_13559() + +function test_read_nbyte() + fn = tempname() + # Write one byte. One byte read should work once + # but 2-byte read should throw EOFError. + f = open(fn, "w+") do f + write(f, 0x55) + flush(f) + seek(f, 0) + @test read(f, UInt8) == 0x55 + @test_throws EOFError read(f, UInt8) + seek(f, 0) + @test_throws EOFError read(f, UInt16) + end + # Write 2 more bytes. Now 2-byte read should work once + # but 4-byte read should fail with EOFError. + open(fn, "a+") do f + write(f, 0x4444) + flush(f) + seek(f, 0) + @test read(f, UInt16) == 0x4455 + @test_throws EOFError read(f, UInt16) + seek(f,0) + @test_throws EOFError read(f, UInt32) + end + # Write 4 more bytes. Now 4-byte read should work once + # but 8-byte read should fail with EOFError. + open(fn, "a+") do f + write(f, 0x33333333) + flush(f) + seek(f, 0) + @test read(f, UInt32) == 0x33444455 + @test_throws EOFError read(f, UInt32) + seek(f,0) + @test_throws EOFError read(f, UInt64) + end + # Writing one more byte should allow an 8-byte + # read to proceed. + open(fn, "a+") do f + write(f, 0x22) + flush(f) + seek(f, 0) + @test read(f, UInt64) == 0x2233333333444455 + end + rm(fn) +end +test_read_nbyte() From 84a1ca65c6a6e59cf193fee6daf6fc603d0b463a Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Tue, 20 Oct 2015 03:30:52 -0700 Subject: [PATCH 0610/1938] Switch from openSUSE 13.1 to 13.2 for winrpm packages --- Makefile | 4 ++-- README.windows.md | 6 +++--- contrib/windows/get_toolchain.sh | 2 +- contrib/windows/winrpm.sh | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index e58e3fc574e1f..5f0003336fc93 100644 --- a/Makefile +++ b/Makefile @@ -596,7 +596,7 @@ ifneq (,$(filter $(ARCH), i386 i486 i586 i686)) cd $(JULIAHOME)/dist-extras && \ $(JLDOWNLOAD) http://downloads.sourceforge.net/sevenzip/7z920.exe && \ 7z x -y 7z920.exe 7z.exe 7z.dll && \ - ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win32/openSUSE_13.1 \ + ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win32/openSUSE_13.2 \ "mingw32-libgfortran3 mingw32-libquadmath0 mingw32-libstdc++6 mingw32-libgcc_s_sjlj1 mingw32-libssp0 mingw32-libexpat1 mingw32-zlib1" && \ cp usr/i686-w64-mingw32/sys-root/mingw/bin/*.dll . && \ $(JLDOWNLOAD) PortableGit.7z https://github.com/git-for-windows/git/releases/download/v2.6.1.windows.1/PortableGit-2.6.1-32-bit.7z.exe @@ -606,7 +606,7 @@ else ifeq ($(ARCH),x86_64) 7z x -y 7z920-x64.msi _7z.exe _7z.dll && \ mv _7z.dll 7z.dll && \ mv _7z.exe 7z.exe && \ - ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win64/openSUSE_13.1 \ + ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win64/openSUSE_13.2 \ "mingw64-libgfortran3 mingw64-libquadmath0 mingw64-libstdc++6 mingw64-libgcc_s_seh1 mingw64-libssp0 mingw64-libexpat1 mingw64-zlib1" && \ cp usr/x86_64-w64-mingw32/sys-root/mingw/bin/*.dll . && \ $(JLDOWNLOAD) PortableGit.7z https://github.com/git-for-windows/git/releases/download/v2.6.1.windows.1/PortableGit-2.6.1-64-bit.7z.exe diff --git a/README.windows.md b/README.windows.md index 5a4576cd8fc29..e176fde0ae5fb 100644 --- a/README.windows.md +++ b/README.windows.md @@ -192,7 +192,7 @@ Julia can be also compiled from source in [Cygwin](http://www.cygwin.com), using If you prefer to cross-compile, the following steps should get you started. -For maximum compatibility with packages that use [WinRPM.jl](https://github.com/JuliaLang/WinRPM.jl) for binary dependencies on Windows, it is recommended that you use OpenSUSE 13.1 for cross-compiling a Windows build of Julia. If you use a different Linux distribution or OS X, install [Vagrant](http://www.vagrantup.com/downloads) and use the following `Vagrantfile`: +For maximum compatibility with packages that use [WinRPM.jl](https://github.com/JuliaLang/WinRPM.jl) for binary dependencies on Windows, it is recommended that you use OpenSUSE 13.2 for cross-compiling a Windows build of Julia. If you use a different Linux distribution or OS X, install [Vagrant](http://www.vagrantup.com/downloads) and use the following `Vagrantfile`: ``` # Vagrantfile for MinGW-w64 cross-compilation of Julia @@ -201,7 +201,7 @@ $script = <