From 67fe4d978df46396b3703a4aec9cf004fa851a6e Mon Sep 17 00:00:00 2001 From: reminisce Date: Wed, 9 Oct 2019 11:21:00 -0700 Subject: [PATCH 1/5] Revert "Numpy det and slogdet operators (#15861)" This reverts commit 295fc145510ac73f296a7581649d7ad371d85cd5. --- python/mxnet/_numpy_op_doc.py | 122 ---------- src/operator/tensor/la_op.cc | 2 - src/operator/tensor/la_op.cu | 2 - src/operator/tensor/la_op.h | 7 +- tests/python/unittest/test_numpy_op.py | 305 +++++++++---------------- 5 files changed, 107 insertions(+), 331 deletions(-) diff --git a/python/mxnet/_numpy_op_doc.py b/python/mxnet/_numpy_op_doc.py index bb9b8314f2aa..6e2f5fa15919 100644 --- a/python/mxnet/_numpy_op_doc.py +++ b/python/mxnet/_numpy_op_doc.py @@ -20,128 +20,6 @@ """Doc placeholder for numpy ops with prefix _np.""" -def _np__linalg_det(a): - """ - det(a) - - Compute the determinant of an array. - - Parameters - ---------- - a : (..., M, M) ndarray - Input array to compute determinants for. - - Returns - ------- - det : (...) ndarray - Determinant of `a`. - - See Also - -------- - slogdet : Another way to represent the determinant, more suitable - for large matrices where underflow/overflow may occur. - - Notes - ----- - - Broadcasting rules apply, see the `numpy.linalg` documentation for - details. - - The determinant is computed via LU factorization using the LAPACK - routine z/dgetrf. - - Examples - -------- - The determinant of a 2-D array [[a, b], [c, d]] is ad - bc: - - >>> a = np.array([[1, 2], [3, 4]]) - >>> np.linalg.det(a) - -2.0 - - Computing determinants for a stack of matrices: - - >>> a = np.array([ [[1, 2], [3, 4]], [[1, 2], [2, 1]], [[1, 3], [3, 1]] ]) - >>> a.shape - (3, 2, 2) - >>> np.linalg.det(a) - array([-2., -3., -8.]) - """ - pass - - -def _np__linalg_slogdet(a): - """ - slogdet(a) - - Compute the sign and (natural) logarithm of the determinant of an array. - - If an array has a very small or very large determinant, then a call to - `det` may overflow or underflow. This routine is more robust against such - issues, because it computes the logarithm of the determinant rather than - the determinant itself. - - Parameters - ---------- - a : (..., M, M) ndarray - Input array, has to be a square 2-D array. - - Returns - ------- - sign : (...) ndarray - A number representing the sign of the determinant. For a real matrix, - this is 1, 0, or -1. - logdet : (...) array_like - The natural log of the absolute value of the determinant. - - If the determinant is zero, then `sign` will be 0 and `logdet` will be - -Inf. In all cases, the determinant is equal to ``sign * np.exp(logdet)``. - - See Also - -------- - det - - Notes - ----- - - Broadcasting rules apply, see the `numpy.linalg` documentation for - details. - - The determinant is computed via LU factorization using the LAPACK - routine z/dgetrf. - - - Examples - -------- - The determinant of a 2-D array ``[[a, b], [c, d]]`` is ``ad - bc``: - - >>> a = np.array([[1, 2], [3, 4]]) - >>> (sign, logdet) = np.linalg.slogdet(a) - >>> (sign, logdet) - (-1., 0.69314718055994529) - >>> sign * np.exp(logdet) - -2.0 - - Computing log-determinants for a stack of matrices: - - >>> a = np.array([ [[1, 2], [3, 4]], [[1, 2], [2, 1]], [[1, 3], [3, 1]] ]) - >>> a.shape - (3, 2, 2) - >>> sign, logdet = np.linalg.slogdet(a) - >>> (sign, logdet) - (array([-1., -1., -1.]), array([ 0.69314718, 1.09861229, 2.07944154])) - >>> sign * np.exp(logdet) - array([-2., -3., -8.]) - - This routine succeeds where ordinary `det` does not: - - >>> np.linalg.det(np.eye(500) * 0.1) - 0.0 - >>> np.linalg.slogdet(np.eye(500) * 0.1) - (1., -1151.2925464970228) - """ - pass - - def _np_ones_like(a): """ Return an array of ones with the same shape and type as a given array. diff --git a/src/operator/tensor/la_op.cc b/src/operator/tensor/la_op.cc index 24968eca3f06..ce7d1d5de692 100644 --- a/src/operator/tensor/la_op.cc +++ b/src/operator/tensor/la_op.cc @@ -941,7 +941,6 @@ NNVM_REGISTER_OP(_backward_linalg_inverse) NNVM_REGISTER_OP(_linalg_det) .add_alias("linalg_det") -.add_alias("_np__linalg_det") .describe(R"code(Compute the determinant of a matrix. Input is a tensor *A* of dimension *n >= 2*. @@ -992,7 +991,6 @@ NNVM_REGISTER_OP(_backward_linalg_det) .set_attr("FCompute", LaOpDetBackward); NNVM_REGISTER_OP(_linalg_slogdet) -.add_alias("_np__linalg_slogdet") .add_alias("linalg_slogdet") .describe(R"code(Compute the sign and log of the determinant of a matrix. Input is a tensor *A* of dimension *n >= 2*. diff --git a/src/operator/tensor/la_op.cu b/src/operator/tensor/la_op.cu index ec622f0b22a2..68c33180e3d5 100644 --- a/src/operator/tensor/la_op.cu +++ b/src/operator/tensor/la_op.cu @@ -100,14 +100,12 @@ NNVM_REGISTER_OP(_backward_linalg_inverse) .set_attr("FCompute", LaOpBackward); NNVM_REGISTER_OP(_linalg_det) -.add_alias("_np__linalg_det") .set_attr("FCompute", LaOpDetForward); NNVM_REGISTER_OP(_backward_linalg_det) .set_attr("FCompute", LaOpDetBackward); NNVM_REGISTER_OP(_linalg_slogdet) -.add_alias("_np__linalg_slogdet") .set_attr("FCompute", LaOpDetForward); NNVM_REGISTER_OP(_backward_linalg_slogdet) diff --git a/src/operator/tensor/la_op.h b/src/operator/tensor/la_op.h index bb56dc52b594..e024693e3819 100644 --- a/src/operator/tensor/la_op.h +++ b/src/operator/tensor/la_op.h @@ -26,7 +26,6 @@ #define MXNET_OPERATOR_TENSOR_LA_OP_H_ #include -#include #include #include #include "../mshadow_op.h" @@ -429,11 +428,7 @@ inline bool DetShape(const nnvm::NodeAttrs& attrs, CHECK_EQ(in[ndim-2], in[ndim-1]) << "Input A's last two dimension must be equal"; mxnet::TShape out; if (ndim == 2) { - if (Imperative::Get()->is_np_shape()) { - out = mxnet::TShape(0, 1); - } else { - out = mxnet::TShape(1, 1); - } + out = mxnet::TShape(1, 1); } else { out = mxnet::TShape(in.begin(), in.end() - 2); } diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py index 45bcc517a641..7068bf20d897 100644 --- a/tests/python/unittest/test_numpy_op.py +++ b/tests/python/unittest/test_numpy_op.py @@ -29,6 +29,8 @@ from common import assertRaises, with_seed import random import scipy.stats as ss +from mxnet.test_utils import verify_generator, gen_buckets_probs_with_ppf, retry +from mxnet.runtime import Features from mxnet.numpy_op_signature import _get_builtin_op from mxnet.test_utils import verify_generator, gen_buckets_probs_with_ppf, has_tvm_ops import platform @@ -139,8 +141,8 @@ def tensordot_backward(a, b, axes=2): test_tensordot = TestTensordot(axes) if hybridize: test_tensordot.hybridize() - a = rand_ndarray(shape=a_shape, dtype=dtype).as_np_ndarray() - b = rand_ndarray(shape=b_shape, dtype=dtype).as_np_ndarray() + a = rand_ndarray(shape = a_shape, dtype = dtype).as_np_ndarray() + b = rand_ndarray(shape = b_shape, dtype = dtype).as_np_ndarray() a.attach_grad() b.attach_grad() @@ -165,7 +167,7 @@ def tensordot_backward(a, b, axes=2): b_sym = mx.sym.Variable("b").as_np_ndarray() mx_sym = mx.sym.np.tensordot(a_sym, b_sym, axes).as_nd_ndarray() check_numeric_gradient(mx_sym, [a.as_nd_ndarray(), b.as_nd_ndarray()], - rtol=1e-1, atol=1e-1, dtype=dtype) + rtol=1e-1, atol=1e-1, dtype = dtype) @with_seed() @@ -217,96 +219,6 @@ def test_np_dot(): assert False -@with_seed() -@use_np -def test_np_linalg_det(): - class TestDet(HybridBlock): - def __init__(self): - super(TestDet, self).__init__() - - def hybrid_forward(self, F, a): - return F.np.linalg.det(a) - - # test non zero size input - tensor_shapes = [ - (5, 5), - (3, 3, 3), - (2, 2, 2, 2, 2), - (1, 1) - ] - - for hybridize in [True, False]: - for shape in tensor_shapes: - for dtype in [_np.float32, _np.float64]: - a_shape = (1,) + shape - test_det = TestDet() - if hybridize: - test_det.hybridize() - a = rand_ndarray(shape=a_shape, dtype=dtype).as_np_ndarray() - a.attach_grad() - - np_out = _np.linalg.det(a.asnumpy()) - with mx.autograd.record(): - mx_out = test_det(a) - assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-1, atol=1e-1) - mx_out.backward() - - # Test imperative once again - mx_out = np.linalg.det(a) - np_out = _np.linalg.det(a.asnumpy()) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-1, atol=1e-1, use_broadcast=False) - - # test numeric gradient - a_sym = mx.sym.Variable("a").as_np_ndarray() - mx_sym = mx.sym.np.linalg.det(a_sym).as_nd_ndarray() - check_numeric_gradient(mx_sym, [a.as_nd_ndarray()], - rtol=1e-1, atol=1e-1, dtype=dtype) - - -@with_seed() -@use_np -def test_np_linalg_slogdet(): - class TestSlogdet(HybridBlock): - def __init__(self): - super(TestSlogdet, self).__init__() - - def hybrid_forward(self, F, a): - return F.np.linalg.slogdet(a) - - # test non zero size input - tensor_shapes = [ - (5, 5), - (3, 3, 3), - (2, 2, 2, 2, 2), - (1, 1) - ] - - for hybridize in [True, False]: - for a_shape in tensor_shapes: - for dtype in [_np.float32, _np.float64]: - test_slogdet = TestSlogdet() - if hybridize: - test_slogdet.hybridize() - a = rand_ndarray(shape=a_shape, dtype=dtype).as_np_ndarray() - a.attach_grad() - - np_out = _np.linalg.slogdet(a.asnumpy()) - with mx.autograd.record(): - mx_out = test_slogdet(a) - assert mx_out[0].shape == np_out[0].shape - assert mx_out[1].shape == np_out[1].shape - assert_almost_equal(mx_out[0].asnumpy(), np_out[0], rtol=1e-1, atol=1e-1, use_broadcast=False) - assert_almost_equal(mx_out[1].asnumpy(), np_out[1], rtol=1e-1, atol=1e-1, use_broadcast=False) - mx_out[1].backward() - - # Test imperative once again - mx_out = np.linalg.slogdet(a) - np_out = _np.linalg.slogdet(a.asnumpy()) - assert_almost_equal(mx_out[0].asnumpy(), np_out[0], rtol=1e-1, atol=1e-1, use_broadcast=False) - assert_almost_equal(mx_out[1].asnumpy(), np_out[1], rtol=1e-1, atol=1e-1, use_broadcast=False) - - @with_seed() @use_np def test_np_ldexp(): @@ -320,24 +232,25 @@ def hybrid_forward(self, F, x1, x2): def _np_ldexp(x1, x2): return x1 * _np.power(2.0, x2) - def dldx(x1, x2): - out = _np_ldexp(x1, x2) - grad_a = _np.broadcast_to(_np.power(2.0, x2), out.shape) - grad_b = _np.broadcast_to(out * _np.log(2.0), out.shape) - grad_a = collapse_sum_like(grad_a, x1.shape) - grad_b = collapse_sum_like(grad_b, x2.shape) + def dldx(x1, x2): + grad_a = _np.power(2.0, x2) + grad_b = _np_ldexp(x1, x2) * _np.log(2.0) + if len(x1) == 1: + grad_a = _np.sum(grad_a) + if len(x2) == 1: + grad_b = _np.sum(grad_b) return [grad_a, grad_b] - shapes = [ + shapes = [ ((3, 1), (3, 1)), ((3, 1, 2), (3, 1, 2)), - ((1, ), (1, )), + ((1, ),(1, )), ((1, ), (2, )), ((3, ), (1, )), ((3, 0), (3, 0)), # zero-size shape ((0, 1), (0, 1)), # zero-size shape ((2, 0, 2), (2, 0, 2)), # zero-size shape - ] + ] for hybridize in [True, False]: for shape1, shape2 in shapes: @@ -345,7 +258,7 @@ def dldx(x1, x2): test_ldexp = TestLdexp() if hybridize: test_ldexp.hybridize() - x1 = rand_ndarray(shape=shape1, dtype=dtype).as_np_ndarray() + x1 = rand_ndarray(shape=shape1, dtype=dtype).as_np_ndarray() x1.attach_grad() x2 = rand_ndarray(shape=shape2, dtype=dtype).as_np_ndarray() x2.attach_grad() @@ -354,17 +267,17 @@ def dldx(x1, x2): with mx.autograd.record(): mx_out = test_ldexp(x1, x2) assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-1, atol=1e-1, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-1, atol=1e-1) mx_out.backward() np_backward = dldx(x1.asnumpy(), x2.asnumpy()) - assert_almost_equal(x1.grad.asnumpy(), np_backward[0], atol=1e-1, rtol=1e-1, use_broadcast=False) - assert_almost_equal(x2.grad.asnumpy(), np_backward[1], atol=1e-1, rtol=1e-1, use_broadcast=False) + assert_almost_equal(x1.grad.asnumpy(), np_backward[0], atol=1e-1, rtol=1e-1) + assert_almost_equal(x2.grad.asnumpy(), np_backward[1], atol=1e-1, rtol=1e-1) # Test imperative once again mx_out = np.ldexp(x1, x2) np_out = _np_ldexp(x1.asnumpy(), x2.asnumpy()) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-1, atol=1e-1, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-1, atol=1e-1) @with_seed() @@ -398,16 +311,16 @@ def vdot_backward(a, b): with mx.autograd.record(): mx_out = test_vdot(a, b) assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol = 1e-3, atol = 1e-5) mx_out.backward() np_backward = vdot_backward(a.asnumpy(), b.asnumpy()) - assert_almost_equal(a.grad.asnumpy(), np_backward[0], rtol=1e-2, atol=1e-2, use_broadcast=False) - assert_almost_equal(b.grad.asnumpy(), np_backward[1], rtol=1e-2, atol=1e-2, use_broadcast=False) + assert_almost_equal(a.grad.asnumpy(), np_backward[0], rtol = 1e-2, atol=1e-2) + assert_almost_equal(b.grad.asnumpy(), np_backward[1], rtol = 1e-2, atol=1e-2) # Test imperative once again mx_out = np.vdot(a, b) np_out = _np.vdot(a.asnumpy(), b.asnumpy()) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) # test numeric gradient if len(shape) > 0 and _np.prod(shape) > 0: @@ -495,16 +408,16 @@ def inner_backward(a, b): with mx.autograd.record(): mx_out = test_inner(a, b) assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol = 1e-3, atol = 1e-5) mx_out.backward() np_backward = inner_backward(a.asnumpy(), b.asnumpy()) - assert_almost_equal(a.grad.asnumpy(), np_backward[0], rtol=1e-2, atol=1e-2, use_broadcast=False) - assert_almost_equal(b.grad.asnumpy(), np_backward[1], rtol=1e-2, atol=1e-2, use_broadcast=False) + assert_almost_equal(a.grad.asnumpy(), np_backward[0], rtol = 1e-2, atol=1e-2) + assert_almost_equal(b.grad.asnumpy(), np_backward[1], rtol = 1e-2, atol=1e-2) # Test imperative once again mx_out = np.inner(a, b) np_out = _np.inner(a.asnumpy(), b.asnumpy()) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) # test numeric gradient a_sym = mx.sym.Variable("a").as_np_ndarray() @@ -546,13 +459,13 @@ def hybrid_forward(self, F, a, b): with mx.autograd.record(): mx_out = test_outer(a, b) assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) mx_out.backward() # Test imperative once again mx_out = np.outer(a, b) np_out = _np.outer(a.asnumpy(), b.asnumpy()) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) # test numeric gradient a_sym = mx.sym.Variable("a").as_np_ndarray() @@ -714,7 +627,7 @@ def _test_np_exception(func, shape, dim): y = test_gluon(x) assert y.shape == expected_ret.shape assert_almost_equal(y.asnumpy(), expected_ret, rtol=1e-3 if itype == 'float16' else 1e-3, - atol=1e-5 if itype == 'float16' else 1e-5, use_broadcast=False) + atol=1e-5 if itype == 'float16' else 1e-5) y.backward() # only check the gradient with hardcoded input if is_int(itype): @@ -728,7 +641,7 @@ def _test_np_exception(func, shape, dim): else: mx_out = np.min(x, axis=axis, keepdims=keepdims) np_out = _np.amin(x.asnumpy(), axis=axis, keepdims=keepdims) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) # test zero and zero dim shapes = [(), (0), (2, 0), (0, 2, 1)] @@ -787,7 +700,7 @@ def is_int(dtype): y = test_mean(x) assert y.shape == expected_ret.shape assert_almost_equal(y.asnumpy(), expected_ret, rtol=1e-3 if dtype == 'float16' else 1e-3, - atol=1e-5 if dtype == 'float16' else 1e-5, use_broadcast=False) + atol=1e-5 if dtype == 'float16' else 1e-5) y.backward() N = x.size / y.size @@ -803,7 +716,7 @@ def is_int(dtype): # test imperative mx_out = np.mean(x, axis=axis, dtype=dtype, keepdims=keepdims) np_out = _np.mean(x.asnumpy(), axis=axis, dtype=acc_type[itype], keepdims=keepdims).astype(dtype) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) @with_seed() @@ -895,17 +808,16 @@ def test_np_linspace(): mx_ret = np.linspace(config, endpoint=endpoint, retstep=retstep, dtype=dtype) np_ret = _np.linspace(config, endpoint=endpoint, retstep=retstep, dtype=dtype) if retstep: - assert_almost_equal(mx_ret[0].asnumpy(), np_ret[0], atol=1e-3, rtol=1e-5, use_broadcast=False) + assert_almost_equal(mx_ret[0].asnumpy(), np_ret[0], atol=1e-3, rtol=1e-5) same(mx_ret[1], np_ret[1]) else: - assert_almost_equal(mx_ret.asnumpy(), np_ret, atol=1e-3, rtol=1e-5, use_broadcast=False) + assert_almost_equal(mx_ret.asnumpy(), np_ret, atol=1e-3, rtol=1e-5) # check for exception input for config in exception_configs: assertRaises(MXNetError, np.linspace, *config) # check linspace equivalent to arange for test_index in range(1000): - assert_almost_equal(mx.np.linspace(0, test_index, test_index + 1).asnumpy(), _np.arange(test_index + 1), - use_broadcast=False) + assert_almost_equal(mx.np.linspace(0, test_index, test_index + 1).asnumpy(), _np.arange(test_index + 1)) class TestLinspace(HybridBlock): def __init__(self, start, stop, num=50, endpoint=None, retstep=False, dtype=None, axis=0): @@ -938,7 +850,7 @@ def hybrid_forward(self, F, x): if hybridize: net.hybridize() mx_out = net(x) - assert_almost_equal(mx_out.asnumpy(), np_out, atol=1e-3, rtol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, atol=1e-3, rtol=1e-5) @with_seed() @@ -1429,12 +1341,11 @@ def hybrid_forward(self, F, a, *args, **kwargs): with mx.autograd.record(): y = mx_func(mx_test_data) assert y.shape == np_out.shape - assert_almost_equal(y.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(y.asnumpy(), np_out, rtol=1e-3, atol=1e-5) if ref_grad: y.backward() - assert_almost_equal(mx_test_data.grad.asnumpy(), ref_grad(np_test_data), rtol=1e-1, atol=1e-2, - equal_nan=True, use_broadcast=False) + assert_almost_equal(mx_test_data.grad.asnumpy(), ref_grad(np_test_data), rtol=1e-1, atol=1e-2, equal_nan=True) funcs = { 'absolute' : (lambda x: -1. * (x < 0) + (x > 0), -1.0, 1.0), @@ -1509,14 +1420,14 @@ def hybrid_forward(self, F, a): with mx.autograd.record(): mx_out = test_relu(x) assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) mx_out.backward() np_backward = np_relu_grad(x.asnumpy()) - assert_almost_equal(x.grad.asnumpy(), np_backward, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(x.grad.asnumpy(), np_backward, rtol=1e-3, atol=1e-5) mx_out = npx.relu(x) np_out = np_relu(x.asnumpy()) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) @with_seed() @@ -1546,14 +1457,14 @@ def hybrid_forward(self, F, a): with mx.autograd.record(): mx_out = test_sigmoid(x) assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) mx_out.backward() np_backward = np_sigmoid_grad(np_out) - assert_almost_equal(x.grad.asnumpy(), np_backward, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(x.grad.asnumpy(), np_backward, rtol=1e-3, atol=1e-5) mx_out = npx.sigmoid(x) np_out = np_sigmoid(x.asnumpy()) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) @with_seed() @@ -1659,17 +1570,17 @@ def get_indices(axis_size): y = test_split(a) assert len(y) == len(expected_ret) for mx_out, np_out in zip(y, expected_ret): - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) mx.autograd.backward(y) - assert_almost_equal(a.grad.asnumpy(), _np.ones(a.shape), rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(a.grad.asnumpy(), _np.ones(a.shape), rtol=1e-3, atol=1e-5) # test imperative mx_outs = np.split(a, indices_or_sections=indices_or_sections, axis=axis) np_outs = _np.split(a.asnumpy(), indices_or_sections=indices_or_sections, axis=axis) for mx_out, np_out in zip(mx_outs, np_outs): - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) @with_seed() @@ -1709,19 +1620,19 @@ def get_new_shape(shape, axis): y = test_concat(a, b, c, d) assert y.shape == expected_ret.shape - assert_almost_equal(y.asnumpy(), expected_ret, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(y.asnumpy(), expected_ret, rtol=1e-3, atol=1e-5) y.backward() - assert_almost_equal(a.grad.asnumpy(), _np.ones(a.shape), rtol=1e-3, atol=1e-5, use_broadcast=False) - assert_almost_equal(b.grad.asnumpy(), _np.ones(b.shape), rtol=1e-3, atol=1e-5, use_broadcast=False) - assert_almost_equal(c.grad.asnumpy(), _np.ones(c.shape), rtol=1e-3, atol=1e-5, use_broadcast=False) - assert_almost_equal(d.grad.asnumpy(), _np.ones(d.shape), rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(a.grad.asnumpy(), _np.ones(a.shape), rtol=1e-3, atol=1e-5) + assert_almost_equal(b.grad.asnumpy(), _np.ones(b.shape), rtol=1e-3, atol=1e-5) + assert_almost_equal(c.grad.asnumpy(), _np.ones(c.shape), rtol=1e-3, atol=1e-5) + assert_almost_equal(d.grad.asnumpy(), _np.ones(d.shape), rtol=1e-3, atol=1e-5) # test imperative mx_out = np.concatenate([a, b, c, d], axis=axis) np_out = _np.concatenate([a.asnumpy(), b.asnumpy(), c.asnumpy(), d.asnumpy()], axis=axis) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) @with_seed() @@ -1764,10 +1675,10 @@ def hybrid_forward(self, F, a, *args): y.backward() - assert_almost_equal(mx_a.grad.asnumpy(), _np.ones(shape), rtol=1e-3, atol=1e-5, use_broadcast=False) - assert_almost_equal(mx_b.grad.asnumpy(), _np.ones(shape), rtol=1e-3, atol=1e-5, use_broadcast=False) - assert_almost_equal(mx_c.grad.asnumpy(), _np.ones(shape), rtol=1e-3, atol=1e-5, use_broadcast=False) - assert_almost_equal(mx_d.grad.asnumpy(), _np.ones(shape), rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_a.grad.asnumpy(), _np.ones(shape), rtol=1e-3, atol=1e-5) + assert_almost_equal(mx_b.grad.asnumpy(), _np.ones(shape), rtol=1e-3, atol=1e-5) + assert_almost_equal(mx_c.grad.asnumpy(), _np.ones(shape), rtol=1e-3, atol=1e-5) + assert_almost_equal(mx_d.grad.asnumpy(), _np.ones(shape), rtol=1e-3, atol=1e-5) np_out = _np.stack([np_a, np_b, np_c, np_d], axis=axis) mx_out = np.stack([mx_a, mx_b, mx_c, mx_d], axis=axis) @@ -1797,14 +1708,14 @@ def hybrid_forward(self, F, a): with mx.autograd.record(): mx_out = test_ravel(x) assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) mx_out.backward() np_backward = _np.ones(shape) - assert_almost_equal(x.grad.asnumpy(), np_backward, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(x.grad.asnumpy(), np_backward, rtol=1e-3, atol=1e-5) mx_out = np.ravel(x) np_out = _np.ravel(x.asnumpy()) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) @with_seed() @@ -2117,15 +2028,15 @@ def hybrid_forward(self, F, a): with mx.autograd.record(): mx_out = test_cumsum(x) assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) mx_out.backward() np_backward = np_cumsum_backward(_np.ones(np_out.shape, dtype=otype), axis=axis, dtype=otype).reshape(x.shape) - assert_almost_equal(x.grad.asnumpy(), np_backward, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(x.grad.asnumpy(), np_backward, rtol=1e-3, atol=1e-5) mx_out = np.cumsum(x, axis=axis, dtype=otype) np_out = _np.cumsum(x.asnumpy(), axis=axis, dtype=otype) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) @with_seed() @@ -2156,7 +2067,7 @@ def test_sample_with_replacement(sampler, num_classes, shape, weight=None): expected_density = (weight.asnumpy() if weight is not None else _np.array([1 / num_classes] * num_classes)) # test almost equal - assert_almost_equal(generated_density, expected_density, rtol=1e-1, atol=1e-1, use_broadcast=False) + assert_almost_equal(generated_density, expected_density, rtol=1e-1, atol=1e-1) # test shape assert (samples.shape == shape) @@ -2173,7 +2084,7 @@ def test_sample_without_replacement(sampler, num_classes, shape, num_trials, wei out = sampler(num_classes, 1, replace=False, p=weight).item() bins[out] += 1 bins /= num_trials - assert_almost_equal(bins, expected_freq, rtol=1e-1, atol=1e-1, use_broadcast=False) + assert_almost_equal(bins, expected_freq, rtol=1e-1, atol=1e-1) def test_indexing_mode(sampler, set_size, samples_size, replace, weight=None): a = np.arange(set_size) @@ -2334,7 +2245,7 @@ def hybrid_forward(self, F, x): net.hybridize() mx_ret = net(a) np_ret = _np.linalg.norm(a.asnumpy(), ord=ord, axis=axis, keepdims=keepdims) - assert_almost_equal(mx_ret.asnumpy(), np_ret, atol=1e-5, rtol=1e-4, use_broadcast=False) + assert_almost_equal(mx_ret.asnumpy(), np_ret, atol=1e-5, rtol=1e-4) @with_seed() @@ -2393,18 +2304,18 @@ def get_grad_right(a1, a2): with mx.autograd.record(): mx_out = test_copysign(a1, a2) assert mx_out.shape == expected_np.shape - assert_almost_equal(mx_out.asnumpy(), expected_np, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), expected_np, rtol=rtol, atol=atol) # Test gradient mx_out.backward() a1_grad, a2_grad = get_grad(a1_np, a2_np) - assert_almost_equal(a1.grad.asnumpy(), a1_grad, rtol=rtol, atol=atol, use_broadcast=False) - assert_almost_equal(a2.grad.asnumpy(), a2_grad, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(a1.grad.asnumpy(), a1_grad, rtol=rtol, atol=atol) + assert_almost_equal(a2.grad.asnumpy(), a2_grad, rtol=rtol, atol=atol) # Test imperative once again mx_out = np.copysign(a1, a2) expected_np = _np.copysign(a1_np, a2_np) - assert_almost_equal(mx_out.asnumpy(), expected_np, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), expected_np, rtol=rtol, atol=atol) types = ['float16', 'float32', 'float64'] for x_shape in shapes: @@ -2418,12 +2329,12 @@ def get_grad_right(a1, a2): with mx.autograd.record(): mx_out = np.copysign(x, scalar) assert mx_out.shape == expected_np.shape - assert_almost_equal(mx_out.asnumpy(), expected_np, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), expected_np, rtol=rtol, atol=atol) # Test gradient mx_out.backward() x_grad = get_grad_left(x_np, scalar) - assert_almost_equal(x.grad.asnumpy(), x_grad, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(x.grad.asnumpy(), x_grad, rtol=rtol, atol=atol) # Test right x_np = _np.array(_np.random.uniform(-2.0, 2.0, x_shape), dtype=dtype) @@ -2434,12 +2345,12 @@ def get_grad_right(a1, a2): with mx.autograd.record(): mx_out = np.copysign(scalar, x) assert mx_out.shape == expected_np.shape - assert_almost_equal(mx_out.asnumpy(), expected_np, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), expected_np, rtol=rtol, atol=atol) # Test gradient mx_out.backward() x_grad = get_grad_right(scalar, x_np) - assert_almost_equal(x.grad.asnumpy(), x_grad, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(x.grad.asnumpy(), x_grad, rtol=rtol, atol=atol) @with_seed() @@ -2508,22 +2419,22 @@ def get_grad(UT, L, V): # check UT @ L @ V == A t = _np.matmul(UT * L[..., None, :], V) assert t.shape == data_np.shape - assert_almost_equal(t, data_np, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(t, data_np, rtol=rtol, atol=atol) # check UT @ U == I I = _np.matmul(UT, _np.swapaxes(UT, -2, -1)) I_np = _np.ones_like(UT) * _np.eye(shape[-2]) assert I.shape == I_np.shape - assert_almost_equal(I, I_np, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(I, I_np, rtol=rtol, atol=atol) # check U @ UT == I I = _np.matmul(_np.swapaxes(UT, -2, -1), UT) I_np = _np.ones_like(UT) * _np.eye(shape[-2]) assert I.shape == I_np.shape - assert_almost_equal(I, I_np, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(I, I_np, rtol=rtol, atol=atol) # check V @ VT == I I = _np.matmul(V, _np.swapaxes(V, -2, -1)) I_np = _np.ones_like(UT) * _np.eye(shape[-2]) assert I.shape == I_np.shape - assert_almost_equal(I, I_np, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(I, I_np, rtol=rtol, atol=atol) # check descending singular values s = [L[..., i] - L[..., i + 1] for i in range(L.shape[-1] - 1)] s = _np.array(s) @@ -2534,8 +2445,7 @@ def get_grad(UT, L, V): mx.autograd.backward(ret) if ((s > 1e-5).all() and (L.size == 0 or (L > 1e-5).all())): backward_expected = get_grad(ret[0].asnumpy(), ret[1].asnumpy(), ret[2].asnumpy()) - assert_almost_equal(data.grad.asnumpy(), backward_expected, rtol=rtol, atol=atol, - use_broadcast=False) + assert_almost_equal(data.grad.asnumpy(), backward_expected, rtol=rtol, atol=atol) @with_seed() @@ -2580,18 +2490,18 @@ def g(data): with mx.autograd.record(): mx_out = test_vstack(*v) assert mx_out.shape == expected_np.shape - assert_almost_equal(mx_out.asnumpy(), expected_np, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), expected_np, rtol=rtol, atol=atol) # Test gradient mx_out.backward() for i in range(3): expected_grad = g(v_np[i]) - assert_almost_equal(v[i].grad.asnumpy(), expected_grad, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(v[i].grad.asnumpy(), expected_grad, rtol=rtol, atol=atol) # Test imperative once again mx_out = np.vstack(v) expected_np = _np.vstack(v_np) - assert_almost_equal(mx_out.asnumpy(), expected_np, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), expected_np, rtol=rtol, atol=atol) @with_seed() @@ -2707,18 +2617,15 @@ def g(data, axis1, axis2, offset): with mx.autograd.record(): out_mx = test_trace(data.as_np_ndarray()) assert out_mx.shape == expected_np.shape - assert_almost_equal(out_mx.asnumpy(), expected_np, rtol=rtol, atol=atol, - use_broadcast=False) + assert_almost_equal(out_mx.asnumpy(), expected_np, rtol=rtol, atol=atol) out_mx.backward() backward_expected = g(data_np, axis1=axis1, axis2=axis2, offset=offset) - assert_almost_equal(data.grad.asnumpy(), backward_expected, rtol=rtol, atol=atol, - use_broadcast=False) + assert_almost_equal(data.grad.asnumpy(), backward_expected, rtol=rtol, atol=atol) # Test imperative once again data = mx.nd.array(data_np, dtype=dtype) out_mx = np.trace(data.as_np_ndarray(), axis1=axis1, axis2=axis2, offset=offset) - assert_almost_equal(out_mx.asnumpy(), expected_np, rtol=rtol, atol=atol, - use_broadcast=False) + assert_almost_equal(out_mx.asnumpy(), expected_np, rtol=rtol, atol=atol) # bad params params = [ @@ -2766,11 +2673,11 @@ def hybrid_forward(self, F, x, *args, **kwargs): if hybridize: mx_func.hybridize() mx_out = mx_func(x) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) # test imperative mx_out = getattr(np, func)(M=config, dtype=dtype) np_out = np_func(M=config).astype(dtype) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) @with_seed() @@ -2802,15 +2709,15 @@ def hybrid_forward(self, F, x): with mx.autograd.record(): mx_out = test_flip(x) assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol) mx_out.backward() np_backward = _np.ones(np_out.shape) - assert_almost_equal(x.grad.asnumpy(), np_backward, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(x.grad.asnumpy(), np_backward, rtol=rtol, atol=atol) # Test imperative once again mx_out = np.flip(x, axis) np_out = _np.flip(x.asnumpy(), axis) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol) @with_seed() @@ -2838,11 +2745,11 @@ def hybrid_forward(self, F, x): np_out = _np.around(x.asnumpy(), d) mx_out = test_around(x) assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol) mx_out = np.around(x, d) np_out = _np.around(x.asnumpy(), d) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol) @with_seed() @@ -2898,18 +2805,18 @@ def dimReduce(src, des): with mx.autograd.record(): mx_out = test_arctan2(x1, x2) assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol) mx_out.backward() np_backward_1 = x21 / (x11 * x11 + x21 * x21) np_backward_2 = -1 * x11 / (x11 * x11 + x21 * x21) np_backward_1 = dimReduce(np_backward_1, x11) np_backward_2 = dimReduce(np_backward_2, x21) - assert_almost_equal(x1.grad.asnumpy(), np_backward_1, rtol=rtol, atol=atol, use_broadcast=False) - assert_almost_equal(x2.grad.asnumpy(), np_backward_2, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(x1.grad.asnumpy(), np_backward_1, rtol=rtol, atol=atol) + assert_almost_equal(x2.grad.asnumpy(), np_backward_2, rtol=rtol, atol=atol) mx_out = np.arctan2(x1, x2) np_out = _np.arctan2(x1.asnumpy(), x2.asnumpy()) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol) @with_seed() @@ -2935,13 +2842,13 @@ def hybrid_forward(self, F, x): np_out = _np.transpose(np_out) mx_out = test_nonzero(x) assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol, atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol, atol) # Test imperative once again mx_out = npx.nonzero(x) np_out = _np.nonzero(x.asnumpy()) np_out = _np.transpose(np_out) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol, atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol, atol) @with_seed() @@ -2996,18 +2903,18 @@ def dimReduce(src, des): with mx.autograd.record(): mx_out = test_hypot(x1, x2) assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol) mx_out.backward() np_backward_1 = x11 / np_out np_backward_2 = x21 / np_out np_backward_1 = dimReduce(np_backward_1, x11) np_backward_2 = dimReduce(np_backward_2, x21) - assert_almost_equal(x1.grad.asnumpy(), np_backward_1, rtol=rtol, atol=atol, use_broadcast=False) - assert_almost_equal(x2.grad.asnumpy(), np_backward_2, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(x1.grad.asnumpy(), np_backward_1, rtol=rtol, atol=atol) + assert_almost_equal(x2.grad.asnumpy(), np_backward_2, rtol=rtol, atol=atol) mx_out = np.hypot(x1, x2) np_out = _np.hypot(x1.asnumpy(), x2.asnumpy()) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol, use_broadcast=False) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol) @with_seed() @@ -3049,13 +2956,13 @@ def hybrid_forward(self, F, a): mx_out = test_unique(x) assert mx_out[0].shape == np_out[0].shape for i in range(4): - assert_almost_equal(mx_out[i].asnumpy(), np_out[i], rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out[i].asnumpy(), np_out[i], rtol=1e-3, atol=1e-5) # Test imperative once again mx_out = np.unique(x, *config[1:]) np_out = _np.unique(x.asnumpy(), *config[1:]) for i in range(4): - assert_almost_equal(mx_out[i].asnumpy(), np_out[i], rtol=1e-3, atol=1e-5, use_broadcast=False) + assert_almost_equal(mx_out[i].asnumpy(), np_out[i], rtol=1e-3, atol=1e-5) @with_seed() From c91587706e74e903c58cc47ff0a01d1ab8171fdf Mon Sep 17 00:00:00 2001 From: reminisce Date: Wed, 9 Oct 2019 11:31:16 -0700 Subject: [PATCH 2/5] Delete non-windows test --- ci/jenkins/Jenkinsfile_centos_cpu | 53 -------------- ci/jenkins/Jenkinsfile_centos_gpu | 51 ------------- ci/jenkins/Jenkinsfile_clang | 51 ------------- ci/jenkins/Jenkinsfile_edge | 56 --------------- ci/jenkins/Jenkinsfile_miscellaneous | 54 -------------- ci/jenkins/Jenkinsfile_sanity | 48 ------------- ci/jenkins/Jenkinsfile_unix_cpu | 80 --------------------- ci/jenkins/Jenkinsfile_unix_gpu | 76 -------------------- ci/jenkins/Jenkinsfile_website | 48 ------------- ci/jenkins/Jenkinsfile_website_c_docs | 52 -------------- ci/jenkins/Jenkinsfile_website_clojure_docs | 53 -------------- ci/jenkins/Jenkinsfile_website_full | 67 ----------------- ci/jenkins/Jenkinsfile_website_full_pr | 62 ---------------- ci/jenkins/Jenkinsfile_website_java_docs | 53 -------------- ci/jenkins/Jenkinsfile_website_jekyll_docs | 49 ------------- ci/jenkins/Jenkinsfile_website_julia_docs | 53 -------------- ci/jenkins/Jenkinsfile_website_mxnet_build | 48 ------------- ci/jenkins/Jenkinsfile_website_python_docs | 53 -------------- ci/jenkins/Jenkinsfile_website_r_docs | 53 -------------- ci/jenkins/Jenkinsfile_website_scala_docs | 53 -------------- 20 files changed, 1113 deletions(-) delete mode 100644 ci/jenkins/Jenkinsfile_centos_cpu delete mode 100644 ci/jenkins/Jenkinsfile_centos_gpu delete mode 100644 ci/jenkins/Jenkinsfile_clang delete mode 100644 ci/jenkins/Jenkinsfile_edge delete mode 100644 ci/jenkins/Jenkinsfile_miscellaneous delete mode 100644 ci/jenkins/Jenkinsfile_sanity delete mode 100644 ci/jenkins/Jenkinsfile_unix_cpu delete mode 100644 ci/jenkins/Jenkinsfile_unix_gpu delete mode 100644 ci/jenkins/Jenkinsfile_website delete mode 100644 ci/jenkins/Jenkinsfile_website_c_docs delete mode 100644 ci/jenkins/Jenkinsfile_website_clojure_docs delete mode 100644 ci/jenkins/Jenkinsfile_website_full delete mode 100644 ci/jenkins/Jenkinsfile_website_full_pr delete mode 100644 ci/jenkins/Jenkinsfile_website_java_docs delete mode 100644 ci/jenkins/Jenkinsfile_website_jekyll_docs delete mode 100644 ci/jenkins/Jenkinsfile_website_julia_docs delete mode 100644 ci/jenkins/Jenkinsfile_website_mxnet_build delete mode 100644 ci/jenkins/Jenkinsfile_website_python_docs delete mode 100644 ci/jenkins/Jenkinsfile_website_r_docs delete mode 100644 ci/jenkins/Jenkinsfile_website_scala_docs diff --git a/ci/jenkins/Jenkinsfile_centos_cpu b/ci/jenkins/Jenkinsfile_centos_cpu deleted file mode 100644 index a47ab3de7fb7..000000000000 --- a/ci/jenkins/Jenkinsfile_centos_cpu +++ /dev/null @@ -1,53 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 180 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_centos7_cpu(), - custom_steps.compile_centos7_cpu_mkldnn() - ]) - - utils.parallel_stage('Tests', [ - custom_steps.test_centos7_python3_cpu(), - custom_steps.test_centos7_scala_cpu() - ]) -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_centos_gpu b/ci/jenkins/Jenkinsfile_centos_gpu deleted file mode 100644 index cad77a9a7dd8..000000000000 --- a/ci/jenkins/Jenkinsfile_centos_gpu +++ /dev/null @@ -1,51 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 180 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_centos7_gpu() - ]) - - utils.parallel_stage('Tests', [ - custom_steps.test_centos7_python3_gpu() - ]) -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_clang b/ci/jenkins/Jenkinsfile_clang deleted file mode 100644 index 029c7208107b..000000000000 --- a/ci/jenkins/Jenkinsfile_clang +++ /dev/null @@ -1,51 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 180 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_clang_3_9_cpu(), - custom_steps.compile_unix_clang_6_cpu(), - custom_steps.compile_unix_clang_tidy_cpu(), - custom_steps.compile_unix_clang_3_9_mkldnn_cpu(), - custom_steps.compile_unix_clang_6_mkldnn_cpu() - ]) -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_edge b/ci/jenkins/Jenkinsfile_edge deleted file mode 100644 index 9d8e01399d7c..000000000000 --- a/ci/jenkins/Jenkinsfile_edge +++ /dev/null @@ -1,56 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 180 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_armv8_jetson_gpu(), - custom_steps.compile_armv7_cpu(), - custom_steps.compile_armv6_cpu(), - custom_steps.compile_armv8_cpu(), - custom_steps.compile_armv8_android_cpu(), - custom_steps.compile_armv7_android_cpu() - ]) - - utils.parallel_stage('Tests', [ - custom_steps.test_qemu_armv7_cpu() - ]) -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_miscellaneous b/ci/jenkins/Jenkinsfile_miscellaneous deleted file mode 100644 index dbf2a9e41c76..000000000000 --- a/ci/jenkins/Jenkinsfile_miscellaneous +++ /dev/null @@ -1,54 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 180 - - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_asan_cpu(), - custom_steps.compile_unix_amalgamation_min(), - custom_steps.compile_unix_amalgamation() - ]) - - utils.parallel_stage('Tests', [ - custom_steps.misc_asan_cpu() - ]) -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_sanity b/ci/jenkins/Jenkinsfile_sanity deleted file mode 100644 index ed4d16ec47db..000000000000 --- a/ci/jenkins/Jenkinsfile_sanity +++ /dev/null @@ -1,48 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 180 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Sanity Check', [ - custom_steps.sanity_lint(), - custom_steps.sanity_rat_license() - ]) -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu deleted file mode 100644 index 3533a123ea86..000000000000 --- a/ci/jenkins/Jenkinsfile_unix_cpu +++ /dev/null @@ -1,80 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 240 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_cpu_openblas(), - custom_steps.compile_unix_openblas_debug_cpu(), - custom_steps.compile_unix_mkl_cpu(), - custom_steps.compile_unix_mkldnn_cpu(), - custom_steps.compile_unix_mkldnn_mkl_cpu(), - custom_steps.compile_unix_int64_cpu() - ]) - - utils.parallel_stage('Tests', [ - custom_steps.test_unix_python2_cpu(), - custom_steps.test_unix_python3_cpu(), - custom_steps.test_unix_python3_debug_cpu(), - custom_steps.test_unix_python3_mkl_cpu(), - custom_steps.test_unix_python2_mkldnn_cpu(), - custom_steps.test_unix_python3_mkldnn_cpu(), - custom_steps.test_unix_python3_mkldnn_mkl_cpu(), - custom_steps.test_unix_scala_cpu(), - custom_steps.test_unix_scala_mkldnn_cpu(), - custom_steps.test_unix_clojure_cpu(), - custom_steps.test_unix_clojure_integration_cpu(), - custom_steps.test_unix_perl_cpu(), - custom_steps.test_unix_r_cpu(), - custom_steps.test_unix_r_mkldnn_cpu(), - custom_steps.test_unix_julia07_cpu(), - custom_steps.test_unix_julia10_cpu(), - custom_steps.test_unix_onnx_cpu(), - custom_steps.test_unix_cpp_cpu(), - custom_steps.test_static_scala_cpu(), - custom_steps.test_static_python_cpu(), - /* Disabled due to master build failure: - * http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/ - * https://github.com/apache/incubator-mxnet/issues/11801 - custom_steps.test_unix_distributed_kvstore_cpu() - */ - ]) -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_unix_gpu b/ci/jenkins/Jenkinsfile_unix_gpu deleted file mode 100644 index e2a089b0469b..000000000000 --- a/ci/jenkins/Jenkinsfile_unix_gpu +++ /dev/null @@ -1,76 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 180 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_mkldnn_gpu(), - custom_steps.compile_unix_mkldnn_nocudnn_gpu(), - custom_steps.compile_unix_full_gpu(), - custom_steps.compile_unix_cmake_mkldnn_gpu(), - custom_steps.compile_unix_cmake_gpu(), - custom_steps.compile_unix_tensorrt_gpu(), - custom_steps.compile_unix_int64_gpu() - ]) - - utils.parallel_stage('Tests', [ - custom_steps.test_unix_python2_gpu(), - custom_steps.test_unix_python3_gpu(), - custom_steps.test_unix_python2_quantize_gpu(), - custom_steps.test_unix_python3_quantize_gpu(), - custom_steps.test_unix_python2_mkldnn_gpu(), - custom_steps.test_unix_python3_mkldnn_gpu(), - custom_steps.test_unix_python3_mkldnn_nocudnn_gpu(), - custom_steps.test_unix_python3_tensorrt_gpu(), - custom_steps.test_unix_perl_gpu(), - custom_steps.test_unix_r_gpu(), - custom_steps.test_unix_cpp_gpu(), - custom_steps.test_unix_cpp_mkldnn_gpu(), - custom_steps.test_unix_python3_integration_gpu(), - custom_steps.test_unix_cpp_package_gpu(), - custom_steps.test_unix_scala_gpu(), - custom_steps.test_unix_distributed_kvstore_gpu(), - custom_steps.test_static_python_gpu() - - // Disabled due to: https://github.com/apache/incubator-mxnet/issues/11407 - //custom_steps.test_unix_caffe_gpu() - ]) -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_website b/ci/jenkins/Jenkinsfile_website deleted file mode 100644 index f6c853046b1c..000000000000 --- a/ci/jenkins/Jenkinsfile_website +++ /dev/null @@ -1,48 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 180 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Placeholder', [ - // Do nothing - ]) - -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_website_c_docs b/ci/jenkins/Jenkinsfile_website_c_docs deleted file mode 100644 index e678920e3a4f..000000000000 --- a/ci/jenkins/Jenkinsfile_website_c_docs +++ /dev/null @@ -1,52 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 20 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite() - ]) - - utils.parallel_stage('C Docs', [ - custom_steps.docs_c() - ]) - -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_website_clojure_docs b/ci/jenkins/Jenkinsfile_website_clojure_docs deleted file mode 100644 index 697dca48e58b..000000000000 --- a/ci/jenkins/Jenkinsfile_website_clojure_docs +++ /dev/null @@ -1,53 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 20 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite() - ]) - - utils.parallel_stage('Clojure Docs', [ - custom_steps.docs_clojure() - - ]) - -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_website_full b/ci/jenkins/Jenkinsfile_website_full deleted file mode 100644 index 39cc6f4e2dc9..000000000000 --- a/ci/jenkins/Jenkinsfile_website_full +++ /dev/null @@ -1,67 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 180 - -node('restricted-utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} - -utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite() - ]) - - utils.parallel_stage('Build Docs', [ - custom_steps.docs_jekyll(), - custom_steps.docs_c(), - custom_steps.docs_python(), - custom_steps.docs_julia(), - custom_steps.docs_r(), - custom_steps.docs_scala(), - custom_steps.docs_java(), - custom_steps.docs_clojure() - ]) - - utils.parallel_stage('Prepare', [ - custom_steps.docs_prepare() - ]) - - utils.parallel_stage('Publish', [ - custom_steps.docs_publish() - ]) -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_website_full_pr b/ci/jenkins/Jenkinsfile_website_full_pr deleted file mode 100644 index 133c6c204964..000000000000 --- a/ci/jenkins/Jenkinsfile_website_full_pr +++ /dev/null @@ -1,62 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 180 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite() - ]) - - utils.parallel_stage('Build Docs', [ - // Optimization would be to flag these not to stash if not previewing them - custom_steps.docs_jekyll(), - custom_steps.docs_c(), - custom_steps.docs_python(), - custom_steps.docs_julia(), - custom_steps.docs_r(), - custom_steps.docs_scala(), - custom_steps.docs_java(), - custom_steps.docs_clojure() - ]) - - // TODO: add a website preview function - -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_website_java_docs b/ci/jenkins/Jenkinsfile_website_java_docs deleted file mode 100644 index 4453b444039b..000000000000 --- a/ci/jenkins/Jenkinsfile_website_java_docs +++ /dev/null @@ -1,53 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 20 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite() - ]) - - utils.parallel_stage('Java Docs', [ - custom_steps.docs_java() - - ]) - -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_website_jekyll_docs b/ci/jenkins/Jenkinsfile_website_jekyll_docs deleted file mode 100644 index c4c5ff89dd86..000000000000 --- a/ci/jenkins/Jenkinsfile_website_jekyll_docs +++ /dev/null @@ -1,49 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 20 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Jekyll Website Docs', [ - custom_steps.docs_jekyll() - - ]) - -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_website_julia_docs b/ci/jenkins/Jenkinsfile_website_julia_docs deleted file mode 100644 index e1e9aaa983fb..000000000000 --- a/ci/jenkins/Jenkinsfile_website_julia_docs +++ /dev/null @@ -1,53 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 60 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite() - ]) - - utils.parallel_stage('Julia Docs', [ - custom_steps.docs_julia() - - ]) - -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_website_mxnet_build b/ci/jenkins/Jenkinsfile_website_mxnet_build deleted file mode 100644 index a3c3330d0349..000000000000 --- a/ci/jenkins/Jenkinsfile_website_mxnet_build +++ /dev/null @@ -1,48 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 180 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite() - ]) - -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_website_python_docs b/ci/jenkins/Jenkinsfile_website_python_docs deleted file mode 100644 index baaf4519541f..000000000000 --- a/ci/jenkins/Jenkinsfile_website_python_docs +++ /dev/null @@ -1,53 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 60 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite() - ]) - - utils.parallel_stage('Python Docs', [ - custom_steps.docs_python() - - ]) - -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_website_r_docs b/ci/jenkins/Jenkinsfile_website_r_docs deleted file mode 100644 index 5b9f6630563f..000000000000 --- a/ci/jenkins/Jenkinsfile_website_r_docs +++ /dev/null @@ -1,53 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 60 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite() - ]) - - utils.parallel_stage('R Docs', [ - custom_steps.docs_r() - - ]) - -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) diff --git a/ci/jenkins/Jenkinsfile_website_scala_docs b/ci/jenkins/Jenkinsfile_website_scala_docs deleted file mode 100644 index 6a083dae7957..000000000000 --- a/ci/jenkins/Jenkinsfile_website_scala_docs +++ /dev/null @@ -1,53 +0,0 @@ -// -*- mode: groovy -*- - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// Jenkins pipeline -// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - -// timeout in minutes -max_time = 20 - -node('utility') { - // Loading the utilities requires a node context unfortunately - checkout scm - utils = load('ci/Jenkinsfile_utils.groovy') - custom_steps = load('ci/jenkins/Jenkins_steps.groovy') -} -utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') - -utils.main_wrapper( -core_logic: { - utils.parallel_stage('Build', [ - custom_steps.compile_unix_lite() - ]) - - utils.parallel_stage('Scala Docs', [ - custom_steps.docs_scala() - - ]) - -} -, -failure_handler: { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } -} -) From 79ceaa75d8460f1c03fa76297f85601bd1093f65 Mon Sep 17 00:00:00 2001 From: reminisce Date: Wed, 9 Oct 2019 12:04:42 -0700 Subject: [PATCH 3/5] Revert "Numpy Operators: Inner, Outer, vdot (#15846)" This reverts commit 2df3282666c5ba55e98340c3615b046278c40b53. --- python/mxnet/ndarray/numpy/_op.py | 163 +-------------------- python/mxnet/numpy/multiarray.py | 149 +------------------ python/mxnet/symbol/numpy/_symbol.py | 155 +------------------- tests/python/unittest/test_numpy_op.py | 195 ------------------------- 4 files changed, 11 insertions(+), 651 deletions(-) diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py index 9dce953004cf..6309e0c0b881 100644 --- a/python/mxnet/ndarray/numpy/_op.py +++ b/python/mxnet/ndarray/numpy/_op.py @@ -35,7 +35,7 @@ 'linspace', 'expand_dims', 'tile', 'arange', 'split', 'concatenate', 'stack', 'vstack', 'mean', 'maximum', 'minimum', 'swapaxes', 'clip', 'argmax', 'std', 'var', 'indices', 'copysign', 'ravel', 'hanning', 'hamming', 'blackman', 'flip', 'around', 'hypot', 'rad2deg', 'deg2rad', - 'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', + 'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal'] @@ -1904,7 +1904,7 @@ def tan(x, out=None, where=True, **kwargs): Parameters: ---------- - x : ndarray + x : array_like Input array. out : ndarray, None, or tuple of ndarray and None, optional A location into which the result is stored. If provided, @@ -2324,7 +2324,7 @@ def concatenate(seq, axis=0, out=None): """Join a sequence of arrays along an existing axis. Parameters ---------- - a1, a2, ... : sequence of ndarray + a1, a2, ... : sequence of array_like The arrays must have the same shape, except in the dimension corresponding to `axis` (the first, by default). axis : int, optional @@ -2349,7 +2349,7 @@ def stack(arrays, axis=0, out=None): For example, if `axis=0` it will be the first dimension and if `axis=-1` it will be the last dimension. Parameters ---------- - arrays : sequence of ndarray + arrays : sequence of array_like Each array must have the same shape. axis : int, optional The axis in the result array along which the input arrays are stacked. @@ -2511,7 +2511,7 @@ def clip(a, a_min, a_max, out=None): Notes ----- - ndarray `a_min` and `a_max` are not supported. + array_like `a_min` and `a_max` are not supported. Examples -------- @@ -2670,7 +2670,7 @@ def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False): # pylint: Parameters ---------- - a : ndarray + a : array_like Calculate the standard deviation of these values. axis : None or int or tuple of ints, optional Axis or axes along which the standard deviation is computed. The @@ -2737,7 +2737,7 @@ def var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False): # pylint: Parameters ---------- - a : ndarray + a : array_like Array containing numbers whose variance is desired. If `a` is not an array, a conversion is attempted. axis : None or int or tuple of ints, optional @@ -3431,7 +3431,7 @@ def hypot(x1, x2, out=None): Parameters ---------- - x1, x2 : ndarray + x1, x2 : array_like Leg of the triangle(s). out : ndarray, None, or tuple of ndarray and None, optional A location into which the result is stored. If provided, it must have @@ -3505,153 +3505,6 @@ def ldexp(x1, x2, out=None): return _ufunc_helper(x1, x2, _npi.ldexp, _np.ldexp, _npi.ldexp_scalar, _npi.rldexp_scalar, out) -@set_module('mxnet.ndarray.numpy') -def inner(a, b): - r""" - Inner product of two arrays. - Ordinary inner product of vectors for 1-D arrays (without complex - conjugation), in higher dimensions a sum product over the last axes. - - Parameters - ---------- - a, b : ndarray - If `a` and `b` are nonscalar, their last dimensions must match. - - Returns - ------- - out : ndarray - `out.shape = a.shape[:-1] + b.shape[:-1]` - - Raises - ------ - ValueError - If the last dimension of `a` and `b` has different size. - - See Also - -------- - tensordot : Sum products over arbitrary axes. - dot : Generalised matrix product, using second last dimension of `b`. - einsum : Einstein summation convention. - - Notes - ----- - For vectors (1-D arrays) it computes the ordinary inner-product:: - np.inner(a, b) = sum(a[:]*b[:]) - More generally, if `ndim(a) = r > 0` and `ndim(b) = s > 0`:: - np.inner(a, b) = np.tensordot(a, b, axes=(-1,-1)) - or explicitly:: - np.inner(a, b)[i0,...,ir-1,j0,...,js-1] - = sum(a[i0,...,ir-1,:]*b[j0,...,js-1,:]) - In addition `a` or `b` may be scalars, in which case:: - np.inner(a,b) = a*b - - Examples - -------- - Ordinary inner product for vectors: - >>> a = np.array([1,2,3]) - >>> b = np.array([0,1,0]) - >>> np.inner(a, b) - 2 - A multidimensional example: - >>> a = np.arange(24).reshape((2,3,4)) - >>> b = np.arange(4) - >>> np.inner(a, b) - array([[ 14, 38, 62], - [ 86, 110, 134]]) - """ - return tensordot(a, b, [-1, -1]) - - -@set_module('mxnet.ndarray.numpy') -def outer(a, b): - r""" - Compute the outer product of two vectors. - Given two vectors, ``a = [a0, a1, ..., aM]`` and - ``b = [b0, b1, ..., bN]``, - the outer product [1]_ is:: - [[a0*b0 a0*b1 ... a0*bN ] - [a1*b0 . - [ ... . - [aM*b0 aM*bN ]] - - Parameters - ---------- - a : (M,) ndarray - First input vector. Input is flattened if - not already 1-dimensional. - b : (N,) ndarray - Second input vector. Input is flattened if - not already 1-dimensional. - - Returns - ------- - out : (M, N) ndarray - ``out[i, j] = a[i] * b[j]`` - See also - -------- - inner - einsum : ``einsum('i,j->ij', a.ravel(), b.ravel())`` is the equivalent. - ufunc.outer : A generalization to N dimensions and other operations. - ``np.multiply.outer(a.ravel(), b.ravel())`` is the equivalent. - References - ---------- - .. [1] : G. H. Golub and C. F. Van Loan, *Matrix Computations*, 3rd - ed., Baltimore, MD, Johns Hopkins University Press, 1996, - pg. 8. - Examples - -------- - Make a (*very* coarse) grid for computing a Mandelbrot set: - >>> rl = np.outer(np.ones((5,)), np.linspace(-2, 2, 5)) - >>> rl - array([[-2., -1., 0., 1., 2.], - [-2., -1., 0., 1., 2.], - [-2., -1., 0., 1., 2.], - [-2., -1., 0., 1., 2.], - [-2., -1., 0., 1., 2.]]) - """ - return tensordot(a.flatten(), b.flatten(), 0) - - -@set_module('mxnet.ndarray.numpy') -def vdot(a, b): - r""" - Return the dot product of two vectors. - Note that `vdot` handles multidimensional arrays differently than `dot`: - it does *not* perform a matrix product, but flattens input arguments - to 1-D vectors first. Consequently, it should only be used for vectors. - - Parameters - ---------- - a : ndarray - First argument to the dot product. - b : ndarray - Second argument to the dot product. - - Returns - ------- - output : ndarray - Dot product of `a` and `b`. - - See Also - -------- - dot : Return the dot product without using the complex conjugate of the - first argument. - - Examples - -------- - Note that higher-dimensional arrays are flattened! - >>> a = np.array([[1, 4], [5, 6]]) - >>> b = np.array([[4, 1], [2, 2]]) - >>> np.vdot(a, b) - 30 - >>> np.vdot(b, a) - 30 - >>> 1*4 + 4*1 + 5*2 + 6*2 - 30 - """ - return tensordot(a.flatten(), b.flatten(), 1) - - @set_module('mxnet.ndarray.numpy') def equal(x1, x2, out=None): """ diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py index 75b7cf65325b..55ee0c9bfcff 100644 --- a/python/mxnet/numpy/multiarray.py +++ b/python/mxnet/numpy/multiarray.py @@ -53,7 +53,7 @@ 'tensordot', 'linspace', 'expand_dims', 'tile', 'arange', 'split', 'concatenate', 'stack', 'vstack', 'mean', 'maximum', 'minimum', 'swapaxes', 'clip', 'argmax', 'std', 'var', 'indices', 'copysign', 'ravel', 'hanning', 'hamming', 'blackman', 'flip', 'around', 'arctan2', 'hypot', - 'rad2deg', 'deg2rad', 'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', + 'rad2deg', 'deg2rad', 'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal'] # Return code for dispatching indexing function call @@ -5028,153 +5028,6 @@ def ldexp(x1, x2, out=None): return _mx_nd_np.ldexp(x1, x2, out) -@set_module('mxnet.numpy') -def inner(a, b): - r"""Inner product of two arrays. - Ordinary inner product of vectors for 1-D arrays (without complex - conjugation), in higher dimensions a sum product over the last axes. - - Parameters - ---------- - a, b : ndarray - If `a` and `b` are nonscalar, their last dimensions must match. - - Returns - ------- - out : ndarray - `out.shape = a.shape[:-1] + b.shape[:-1]` - - Raises - ------ - ValueError - If the last dimension of `a` and `b` has different size. - - See Also - -------- - tensordot : Sum products over arbitrary axes. - dot : Generalised matrix product, using second last dimension of `b`. - einsum : Einstein summation convention. - - Notes - ----- - For vectors (1-D arrays) it computes the ordinary inner-product:: - np.inner(a, b) = sum(a[:]*b[:]) - More generally, if `ndim(a) = r > 0` and `ndim(b) = s > 0`:: - np.inner(a, b) = np.tensordot(a, b, axes=(-1,-1)) - or explicitly:: - np.inner(a, b)[i0,...,ir-1,j0,...,js-1] - = sum(a[i0,...,ir-1,:]*b[j0,...,js-1,:]) - In addition `a` or `b` may be scalars, in which case:: - np.inner(a,b) = a*b - - Examples - -------- - Ordinary inner product for vectors: - >>> a = np.array([1,2,3]) - >>> b = np.array([0,1,0]) - >>> np.inner(a, b) - 2 - A multidimensional example: - >>> a = np.arange(24).reshape((2,3,4)) - >>> b = np.arange(4) - >>> np.inner(a, b) - array([[ 14, 38, 62], - [ 86, 110, 134]]) - """ - return tensordot(a, b, [-1, -1]) - - -@set_module('mxnet.numpy') -def outer(a, b): - r"""Compute the outer product of two vectors. - Given two vectors, ``a = [a0, a1, ..., aM]`` and - ``b = [b0, b1, ..., bN]``, - the outer product [1]_ is:: - [[a0*b0 a0*b1 ... a0*bN ] - [a1*b0 . - [ ... . - [aM*b0 aM*bN ]] - - Parameters - ---------- - a : (M,) ndarray - First input vector. Input is flattened if - not already 1-dimensional. - b : (N,) ndarray - Second input vector. Input is flattened if - not already 1-dimensional. - - Returns - ------- - out : (M, N) ndarray - ``out[i, j] = a[i] * b[j]`` - See also - -------- - inner - einsum : ``einsum('i,j->ij', a.ravel(), b.ravel())`` is the equivalent. - ufunc.outer : A generalization to N dimensions and other operations. - ``np.multiply.outer(a.ravel(), b.ravel())`` is the equivalent. - - References - ---------- - .. [1] : G. H. Golub and C. F. Van Loan, *Matrix Computations*, 3rd - ed., Baltimore, MD, Johns Hopkins University Press, 1996, - pg. 8. - - Examples - -------- - Make a (*very* coarse) grid for computing a Mandelbrot set: - >>> rl = np.outer(np.ones((5,)), np.linspace(-2, 2, 5)) - >>> rl - array([[-2., -1., 0., 1., 2.], - [-2., -1., 0., 1., 2.], - [-2., -1., 0., 1., 2.], - [-2., -1., 0., 1., 2.], - [-2., -1., 0., 1., 2.]]) - """ - return tensordot(a.flatten(), b.flatten(), 0) - - -@set_module('mxnet.numpy') -def vdot(a, b): - r""" - Return the dot product of two vectors. - Note that `vdot` handles multidimensional arrays differently than `dot`: - it does *not* perform a matrix product, but flattens input arguments - to 1-D vectors first. Consequently, it should only be used for vectors. - - Parameters - ---------- - a : ndarray - First argument to the dot product. - b : ndarray - Second argument to the dot product. - - Returns - ------- - output : ndarray - Dot product of `a` and `b`. - - See Also - -------- - dot : Return the dot product without using the complex conjugate of the - first argument. - - Examples - -------- - Note that higher-dimensional arrays are flattened! - >>> a = np.array([[1, 4], [5, 6]]) - >>> b = np.array([[4, 1], [2, 2]]) - >>> np.vdot(a, b) - 30 - >>> np.vdot(b, a) - 30 - >>> 1*4 + 4*1 + 5*2 + 6*2 - 30 - """ - return tensordot(a.flatten(), b.flatten(), 1) - - @set_module('mxnet.numpy') def equal(x1, x2, out=None): """ diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py index 3eaf80a1b6fb..11c8e4d05398 100644 --- a/python/mxnet/symbol/numpy/_symbol.py +++ b/python/mxnet/symbol/numpy/_symbol.py @@ -37,7 +37,7 @@ 'linspace', 'expand_dims', 'tile', 'arange', 'split', 'concatenate', 'stack', 'vstack', 'mean', 'maximum', 'minimum', 'swapaxes', 'clip', 'argmax', 'std', 'var', 'indices', 'copysign', 'ravel', 'hanning', 'hamming', 'blackman', 'flip', 'around', 'hypot', 'rad2deg', 'deg2rad', - 'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', + 'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal'] @@ -1592,7 +1592,6 @@ def absolute(x, out=None, **kwargs): r""" Calculate the absolute value element-wise. np.abs is a shorthand for this function. - Parameters ---------- x : _Symbol @@ -3521,10 +3520,10 @@ def unique(ar, return_index=False, return_inverse=False, return_counts=False, ax @set_module('mxnet.symbol.numpy') def ldexp(x1, x2, out=None): """ + ldexp(x1, x2, out=None) Returns x1 * 2**x2, element-wise. The mantissas `x1` and twos exponents `x2` are used to construct floating point numbers ``x1 * 2**x2``. - Parameters ---------- x1 : _Symbol @@ -3533,12 +3532,10 @@ def ldexp(x1, x2, out=None): Array of twos exponents. out : _Symbol or None Dummy parameter to keep the consistency with the ndarray counterpart. - Returns ------- y : _Symbol The result of ``x1 * 2**x2``. - Notes ----- Complex dtypes are not supported, they will raise a TypeError. @@ -3549,154 +3546,6 @@ def ldexp(x1, x2, out=None): return _ufunc_helper(x1, x2, _npi.ldexp, _np.ldexp, _npi.ldexp_scalar, _npi.rldexp_scalar, out) -@set_module('mxnet.symbol.numpy') -def inner(a, b): - r"""Inner product of two arrays. - Ordinary inner product of vectors for 1-D arrays (without complex - conjugation), in higher dimensions a sum product over the last axes. - - Parameters - ---------- - a, b : _Symbol - If `a` and `b` are nonscalar, their last dimensions must match. - - Returns - ------- - out : _Symbol - `out.shape = a.shape[:-1] + b.shape[:-1]` - - Raises - ------ - ValueError - If the last dimension of `a` and `b` has different size. - - See Also - -------- - tensordot : Sum products over arbitrary axes. - dot : Generalised matrix product, using second last dimension of `b`. - einsum : Einstein summation convention. - - Notes - ----- - For vectors (1-D arrays) it computes the ordinary inner-product:: - np.inner(a, b) = sum(a[:]*b[:]) - More generally, if `ndim(a) = r > 0` and `ndim(b) = s > 0`:: - np.inner(a, b) = np.tensordot(a, b, axes=(-1,-1)) - or explicitly:: - np.inner(a, b)[i0,...,ir-1,j0,...,js-1] - = sum(a[i0,...,ir-1,:]*b[j0,...,js-1,:]) - In addition `a` or `b` may be scalars, in which case:: - np.inner(a,b) = a*b - - Examples - -------- - Ordinary inner product for vectors: - >>> a = np.array([1,2,3]) - >>> b = np.array([0,1,0]) - >>> np.inner(a, b) - 2 - A multidimensional example: - >>> a = np.arange(24).reshape((2,3,4)) - >>> b = np.arange(4) - >>> np.inner(a, b) - array([[ 14, 38, 62], - [ 86, 110, 134]]) - """ - return tensordot(a, b, [-1, -1]) - - -@set_module('mxnet.symbol.numpy') -def outer(a, b): - r"""Compute the outer product of two vectors. - Given two vectors, ``a = [a0, a1, ..., aM]`` and - ``b = [b0, b1, ..., bN]``, - the outer product [1]_ is:: - [[a0*b0 a0*b1 ... a0*bN ] - [a1*b0 . - [ ... . - [aM*b0 aM*bN ]] - - Parameters - ---------- - a : (M,) ndarray - First input vector. Input is flattened if - not already 1-dimensional. - b : (N,) ndarray - Second input vector. Input is flattened if - not already 1-dimensional. - - Returns - ------- - out : (M, N) ndarray - ``out[i, j] = a[i] * b[j]`` - - See also - -------- - inner - einsum : ``einsum('i,j->ij', a.ravel(), b.ravel())`` is the equivalent. - ufunc.outer : A generalization to N dimensions and other operations. - ``np.multiply.outer(a.ravel(), b.ravel())`` is the equivalent. - - References - ---------- - .. [1] : G. H. Golub and C. F. Van Loan, *Matrix Computations*, 3rd - ed., Baltimore, MD, Johns Hopkins University Press, 1996, - pg. 8. - - Examples - -------- - Make a (*very* coarse) grid for computing a Mandelbrot set: - >>> rl = np.outer(np.ones((5,)), np.linspace(-2, 2, 5)) - >>> rl - array([[-2., -1., 0., 1., 2.], - [-2., -1., 0., 1., 2.], - [-2., -1., 0., 1., 2.], - [-2., -1., 0., 1., 2.], - [-2., -1., 0., 1., 2.]]) - """ - return tensordot(a.flatten(), b.flatten(), 0) - - -@set_module('mxnet.symbol.numpy') -def vdot(a, b): - r""" - Return the dot product of two vectors. - Note that `vdot` handles multidimensional arrays differently than `dot`: - it does *not* perform a matrix product, but flattens input arguments - to 1-D vectors first. Consequently, it should only be used for vectors. - - Parameters - ---------- - a : _Symbol - First argument to the dot product. - b : _Symbol - Second argument to the dot product. - - Returns - ------- - output : _Symbol - Dot product of `a` and `b`. - - See Also - -------- - dot : Return the dot product without using the complex conjugate of the - first argument. - - Examples - -------- - Note that higher-dimensional arrays are flattened! - >>> a = np.array([[1, 4], [5, 6]]) - >>> b = np.array([[4, 1], [2, 2]]) - >>> np.vdot(a, b) - 30 - >>> np.vdot(b, a) - 30 - >>> 1*4 + 4*1 + 5*2 + 6*2 - 30 - """ - return tensordot(a.flatten(), b.flatten(), 1) - - @set_module('mxnet.symbol.numpy') def equal(x1, x2, out=None): """ diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py index 7068bf20d897..494311ec0035 100644 --- a/tests/python/unittest/test_numpy_op.py +++ b/tests/python/unittest/test_numpy_op.py @@ -280,201 +280,6 @@ def dldx(x1, x2): assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-1, atol=1e-1) -@with_seed() -@use_np -def test_np_vdot(): - class TestVdot(HybridBlock): - def __init__(self): - super(TestVdot, self).__init__() - - def hybrid_forward(self, F, a, b): - return F.np.vdot(a, b) - - def vdot_backward(a, b): - return [b, a] - - # test different size inputs - tensor_shapes = [(), (5,), (3, 3)] - - for hybridize in [True, False]: - for shape in tensor_shapes: - for dtype in [_np.float32, _np.float64]: - test_vdot = TestVdot() - if hybridize: - test_vdot.hybridize() - a = rand_ndarray(shape=shape, dtype=dtype).as_np_ndarray() - b = rand_ndarray(shape=shape, dtype=dtype).as_np_ndarray() - a.attach_grad() - b.attach_grad() - - np_out = _np.vdot(a.asnumpy(), b.asnumpy()) - with mx.autograd.record(): - mx_out = test_vdot(a, b) - assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol = 1e-3, atol = 1e-5) - mx_out.backward() - np_backward = vdot_backward(a.asnumpy(), b.asnumpy()) - assert_almost_equal(a.grad.asnumpy(), np_backward[0], rtol = 1e-2, atol=1e-2) - assert_almost_equal(b.grad.asnumpy(), np_backward[1], rtol = 1e-2, atol=1e-2) - - # Test imperative once again - mx_out = np.vdot(a, b) - np_out = _np.vdot(a.asnumpy(), b.asnumpy()) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) - - # test numeric gradient - if len(shape) > 0 and _np.prod(shape) > 0: - a_sym = mx.sym.Variable("a").as_np_ndarray() - b_sym = mx.sym.Variable("b").as_np_ndarray() - mx_sym = mx.sym.np.vdot(a_sym, b_sym).as_nd_ndarray() - check_numeric_gradient(mx_sym, [a.as_nd_ndarray(), b.as_nd_ndarray()], - rtol=1e-1, atol=1e-1, dtype=dtype) - - -@with_seed() -@use_np -def test_np_inner(): - class TestInner(HybridBlock): - def __init__(self): - super(TestInner, self).__init__() - - def hybrid_forward(self, F, a, b): - return F.np.inner(a, b) - - def inner_backward(a, b): - a_axes_summed = [a.ndim - 1] - b_axes_summed = [b.ndim - 1] - - a_axes_remained = [] - for i in range(a.ndim): - if not (i in a_axes_summed): - a_axes_remained.append(i) - a_axes = a_axes_remained[:] + a_axes_summed[:] - - b_axes_remained = [] - for i in range(b.ndim): - if not (i in b_axes_summed): - b_axes_remained.append(i) - b_axes = b_axes_summed[:] + b_axes_remained[:] - - ad1 = _np.prod([a.shape[i] for i in a_axes_remained]) if len(a_axes_remained) > 0 else 1 - ad2 = _np.prod([a.shape[i] for i in a_axes_summed]) if len(a_axes_summed) > 0 else 1 - bd1 = _np.prod([b.shape[i] for i in b_axes_summed]) if len(b_axes_summed) > 0 else 1 - bd2 = _np.prod([b.shape[i] for i in b_axes_remained]) if len(b_axes_remained) > 0 else 1 - - out_grad = _np.ones((ad1, bd2)) - - new_a = _np.transpose(a, a_axes) - new_a_shape = new_a.shape[:] - new_a = new_a.reshape((ad1, ad2)) - new_b = _np.transpose(b, b_axes) - new_b_shape = new_b.shape[:] - new_b = new_b.reshape((bd1, bd2)) - - reverse_a_axes = [0 for i in a_axes] - for i in range(len(a_axes)): - reverse_a_axes[a_axes[i]] = i - - reverse_b_axes = [0 for i in b_axes] - for i in range(len(b_axes)): - reverse_b_axes[b_axes[i]] = i - - grad_b = _np.dot(new_a.T, out_grad).reshape(new_b_shape) - grad_b = _np.transpose(grad_b, reverse_b_axes) - grad_a = _np.dot(out_grad, new_b.T).reshape(new_a_shape) - grad_a = _np.transpose(grad_a, reverse_a_axes) - - return [grad_a, grad_b] - - # test non zero size input - tensor_shapes = [ - ((3,), (3,)), - ((2, 3), (3,)), - ((3,), (2, 3)) - ] - - for hybridize in [True, False]: - for a_shape, b_shape in tensor_shapes: - for dtype in [_np.float32, _np.float64]: - test_inner = TestInner() - if hybridize: - test_inner.hybridize() - a = rand_ndarray(shape=a_shape, dtype=dtype).as_np_ndarray() - b = rand_ndarray(shape=b_shape, dtype=dtype).as_np_ndarray() - a.attach_grad() - b.attach_grad() - - np_out = _np.inner(a.asnumpy(), b.asnumpy()) - with mx.autograd.record(): - mx_out = test_inner(a, b) - assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol = 1e-3, atol = 1e-5) - mx_out.backward() - np_backward = inner_backward(a.asnumpy(), b.asnumpy()) - assert_almost_equal(a.grad.asnumpy(), np_backward[0], rtol = 1e-2, atol=1e-2) - assert_almost_equal(b.grad.asnumpy(), np_backward[1], rtol = 1e-2, atol=1e-2) - - # Test imperative once again - mx_out = np.inner(a, b) - np_out = _np.inner(a.asnumpy(), b.asnumpy()) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) - - # test numeric gradient - a_sym = mx.sym.Variable("a").as_np_ndarray() - b_sym = mx.sym.Variable("b").as_np_ndarray() - mx_sym = mx.sym.np.inner(a_sym, b_sym).as_nd_ndarray() - check_numeric_gradient(mx_sym, [a.as_nd_ndarray(), b.as_nd_ndarray()], - rtol=1e-1, atol=1e-1, dtype=dtype) - - -@with_seed() -@use_np -def test_np_outer(): - class TestOuter(HybridBlock): - def __init__(self): - super(TestOuter, self).__init__() - - def hybrid_forward(self, F, a, b): - return F.np.outer(a, b) - - # test non zero size input - tensor_shapes = [ - ((3,), (3,)), - ((2, 3), (6,)), - ((6,), (2, 3)) - ] - - for hybridize in [True, False]: - for a_shape, b_shape in tensor_shapes: - for dtype in [_np.float32, _np.float64]: - test_outer = TestOuter() - if hybridize: - test_outer.hybridize() - a = rand_ndarray(shape=a_shape, dtype=dtype).as_np_ndarray() - b = rand_ndarray(shape=b_shape, dtype=dtype).as_np_ndarray() - a.attach_grad() - b.attach_grad() - - np_out = _np.outer(a.asnumpy(), b.asnumpy()) - with mx.autograd.record(): - mx_out = test_outer(a, b) - assert mx_out.shape == np_out.shape - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) - mx_out.backward() - - # Test imperative once again - mx_out = np.outer(a, b) - np_out = _np.outer(a.asnumpy(), b.asnumpy()) - assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) - - # test numeric gradient - a_sym = mx.sym.Variable("a").as_np_ndarray() - b_sym = mx.sym.Variable("b").as_np_ndarray() - mx_sym = mx.sym.np.outer(a_sym, b_sym).as_nd_ndarray() - check_numeric_gradient(mx_sym, [a.as_nd_ndarray(), b.as_nd_ndarray()], - rtol=1e-1, atol=1e-1, dtype=dtype) - - @with_seed() @use_np def test_np_sum(): From b05b63026e40839fa0495fb610215d4c5edd8b98 Mon Sep 17 00:00:00 2001 From: reminisce Date: Wed, 9 Oct 2019 15:07:48 -0700 Subject: [PATCH 4/5] Revert "Revert "Numpy Operators: Inner, Outer, vdot (#15846)"" This reverts commit 79ceaa75d8460f1c03fa76297f85601bd1093f65. --- python/mxnet/ndarray/numpy/_op.py | 163 ++++++++++++++++++++- python/mxnet/numpy/multiarray.py | 149 ++++++++++++++++++- python/mxnet/symbol/numpy/_symbol.py | 155 +++++++++++++++++++- tests/python/unittest/test_numpy_op.py | 195 +++++++++++++++++++++++++ 4 files changed, 651 insertions(+), 11 deletions(-) diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py index 6309e0c0b881..9dce953004cf 100644 --- a/python/mxnet/ndarray/numpy/_op.py +++ b/python/mxnet/ndarray/numpy/_op.py @@ -35,7 +35,7 @@ 'linspace', 'expand_dims', 'tile', 'arange', 'split', 'concatenate', 'stack', 'vstack', 'mean', 'maximum', 'minimum', 'swapaxes', 'clip', 'argmax', 'std', 'var', 'indices', 'copysign', 'ravel', 'hanning', 'hamming', 'blackman', 'flip', 'around', 'hypot', 'rad2deg', 'deg2rad', - 'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', + 'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal'] @@ -1904,7 +1904,7 @@ def tan(x, out=None, where=True, **kwargs): Parameters: ---------- - x : array_like + x : ndarray Input array. out : ndarray, None, or tuple of ndarray and None, optional A location into which the result is stored. If provided, @@ -2324,7 +2324,7 @@ def concatenate(seq, axis=0, out=None): """Join a sequence of arrays along an existing axis. Parameters ---------- - a1, a2, ... : sequence of array_like + a1, a2, ... : sequence of ndarray The arrays must have the same shape, except in the dimension corresponding to `axis` (the first, by default). axis : int, optional @@ -2349,7 +2349,7 @@ def stack(arrays, axis=0, out=None): For example, if `axis=0` it will be the first dimension and if `axis=-1` it will be the last dimension. Parameters ---------- - arrays : sequence of array_like + arrays : sequence of ndarray Each array must have the same shape. axis : int, optional The axis in the result array along which the input arrays are stacked. @@ -2511,7 +2511,7 @@ def clip(a, a_min, a_max, out=None): Notes ----- - array_like `a_min` and `a_max` are not supported. + ndarray `a_min` and `a_max` are not supported. Examples -------- @@ -2670,7 +2670,7 @@ def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False): # pylint: Parameters ---------- - a : array_like + a : ndarray Calculate the standard deviation of these values. axis : None or int or tuple of ints, optional Axis or axes along which the standard deviation is computed. The @@ -2737,7 +2737,7 @@ def var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False): # pylint: Parameters ---------- - a : array_like + a : ndarray Array containing numbers whose variance is desired. If `a` is not an array, a conversion is attempted. axis : None or int or tuple of ints, optional @@ -3431,7 +3431,7 @@ def hypot(x1, x2, out=None): Parameters ---------- - x1, x2 : array_like + x1, x2 : ndarray Leg of the triangle(s). out : ndarray, None, or tuple of ndarray and None, optional A location into which the result is stored. If provided, it must have @@ -3505,6 +3505,153 @@ def ldexp(x1, x2, out=None): return _ufunc_helper(x1, x2, _npi.ldexp, _np.ldexp, _npi.ldexp_scalar, _npi.rldexp_scalar, out) +@set_module('mxnet.ndarray.numpy') +def inner(a, b): + r""" + Inner product of two arrays. + Ordinary inner product of vectors for 1-D arrays (without complex + conjugation), in higher dimensions a sum product over the last axes. + + Parameters + ---------- + a, b : ndarray + If `a` and `b` are nonscalar, their last dimensions must match. + + Returns + ------- + out : ndarray + `out.shape = a.shape[:-1] + b.shape[:-1]` + + Raises + ------ + ValueError + If the last dimension of `a` and `b` has different size. + + See Also + -------- + tensordot : Sum products over arbitrary axes. + dot : Generalised matrix product, using second last dimension of `b`. + einsum : Einstein summation convention. + + Notes + ----- + For vectors (1-D arrays) it computes the ordinary inner-product:: + np.inner(a, b) = sum(a[:]*b[:]) + More generally, if `ndim(a) = r > 0` and `ndim(b) = s > 0`:: + np.inner(a, b) = np.tensordot(a, b, axes=(-1,-1)) + or explicitly:: + np.inner(a, b)[i0,...,ir-1,j0,...,js-1] + = sum(a[i0,...,ir-1,:]*b[j0,...,js-1,:]) + In addition `a` or `b` may be scalars, in which case:: + np.inner(a,b) = a*b + + Examples + -------- + Ordinary inner product for vectors: + >>> a = np.array([1,2,3]) + >>> b = np.array([0,1,0]) + >>> np.inner(a, b) + 2 + A multidimensional example: + >>> a = np.arange(24).reshape((2,3,4)) + >>> b = np.arange(4) + >>> np.inner(a, b) + array([[ 14, 38, 62], + [ 86, 110, 134]]) + """ + return tensordot(a, b, [-1, -1]) + + +@set_module('mxnet.ndarray.numpy') +def outer(a, b): + r""" + Compute the outer product of two vectors. + Given two vectors, ``a = [a0, a1, ..., aM]`` and + ``b = [b0, b1, ..., bN]``, + the outer product [1]_ is:: + [[a0*b0 a0*b1 ... a0*bN ] + [a1*b0 . + [ ... . + [aM*b0 aM*bN ]] + + Parameters + ---------- + a : (M,) ndarray + First input vector. Input is flattened if + not already 1-dimensional. + b : (N,) ndarray + Second input vector. Input is flattened if + not already 1-dimensional. + + Returns + ------- + out : (M, N) ndarray + ``out[i, j] = a[i] * b[j]`` + See also + -------- + inner + einsum : ``einsum('i,j->ij', a.ravel(), b.ravel())`` is the equivalent. + ufunc.outer : A generalization to N dimensions and other operations. + ``np.multiply.outer(a.ravel(), b.ravel())`` is the equivalent. + References + ---------- + .. [1] : G. H. Golub and C. F. Van Loan, *Matrix Computations*, 3rd + ed., Baltimore, MD, Johns Hopkins University Press, 1996, + pg. 8. + Examples + -------- + Make a (*very* coarse) grid for computing a Mandelbrot set: + >>> rl = np.outer(np.ones((5,)), np.linspace(-2, 2, 5)) + >>> rl + array([[-2., -1., 0., 1., 2.], + [-2., -1., 0., 1., 2.], + [-2., -1., 0., 1., 2.], + [-2., -1., 0., 1., 2.], + [-2., -1., 0., 1., 2.]]) + """ + return tensordot(a.flatten(), b.flatten(), 0) + + +@set_module('mxnet.ndarray.numpy') +def vdot(a, b): + r""" + Return the dot product of two vectors. + Note that `vdot` handles multidimensional arrays differently than `dot`: + it does *not* perform a matrix product, but flattens input arguments + to 1-D vectors first. Consequently, it should only be used for vectors. + + Parameters + ---------- + a : ndarray + First argument to the dot product. + b : ndarray + Second argument to the dot product. + + Returns + ------- + output : ndarray + Dot product of `a` and `b`. + + See Also + -------- + dot : Return the dot product without using the complex conjugate of the + first argument. + + Examples + -------- + Note that higher-dimensional arrays are flattened! + >>> a = np.array([[1, 4], [5, 6]]) + >>> b = np.array([[4, 1], [2, 2]]) + >>> np.vdot(a, b) + 30 + >>> np.vdot(b, a) + 30 + >>> 1*4 + 4*1 + 5*2 + 6*2 + 30 + """ + return tensordot(a.flatten(), b.flatten(), 1) + + @set_module('mxnet.ndarray.numpy') def equal(x1, x2, out=None): """ diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py index 55ee0c9bfcff..75b7cf65325b 100644 --- a/python/mxnet/numpy/multiarray.py +++ b/python/mxnet/numpy/multiarray.py @@ -53,7 +53,7 @@ 'tensordot', 'linspace', 'expand_dims', 'tile', 'arange', 'split', 'concatenate', 'stack', 'vstack', 'mean', 'maximum', 'minimum', 'swapaxes', 'clip', 'argmax', 'std', 'var', 'indices', 'copysign', 'ravel', 'hanning', 'hamming', 'blackman', 'flip', 'around', 'arctan2', 'hypot', - 'rad2deg', 'deg2rad', 'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', + 'rad2deg', 'deg2rad', 'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal'] # Return code for dispatching indexing function call @@ -5028,6 +5028,153 @@ def ldexp(x1, x2, out=None): return _mx_nd_np.ldexp(x1, x2, out) +@set_module('mxnet.numpy') +def inner(a, b): + r"""Inner product of two arrays. + Ordinary inner product of vectors for 1-D arrays (without complex + conjugation), in higher dimensions a sum product over the last axes. + + Parameters + ---------- + a, b : ndarray + If `a` and `b` are nonscalar, their last dimensions must match. + + Returns + ------- + out : ndarray + `out.shape = a.shape[:-1] + b.shape[:-1]` + + Raises + ------ + ValueError + If the last dimension of `a` and `b` has different size. + + See Also + -------- + tensordot : Sum products over arbitrary axes. + dot : Generalised matrix product, using second last dimension of `b`. + einsum : Einstein summation convention. + + Notes + ----- + For vectors (1-D arrays) it computes the ordinary inner-product:: + np.inner(a, b) = sum(a[:]*b[:]) + More generally, if `ndim(a) = r > 0` and `ndim(b) = s > 0`:: + np.inner(a, b) = np.tensordot(a, b, axes=(-1,-1)) + or explicitly:: + np.inner(a, b)[i0,...,ir-1,j0,...,js-1] + = sum(a[i0,...,ir-1,:]*b[j0,...,js-1,:]) + In addition `a` or `b` may be scalars, in which case:: + np.inner(a,b) = a*b + + Examples + -------- + Ordinary inner product for vectors: + >>> a = np.array([1,2,3]) + >>> b = np.array([0,1,0]) + >>> np.inner(a, b) + 2 + A multidimensional example: + >>> a = np.arange(24).reshape((2,3,4)) + >>> b = np.arange(4) + >>> np.inner(a, b) + array([[ 14, 38, 62], + [ 86, 110, 134]]) + """ + return tensordot(a, b, [-1, -1]) + + +@set_module('mxnet.numpy') +def outer(a, b): + r"""Compute the outer product of two vectors. + Given two vectors, ``a = [a0, a1, ..., aM]`` and + ``b = [b0, b1, ..., bN]``, + the outer product [1]_ is:: + [[a0*b0 a0*b1 ... a0*bN ] + [a1*b0 . + [ ... . + [aM*b0 aM*bN ]] + + Parameters + ---------- + a : (M,) ndarray + First input vector. Input is flattened if + not already 1-dimensional. + b : (N,) ndarray + Second input vector. Input is flattened if + not already 1-dimensional. + + Returns + ------- + out : (M, N) ndarray + ``out[i, j] = a[i] * b[j]`` + See also + -------- + inner + einsum : ``einsum('i,j->ij', a.ravel(), b.ravel())`` is the equivalent. + ufunc.outer : A generalization to N dimensions and other operations. + ``np.multiply.outer(a.ravel(), b.ravel())`` is the equivalent. + + References + ---------- + .. [1] : G. H. Golub and C. F. Van Loan, *Matrix Computations*, 3rd + ed., Baltimore, MD, Johns Hopkins University Press, 1996, + pg. 8. + + Examples + -------- + Make a (*very* coarse) grid for computing a Mandelbrot set: + >>> rl = np.outer(np.ones((5,)), np.linspace(-2, 2, 5)) + >>> rl + array([[-2., -1., 0., 1., 2.], + [-2., -1., 0., 1., 2.], + [-2., -1., 0., 1., 2.], + [-2., -1., 0., 1., 2.], + [-2., -1., 0., 1., 2.]]) + """ + return tensordot(a.flatten(), b.flatten(), 0) + + +@set_module('mxnet.numpy') +def vdot(a, b): + r""" + Return the dot product of two vectors. + Note that `vdot` handles multidimensional arrays differently than `dot`: + it does *not* perform a matrix product, but flattens input arguments + to 1-D vectors first. Consequently, it should only be used for vectors. + + Parameters + ---------- + a : ndarray + First argument to the dot product. + b : ndarray + Second argument to the dot product. + + Returns + ------- + output : ndarray + Dot product of `a` and `b`. + + See Also + -------- + dot : Return the dot product without using the complex conjugate of the + first argument. + + Examples + -------- + Note that higher-dimensional arrays are flattened! + >>> a = np.array([[1, 4], [5, 6]]) + >>> b = np.array([[4, 1], [2, 2]]) + >>> np.vdot(a, b) + 30 + >>> np.vdot(b, a) + 30 + >>> 1*4 + 4*1 + 5*2 + 6*2 + 30 + """ + return tensordot(a.flatten(), b.flatten(), 1) + + @set_module('mxnet.numpy') def equal(x1, x2, out=None): """ diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py index 11c8e4d05398..3eaf80a1b6fb 100644 --- a/python/mxnet/symbol/numpy/_symbol.py +++ b/python/mxnet/symbol/numpy/_symbol.py @@ -37,7 +37,7 @@ 'linspace', 'expand_dims', 'tile', 'arange', 'split', 'concatenate', 'stack', 'vstack', 'mean', 'maximum', 'minimum', 'swapaxes', 'clip', 'argmax', 'std', 'var', 'indices', 'copysign', 'ravel', 'hanning', 'hamming', 'blackman', 'flip', 'around', 'hypot', 'rad2deg', 'deg2rad', - 'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', + 'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal'] @@ -1592,6 +1592,7 @@ def absolute(x, out=None, **kwargs): r""" Calculate the absolute value element-wise. np.abs is a shorthand for this function. + Parameters ---------- x : _Symbol @@ -3520,10 +3521,10 @@ def unique(ar, return_index=False, return_inverse=False, return_counts=False, ax @set_module('mxnet.symbol.numpy') def ldexp(x1, x2, out=None): """ - ldexp(x1, x2, out=None) Returns x1 * 2**x2, element-wise. The mantissas `x1` and twos exponents `x2` are used to construct floating point numbers ``x1 * 2**x2``. + Parameters ---------- x1 : _Symbol @@ -3532,10 +3533,12 @@ def ldexp(x1, x2, out=None): Array of twos exponents. out : _Symbol or None Dummy parameter to keep the consistency with the ndarray counterpart. + Returns ------- y : _Symbol The result of ``x1 * 2**x2``. + Notes ----- Complex dtypes are not supported, they will raise a TypeError. @@ -3546,6 +3549,154 @@ def ldexp(x1, x2, out=None): return _ufunc_helper(x1, x2, _npi.ldexp, _np.ldexp, _npi.ldexp_scalar, _npi.rldexp_scalar, out) +@set_module('mxnet.symbol.numpy') +def inner(a, b): + r"""Inner product of two arrays. + Ordinary inner product of vectors for 1-D arrays (without complex + conjugation), in higher dimensions a sum product over the last axes. + + Parameters + ---------- + a, b : _Symbol + If `a` and `b` are nonscalar, their last dimensions must match. + + Returns + ------- + out : _Symbol + `out.shape = a.shape[:-1] + b.shape[:-1]` + + Raises + ------ + ValueError + If the last dimension of `a` and `b` has different size. + + See Also + -------- + tensordot : Sum products over arbitrary axes. + dot : Generalised matrix product, using second last dimension of `b`. + einsum : Einstein summation convention. + + Notes + ----- + For vectors (1-D arrays) it computes the ordinary inner-product:: + np.inner(a, b) = sum(a[:]*b[:]) + More generally, if `ndim(a) = r > 0` and `ndim(b) = s > 0`:: + np.inner(a, b) = np.tensordot(a, b, axes=(-1,-1)) + or explicitly:: + np.inner(a, b)[i0,...,ir-1,j0,...,js-1] + = sum(a[i0,...,ir-1,:]*b[j0,...,js-1,:]) + In addition `a` or `b` may be scalars, in which case:: + np.inner(a,b) = a*b + + Examples + -------- + Ordinary inner product for vectors: + >>> a = np.array([1,2,3]) + >>> b = np.array([0,1,0]) + >>> np.inner(a, b) + 2 + A multidimensional example: + >>> a = np.arange(24).reshape((2,3,4)) + >>> b = np.arange(4) + >>> np.inner(a, b) + array([[ 14, 38, 62], + [ 86, 110, 134]]) + """ + return tensordot(a, b, [-1, -1]) + + +@set_module('mxnet.symbol.numpy') +def outer(a, b): + r"""Compute the outer product of two vectors. + Given two vectors, ``a = [a0, a1, ..., aM]`` and + ``b = [b0, b1, ..., bN]``, + the outer product [1]_ is:: + [[a0*b0 a0*b1 ... a0*bN ] + [a1*b0 . + [ ... . + [aM*b0 aM*bN ]] + + Parameters + ---------- + a : (M,) ndarray + First input vector. Input is flattened if + not already 1-dimensional. + b : (N,) ndarray + Second input vector. Input is flattened if + not already 1-dimensional. + + Returns + ------- + out : (M, N) ndarray + ``out[i, j] = a[i] * b[j]`` + + See also + -------- + inner + einsum : ``einsum('i,j->ij', a.ravel(), b.ravel())`` is the equivalent. + ufunc.outer : A generalization to N dimensions and other operations. + ``np.multiply.outer(a.ravel(), b.ravel())`` is the equivalent. + + References + ---------- + .. [1] : G. H. Golub and C. F. Van Loan, *Matrix Computations*, 3rd + ed., Baltimore, MD, Johns Hopkins University Press, 1996, + pg. 8. + + Examples + -------- + Make a (*very* coarse) grid for computing a Mandelbrot set: + >>> rl = np.outer(np.ones((5,)), np.linspace(-2, 2, 5)) + >>> rl + array([[-2., -1., 0., 1., 2.], + [-2., -1., 0., 1., 2.], + [-2., -1., 0., 1., 2.], + [-2., -1., 0., 1., 2.], + [-2., -1., 0., 1., 2.]]) + """ + return tensordot(a.flatten(), b.flatten(), 0) + + +@set_module('mxnet.symbol.numpy') +def vdot(a, b): + r""" + Return the dot product of two vectors. + Note that `vdot` handles multidimensional arrays differently than `dot`: + it does *not* perform a matrix product, but flattens input arguments + to 1-D vectors first. Consequently, it should only be used for vectors. + + Parameters + ---------- + a : _Symbol + First argument to the dot product. + b : _Symbol + Second argument to the dot product. + + Returns + ------- + output : _Symbol + Dot product of `a` and `b`. + + See Also + -------- + dot : Return the dot product without using the complex conjugate of the + first argument. + + Examples + -------- + Note that higher-dimensional arrays are flattened! + >>> a = np.array([[1, 4], [5, 6]]) + >>> b = np.array([[4, 1], [2, 2]]) + >>> np.vdot(a, b) + 30 + >>> np.vdot(b, a) + 30 + >>> 1*4 + 4*1 + 5*2 + 6*2 + 30 + """ + return tensordot(a.flatten(), b.flatten(), 1) + + @set_module('mxnet.symbol.numpy') def equal(x1, x2, out=None): """ diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py index 494311ec0035..7068bf20d897 100644 --- a/tests/python/unittest/test_numpy_op.py +++ b/tests/python/unittest/test_numpy_op.py @@ -280,6 +280,201 @@ def dldx(x1, x2): assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-1, atol=1e-1) +@with_seed() +@use_np +def test_np_vdot(): + class TestVdot(HybridBlock): + def __init__(self): + super(TestVdot, self).__init__() + + def hybrid_forward(self, F, a, b): + return F.np.vdot(a, b) + + def vdot_backward(a, b): + return [b, a] + + # test different size inputs + tensor_shapes = [(), (5,), (3, 3)] + + for hybridize in [True, False]: + for shape in tensor_shapes: + for dtype in [_np.float32, _np.float64]: + test_vdot = TestVdot() + if hybridize: + test_vdot.hybridize() + a = rand_ndarray(shape=shape, dtype=dtype).as_np_ndarray() + b = rand_ndarray(shape=shape, dtype=dtype).as_np_ndarray() + a.attach_grad() + b.attach_grad() + + np_out = _np.vdot(a.asnumpy(), b.asnumpy()) + with mx.autograd.record(): + mx_out = test_vdot(a, b) + assert mx_out.shape == np_out.shape + assert_almost_equal(mx_out.asnumpy(), np_out, rtol = 1e-3, atol = 1e-5) + mx_out.backward() + np_backward = vdot_backward(a.asnumpy(), b.asnumpy()) + assert_almost_equal(a.grad.asnumpy(), np_backward[0], rtol = 1e-2, atol=1e-2) + assert_almost_equal(b.grad.asnumpy(), np_backward[1], rtol = 1e-2, atol=1e-2) + + # Test imperative once again + mx_out = np.vdot(a, b) + np_out = _np.vdot(a.asnumpy(), b.asnumpy()) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) + + # test numeric gradient + if len(shape) > 0 and _np.prod(shape) > 0: + a_sym = mx.sym.Variable("a").as_np_ndarray() + b_sym = mx.sym.Variable("b").as_np_ndarray() + mx_sym = mx.sym.np.vdot(a_sym, b_sym).as_nd_ndarray() + check_numeric_gradient(mx_sym, [a.as_nd_ndarray(), b.as_nd_ndarray()], + rtol=1e-1, atol=1e-1, dtype=dtype) + + +@with_seed() +@use_np +def test_np_inner(): + class TestInner(HybridBlock): + def __init__(self): + super(TestInner, self).__init__() + + def hybrid_forward(self, F, a, b): + return F.np.inner(a, b) + + def inner_backward(a, b): + a_axes_summed = [a.ndim - 1] + b_axes_summed = [b.ndim - 1] + + a_axes_remained = [] + for i in range(a.ndim): + if not (i in a_axes_summed): + a_axes_remained.append(i) + a_axes = a_axes_remained[:] + a_axes_summed[:] + + b_axes_remained = [] + for i in range(b.ndim): + if not (i in b_axes_summed): + b_axes_remained.append(i) + b_axes = b_axes_summed[:] + b_axes_remained[:] + + ad1 = _np.prod([a.shape[i] for i in a_axes_remained]) if len(a_axes_remained) > 0 else 1 + ad2 = _np.prod([a.shape[i] for i in a_axes_summed]) if len(a_axes_summed) > 0 else 1 + bd1 = _np.prod([b.shape[i] for i in b_axes_summed]) if len(b_axes_summed) > 0 else 1 + bd2 = _np.prod([b.shape[i] for i in b_axes_remained]) if len(b_axes_remained) > 0 else 1 + + out_grad = _np.ones((ad1, bd2)) + + new_a = _np.transpose(a, a_axes) + new_a_shape = new_a.shape[:] + new_a = new_a.reshape((ad1, ad2)) + new_b = _np.transpose(b, b_axes) + new_b_shape = new_b.shape[:] + new_b = new_b.reshape((bd1, bd2)) + + reverse_a_axes = [0 for i in a_axes] + for i in range(len(a_axes)): + reverse_a_axes[a_axes[i]] = i + + reverse_b_axes = [0 for i in b_axes] + for i in range(len(b_axes)): + reverse_b_axes[b_axes[i]] = i + + grad_b = _np.dot(new_a.T, out_grad).reshape(new_b_shape) + grad_b = _np.transpose(grad_b, reverse_b_axes) + grad_a = _np.dot(out_grad, new_b.T).reshape(new_a_shape) + grad_a = _np.transpose(grad_a, reverse_a_axes) + + return [grad_a, grad_b] + + # test non zero size input + tensor_shapes = [ + ((3,), (3,)), + ((2, 3), (3,)), + ((3,), (2, 3)) + ] + + for hybridize in [True, False]: + for a_shape, b_shape in tensor_shapes: + for dtype in [_np.float32, _np.float64]: + test_inner = TestInner() + if hybridize: + test_inner.hybridize() + a = rand_ndarray(shape=a_shape, dtype=dtype).as_np_ndarray() + b = rand_ndarray(shape=b_shape, dtype=dtype).as_np_ndarray() + a.attach_grad() + b.attach_grad() + + np_out = _np.inner(a.asnumpy(), b.asnumpy()) + with mx.autograd.record(): + mx_out = test_inner(a, b) + assert mx_out.shape == np_out.shape + assert_almost_equal(mx_out.asnumpy(), np_out, rtol = 1e-3, atol = 1e-5) + mx_out.backward() + np_backward = inner_backward(a.asnumpy(), b.asnumpy()) + assert_almost_equal(a.grad.asnumpy(), np_backward[0], rtol = 1e-2, atol=1e-2) + assert_almost_equal(b.grad.asnumpy(), np_backward[1], rtol = 1e-2, atol=1e-2) + + # Test imperative once again + mx_out = np.inner(a, b) + np_out = _np.inner(a.asnumpy(), b.asnumpy()) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) + + # test numeric gradient + a_sym = mx.sym.Variable("a").as_np_ndarray() + b_sym = mx.sym.Variable("b").as_np_ndarray() + mx_sym = mx.sym.np.inner(a_sym, b_sym).as_nd_ndarray() + check_numeric_gradient(mx_sym, [a.as_nd_ndarray(), b.as_nd_ndarray()], + rtol=1e-1, atol=1e-1, dtype=dtype) + + +@with_seed() +@use_np +def test_np_outer(): + class TestOuter(HybridBlock): + def __init__(self): + super(TestOuter, self).__init__() + + def hybrid_forward(self, F, a, b): + return F.np.outer(a, b) + + # test non zero size input + tensor_shapes = [ + ((3,), (3,)), + ((2, 3), (6,)), + ((6,), (2, 3)) + ] + + for hybridize in [True, False]: + for a_shape, b_shape in tensor_shapes: + for dtype in [_np.float32, _np.float64]: + test_outer = TestOuter() + if hybridize: + test_outer.hybridize() + a = rand_ndarray(shape=a_shape, dtype=dtype).as_np_ndarray() + b = rand_ndarray(shape=b_shape, dtype=dtype).as_np_ndarray() + a.attach_grad() + b.attach_grad() + + np_out = _np.outer(a.asnumpy(), b.asnumpy()) + with mx.autograd.record(): + mx_out = test_outer(a, b) + assert mx_out.shape == np_out.shape + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) + mx_out.backward() + + # Test imperative once again + mx_out = np.outer(a, b) + np_out = _np.outer(a.asnumpy(), b.asnumpy()) + assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5) + + # test numeric gradient + a_sym = mx.sym.Variable("a").as_np_ndarray() + b_sym = mx.sym.Variable("b").as_np_ndarray() + mx_sym = mx.sym.np.outer(a_sym, b_sym).as_nd_ndarray() + check_numeric_gradient(mx_sym, [a.as_nd_ndarray(), b.as_nd_ndarray()], + rtol=1e-1, atol=1e-1, dtype=dtype) + + @with_seed() @use_np def test_np_sum(): From 2208042bc333ffcc7d9a619fc1e4e01ff6182766 Mon Sep 17 00:00:00 2001 From: reminisce Date: Wed, 9 Oct 2019 15:08:04 -0700 Subject: [PATCH 5/5] Revert "Delete non-windows test" This reverts commit c91587706e74e903c58cc47ff0a01d1ab8171fdf. --- ci/jenkins/Jenkinsfile_centos_cpu | 53 ++++++++++++++ ci/jenkins/Jenkinsfile_centos_gpu | 51 +++++++++++++ ci/jenkins/Jenkinsfile_clang | 51 +++++++++++++ ci/jenkins/Jenkinsfile_edge | 56 +++++++++++++++ ci/jenkins/Jenkinsfile_miscellaneous | 54 ++++++++++++++ ci/jenkins/Jenkinsfile_sanity | 48 +++++++++++++ ci/jenkins/Jenkinsfile_unix_cpu | 80 +++++++++++++++++++++ ci/jenkins/Jenkinsfile_unix_gpu | 76 ++++++++++++++++++++ ci/jenkins/Jenkinsfile_website | 48 +++++++++++++ ci/jenkins/Jenkinsfile_website_c_docs | 52 ++++++++++++++ ci/jenkins/Jenkinsfile_website_clojure_docs | 53 ++++++++++++++ ci/jenkins/Jenkinsfile_website_full | 67 +++++++++++++++++ ci/jenkins/Jenkinsfile_website_full_pr | 62 ++++++++++++++++ ci/jenkins/Jenkinsfile_website_java_docs | 53 ++++++++++++++ ci/jenkins/Jenkinsfile_website_jekyll_docs | 49 +++++++++++++ ci/jenkins/Jenkinsfile_website_julia_docs | 53 ++++++++++++++ ci/jenkins/Jenkinsfile_website_mxnet_build | 48 +++++++++++++ ci/jenkins/Jenkinsfile_website_python_docs | 53 ++++++++++++++ ci/jenkins/Jenkinsfile_website_r_docs | 53 ++++++++++++++ ci/jenkins/Jenkinsfile_website_scala_docs | 53 ++++++++++++++ 20 files changed, 1113 insertions(+) create mode 100644 ci/jenkins/Jenkinsfile_centos_cpu create mode 100644 ci/jenkins/Jenkinsfile_centos_gpu create mode 100644 ci/jenkins/Jenkinsfile_clang create mode 100644 ci/jenkins/Jenkinsfile_edge create mode 100644 ci/jenkins/Jenkinsfile_miscellaneous create mode 100644 ci/jenkins/Jenkinsfile_sanity create mode 100644 ci/jenkins/Jenkinsfile_unix_cpu create mode 100644 ci/jenkins/Jenkinsfile_unix_gpu create mode 100644 ci/jenkins/Jenkinsfile_website create mode 100644 ci/jenkins/Jenkinsfile_website_c_docs create mode 100644 ci/jenkins/Jenkinsfile_website_clojure_docs create mode 100644 ci/jenkins/Jenkinsfile_website_full create mode 100644 ci/jenkins/Jenkinsfile_website_full_pr create mode 100644 ci/jenkins/Jenkinsfile_website_java_docs create mode 100644 ci/jenkins/Jenkinsfile_website_jekyll_docs create mode 100644 ci/jenkins/Jenkinsfile_website_julia_docs create mode 100644 ci/jenkins/Jenkinsfile_website_mxnet_build create mode 100644 ci/jenkins/Jenkinsfile_website_python_docs create mode 100644 ci/jenkins/Jenkinsfile_website_r_docs create mode 100644 ci/jenkins/Jenkinsfile_website_scala_docs diff --git a/ci/jenkins/Jenkinsfile_centos_cpu b/ci/jenkins/Jenkinsfile_centos_cpu new file mode 100644 index 000000000000..a47ab3de7fb7 --- /dev/null +++ b/ci/jenkins/Jenkinsfile_centos_cpu @@ -0,0 +1,53 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 180 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_centos7_cpu(), + custom_steps.compile_centos7_cpu_mkldnn() + ]) + + utils.parallel_stage('Tests', [ + custom_steps.test_centos7_python3_cpu(), + custom_steps.test_centos7_scala_cpu() + ]) +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_centos_gpu b/ci/jenkins/Jenkinsfile_centos_gpu new file mode 100644 index 000000000000..cad77a9a7dd8 --- /dev/null +++ b/ci/jenkins/Jenkinsfile_centos_gpu @@ -0,0 +1,51 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 180 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_centos7_gpu() + ]) + + utils.parallel_stage('Tests', [ + custom_steps.test_centos7_python3_gpu() + ]) +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_clang b/ci/jenkins/Jenkinsfile_clang new file mode 100644 index 000000000000..029c7208107b --- /dev/null +++ b/ci/jenkins/Jenkinsfile_clang @@ -0,0 +1,51 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 180 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_clang_3_9_cpu(), + custom_steps.compile_unix_clang_6_cpu(), + custom_steps.compile_unix_clang_tidy_cpu(), + custom_steps.compile_unix_clang_3_9_mkldnn_cpu(), + custom_steps.compile_unix_clang_6_mkldnn_cpu() + ]) +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_edge b/ci/jenkins/Jenkinsfile_edge new file mode 100644 index 000000000000..9d8e01399d7c --- /dev/null +++ b/ci/jenkins/Jenkinsfile_edge @@ -0,0 +1,56 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 180 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_armv8_jetson_gpu(), + custom_steps.compile_armv7_cpu(), + custom_steps.compile_armv6_cpu(), + custom_steps.compile_armv8_cpu(), + custom_steps.compile_armv8_android_cpu(), + custom_steps.compile_armv7_android_cpu() + ]) + + utils.parallel_stage('Tests', [ + custom_steps.test_qemu_armv7_cpu() + ]) +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_miscellaneous b/ci/jenkins/Jenkinsfile_miscellaneous new file mode 100644 index 000000000000..dbf2a9e41c76 --- /dev/null +++ b/ci/jenkins/Jenkinsfile_miscellaneous @@ -0,0 +1,54 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 180 + + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_asan_cpu(), + custom_steps.compile_unix_amalgamation_min(), + custom_steps.compile_unix_amalgamation() + ]) + + utils.parallel_stage('Tests', [ + custom_steps.misc_asan_cpu() + ]) +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_sanity b/ci/jenkins/Jenkinsfile_sanity new file mode 100644 index 000000000000..ed4d16ec47db --- /dev/null +++ b/ci/jenkins/Jenkinsfile_sanity @@ -0,0 +1,48 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 180 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Sanity Check', [ + custom_steps.sanity_lint(), + custom_steps.sanity_rat_license() + ]) +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu new file mode 100644 index 000000000000..3533a123ea86 --- /dev/null +++ b/ci/jenkins/Jenkinsfile_unix_cpu @@ -0,0 +1,80 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 240 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_cpu_openblas(), + custom_steps.compile_unix_openblas_debug_cpu(), + custom_steps.compile_unix_mkl_cpu(), + custom_steps.compile_unix_mkldnn_cpu(), + custom_steps.compile_unix_mkldnn_mkl_cpu(), + custom_steps.compile_unix_int64_cpu() + ]) + + utils.parallel_stage('Tests', [ + custom_steps.test_unix_python2_cpu(), + custom_steps.test_unix_python3_cpu(), + custom_steps.test_unix_python3_debug_cpu(), + custom_steps.test_unix_python3_mkl_cpu(), + custom_steps.test_unix_python2_mkldnn_cpu(), + custom_steps.test_unix_python3_mkldnn_cpu(), + custom_steps.test_unix_python3_mkldnn_mkl_cpu(), + custom_steps.test_unix_scala_cpu(), + custom_steps.test_unix_scala_mkldnn_cpu(), + custom_steps.test_unix_clojure_cpu(), + custom_steps.test_unix_clojure_integration_cpu(), + custom_steps.test_unix_perl_cpu(), + custom_steps.test_unix_r_cpu(), + custom_steps.test_unix_r_mkldnn_cpu(), + custom_steps.test_unix_julia07_cpu(), + custom_steps.test_unix_julia10_cpu(), + custom_steps.test_unix_onnx_cpu(), + custom_steps.test_unix_cpp_cpu(), + custom_steps.test_static_scala_cpu(), + custom_steps.test_static_python_cpu(), + /* Disabled due to master build failure: + * http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/ + * https://github.com/apache/incubator-mxnet/issues/11801 + custom_steps.test_unix_distributed_kvstore_cpu() + */ + ]) +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_unix_gpu b/ci/jenkins/Jenkinsfile_unix_gpu new file mode 100644 index 000000000000..e2a089b0469b --- /dev/null +++ b/ci/jenkins/Jenkinsfile_unix_gpu @@ -0,0 +1,76 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 180 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_mkldnn_gpu(), + custom_steps.compile_unix_mkldnn_nocudnn_gpu(), + custom_steps.compile_unix_full_gpu(), + custom_steps.compile_unix_cmake_mkldnn_gpu(), + custom_steps.compile_unix_cmake_gpu(), + custom_steps.compile_unix_tensorrt_gpu(), + custom_steps.compile_unix_int64_gpu() + ]) + + utils.parallel_stage('Tests', [ + custom_steps.test_unix_python2_gpu(), + custom_steps.test_unix_python3_gpu(), + custom_steps.test_unix_python2_quantize_gpu(), + custom_steps.test_unix_python3_quantize_gpu(), + custom_steps.test_unix_python2_mkldnn_gpu(), + custom_steps.test_unix_python3_mkldnn_gpu(), + custom_steps.test_unix_python3_mkldnn_nocudnn_gpu(), + custom_steps.test_unix_python3_tensorrt_gpu(), + custom_steps.test_unix_perl_gpu(), + custom_steps.test_unix_r_gpu(), + custom_steps.test_unix_cpp_gpu(), + custom_steps.test_unix_cpp_mkldnn_gpu(), + custom_steps.test_unix_python3_integration_gpu(), + custom_steps.test_unix_cpp_package_gpu(), + custom_steps.test_unix_scala_gpu(), + custom_steps.test_unix_distributed_kvstore_gpu(), + custom_steps.test_static_python_gpu() + + // Disabled due to: https://github.com/apache/incubator-mxnet/issues/11407 + //custom_steps.test_unix_caffe_gpu() + ]) +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_website b/ci/jenkins/Jenkinsfile_website new file mode 100644 index 000000000000..f6c853046b1c --- /dev/null +++ b/ci/jenkins/Jenkinsfile_website @@ -0,0 +1,48 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 180 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Placeholder', [ + // Do nothing + ]) + +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_website_c_docs b/ci/jenkins/Jenkinsfile_website_c_docs new file mode 100644 index 000000000000..e678920e3a4f --- /dev/null +++ b/ci/jenkins/Jenkinsfile_website_c_docs @@ -0,0 +1,52 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 20 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_lite() + ]) + + utils.parallel_stage('C Docs', [ + custom_steps.docs_c() + ]) + +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_website_clojure_docs b/ci/jenkins/Jenkinsfile_website_clojure_docs new file mode 100644 index 000000000000..697dca48e58b --- /dev/null +++ b/ci/jenkins/Jenkinsfile_website_clojure_docs @@ -0,0 +1,53 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 20 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_lite() + ]) + + utils.parallel_stage('Clojure Docs', [ + custom_steps.docs_clojure() + + ]) + +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_website_full b/ci/jenkins/Jenkinsfile_website_full new file mode 100644 index 000000000000..39cc6f4e2dc9 --- /dev/null +++ b/ci/jenkins/Jenkinsfile_website_full @@ -0,0 +1,67 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 180 + +node('restricted-utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} + +utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_lite() + ]) + + utils.parallel_stage('Build Docs', [ + custom_steps.docs_jekyll(), + custom_steps.docs_c(), + custom_steps.docs_python(), + custom_steps.docs_julia(), + custom_steps.docs_r(), + custom_steps.docs_scala(), + custom_steps.docs_java(), + custom_steps.docs_clojure() + ]) + + utils.parallel_stage('Prepare', [ + custom_steps.docs_prepare() + ]) + + utils.parallel_stage('Publish', [ + custom_steps.docs_publish() + ]) +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_website_full_pr b/ci/jenkins/Jenkinsfile_website_full_pr new file mode 100644 index 000000000000..133c6c204964 --- /dev/null +++ b/ci/jenkins/Jenkinsfile_website_full_pr @@ -0,0 +1,62 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 180 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_lite() + ]) + + utils.parallel_stage('Build Docs', [ + // Optimization would be to flag these not to stash if not previewing them + custom_steps.docs_jekyll(), + custom_steps.docs_c(), + custom_steps.docs_python(), + custom_steps.docs_julia(), + custom_steps.docs_r(), + custom_steps.docs_scala(), + custom_steps.docs_java(), + custom_steps.docs_clojure() + ]) + + // TODO: add a website preview function + +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_website_java_docs b/ci/jenkins/Jenkinsfile_website_java_docs new file mode 100644 index 000000000000..4453b444039b --- /dev/null +++ b/ci/jenkins/Jenkinsfile_website_java_docs @@ -0,0 +1,53 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 20 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_lite() + ]) + + utils.parallel_stage('Java Docs', [ + custom_steps.docs_java() + + ]) + +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_website_jekyll_docs b/ci/jenkins/Jenkinsfile_website_jekyll_docs new file mode 100644 index 000000000000..c4c5ff89dd86 --- /dev/null +++ b/ci/jenkins/Jenkinsfile_website_jekyll_docs @@ -0,0 +1,49 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 20 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Jekyll Website Docs', [ + custom_steps.docs_jekyll() + + ]) + +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_website_julia_docs b/ci/jenkins/Jenkinsfile_website_julia_docs new file mode 100644 index 000000000000..e1e9aaa983fb --- /dev/null +++ b/ci/jenkins/Jenkinsfile_website_julia_docs @@ -0,0 +1,53 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 60 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_lite() + ]) + + utils.parallel_stage('Julia Docs', [ + custom_steps.docs_julia() + + ]) + +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_website_mxnet_build b/ci/jenkins/Jenkinsfile_website_mxnet_build new file mode 100644 index 000000000000..a3c3330d0349 --- /dev/null +++ b/ci/jenkins/Jenkinsfile_website_mxnet_build @@ -0,0 +1,48 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 180 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_lite() + ]) + +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_website_python_docs b/ci/jenkins/Jenkinsfile_website_python_docs new file mode 100644 index 000000000000..baaf4519541f --- /dev/null +++ b/ci/jenkins/Jenkinsfile_website_python_docs @@ -0,0 +1,53 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 60 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_lite() + ]) + + utils.parallel_stage('Python Docs', [ + custom_steps.docs_python() + + ]) + +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_website_r_docs b/ci/jenkins/Jenkinsfile_website_r_docs new file mode 100644 index 000000000000..5b9f6630563f --- /dev/null +++ b/ci/jenkins/Jenkinsfile_website_r_docs @@ -0,0 +1,53 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 60 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_lite() + ]) + + utils.parallel_stage('R Docs', [ + custom_steps.docs_r() + + ]) + +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/ci/jenkins/Jenkinsfile_website_scala_docs b/ci/jenkins/Jenkinsfile_website_scala_docs new file mode 100644 index 000000000000..6a083dae7957 --- /dev/null +++ b/ci/jenkins/Jenkinsfile_website_scala_docs @@ -0,0 +1,53 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 20 + +node('utility') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') + custom_steps = load('ci/jenkins/Jenkins_steps.groovy') +} +utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu') + +utils.main_wrapper( +core_logic: { + utils.parallel_stage('Build', [ + custom_steps.compile_unix_lite() + ]) + + utils.parallel_stage('Scala Docs', [ + custom_steps.docs_scala() + + ]) + +} +, +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +)