timeout option for property tests

peterzeller · May 24, 2019 · 258f2f5 · 258f2f5
1 parent 740245c
commit 258f2f5
Show file tree

Hide file tree

Showing 3 changed files with 82 additions and 38 deletions.
diff --git a/src/main/java/smallcheck/PropertyStatement.java b/src/main/java/smallcheck/PropertyStatement.java
@@ -12,11 +12,12 @@
 import smallcheck.generators.ParamGen;
 
 import java.lang.annotation.Annotation;
-import java.lang.reflect.AnnotatedElement;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Parameter;
+import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Stream;
 
 /**
@@ -56,29 +57,27 @@ public void evaluate() throws Throwable {
         Parameter[] parameters = m.getParameters();
         AtomicLong invocations = new AtomicLong(0);
         AtomicLong preConditionFailures = new AtomicLong(0);
+        AtomicReference<Object[]> lastArgs = new AtomicReference<>();
         try {
             int maxDepth = property.maxDepth();
             int maxInvocations = property.maxInvocations();
-            for (int depth = 0; depth <= maxDepth; depth++) {
-                Stream<Object[]> argStream = ParamGen.generate(genFactory, parameters, depth);
-                argStream.forEach(args -> {
-                    try {
-                        if (invocations.incrementAndGet() > maxInvocations) {
-                            throw new MaxInvocationsReached();
-                        }
-                        m.invoke(testInstance, args);
-                    } catch (IllegalAccessException e) {
-                        throw new RuntimeException(e);
-                    } catch (InvocationTargetException e) {
-                        Throwable cause = e.getCause();
-                        if (cause instanceof AssumptionViolatedException) {
-                            preConditionFailures.incrementAndGet();
-                            // ignore this case
-                            return;
-                        }
-                        throw new SmallcheckException(m, args, cause);
-                    }
-                });
+            int timeout = property.timeout();
+            if (timeout < 0) {
+                execute(m, parameters, invocations, preConditionFailures, maxDepth, maxInvocations, lastArgs);
+            } else {
+                ExecutorService executor = Executors.newSingleThreadExecutor();
+                Future f = executor.submit(() -> execute(m, parameters, invocations, preConditionFailures, maxDepth, maxInvocations, lastArgs));
+                executor.shutdown();
+                try {
+                    f.get(timeout, TimeUnit.SECONDS);
+                } catch (ExecutionException e) {
+                    throw e.getCause();
+                } catch (TimeoutException e) {
+                    throw new SmallcheckException(m, lastArgs.get(),
+                            new Exception("Test timed out after " + timeout + " seconds.", e));
+                } catch (Exception e) {
+                    throw new SmallcheckException(m, lastArgs.get(), e);
+                }
             }
         } catch (SmallcheckException e) {
             throw e.skipInternals();
@@ -93,6 +92,31 @@ public void evaluate() throws Throwable {
 //        System.out.println("execute " + this.method + " in " + testClass);
     }
 
+    private void execute(Method m, Parameter[] parameters, AtomicLong invocations, AtomicLong preConditionFailures, int maxDepth, int maxInvocations, AtomicReference<Object[]> lastArgs) {
+        for (int depth = 0; depth <= maxDepth; depth++) {
+            Stream<Object[]> argStream = ParamGen.generate(genFactory, parameters, depth);
+            argStream.forEach(args -> {
+                lastArgs.set(args);
+                try {
+                    if (invocations.incrementAndGet() > maxInvocations) {
+                        throw new MaxInvocationsReached();
+                    }
+                    m.invoke(testInstance, args);
+                } catch (IllegalAccessException e) {
+                    throw new RuntimeException(e);
+                } catch (InvocationTargetException e) {
+                    Throwable cause = e.getCause();
+                    if (cause instanceof AssumptionViolatedException) {
+                        preConditionFailures.incrementAndGet();
+                        // ignore this case
+                        return;
+                    }
+                    throw new SmallcheckException(m, args, cause);
+                }
+            });
+        }
+    }
+
     private class MaxInvocationsReached extends RuntimeException {
     }
 }
diff --git a/src/main/java/smallcheck/annotations/Property.java b/src/main/java/smallcheck/annotations/Property.java
@@ -3,6 +3,7 @@
 import java.lang.annotation.Annotation;
 import java.lang.annotation.Retention;
 import java.lang.annotation.Target;
+import java.time.Duration;
 
 import static java.lang.annotation.ElementType.METHOD;
 import static java.lang.annotation.RetentionPolicy.RUNTIME;
@@ -15,4 +16,7 @@
     int maxInvocations() default 100000;
 
     long minExamples() default 20;
+
+    /** timeout in seconds */
+    int timeout() default -1;
 }
diff --git a/src/test/java/StandardTypes.java b/src/test/java/StandardTypes.java
@@ -3,25 +3,23 @@
  */
 
 import org.junit.Assume;
-import org.junit.Rule;
 import org.junit.Test;
-import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
-import smallcheck.annotations.Property;
 import smallcheck.SmallCheckRunner;
+import smallcheck.annotations.Property;
 
-import java.util.*;
-import java.util.stream.StreamSupport;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 
 @RunWith(SmallCheckRunner.class)
 public class StandardTypes {
 
-    @Rule
-    public Timeout globalTimeout = Timeout.seconds(1);
-
     @Test
     public void blub() {
         assertEquals(5, 2 + 2);
@@ -91,26 +89,44 @@ public void testAssume(List<String> x, List<String> y) {
 
     @Property
     public void testAssume2(List<String> x) {
-        System.out.println(x);
         Assume.assumeTrue(x.size() > 4);
         assertEquals(1, 2);
     }
 
-    enum X {
-        A, B, C
-    }
-
     @Property
     public void testEnum(Set<X> s) {
         assertTrue(s.size() < 3);
     }
 
-    @Property
+    @Property(timeout = 5)
     public void testIntArray(int[] ar) {
-        for (int i = 0; i < ar.length-1; i++) {
-            assertTrue(ar[i] <= ar[i+1]);
+        for (int i = 0; i < ar.length - 1; i++) {
+            assertTrue(ar[i] <= ar[i + 1]);
+        }
+    }
+
+    @Property(timeout = 5)
+    public void testIntArrayException(int[] ar) {
+        for (int i = 0; i < ar.length; i++) {
+            assertTrue(ar[i] <= ar[i + 1]);
         }
     }
 
+    @Property(timeout = 5)
+    public void testIntArrayInfiniteLoop(int[] ar) {
+        int sum = 0;
+        for (int i : ar) {
+            sum += i;
+        }
+        if (sum + ar.length > 6) {
+            while (true) {
+            }
+        }
+    }
+
+    enum X {
+        A, B, C
+    }
+
 
 }