Add support for Amazon EC2 spot instances

.gitignore

@@ -19,3 +19,5 @@ target
 core/.xml
 felix-cache
 .metadata
+test-support/.externalToolBuilders
+test-support/maven-eclipse.xml

karaf/commands/src/main/java/com/axemblr/provisionr/commands/CreatePoolCommand.java

@@ -16,6 +16,9 @@
 
 package com.axemblr.provisionr.commands;
 
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import com.axemblr.provisionr.api.Provisionr;
 import com.axemblr.provisionr.api.access.AdminAccess;
 import com.axemblr.provisionr.api.hardware.Hardware;
@@ -29,18 +32,21 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Charsets;
 import com.google.common.base.Optional;
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import com.google.common.io.Files;
+
 import java.io.File;
 import java.util.List;
+import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Set;
+
 import org.apache.felix.gogo.commands.Command;
 import org.apache.felix.gogo.commands.Option;
 import org.apache.karaf.shell.console.OsgiCommandSupport;
@@ -69,6 +75,11 @@ public class CreatePoolCommand extends OsgiCommandSupport {
     @Option(name = "-h", aliases = "--hardware-type", description = "Virtual machine hardware type")
     private String hardwareType = "t1.micro";
 
+    @Option(name = "-o", aliases = "--provider-options", description = "Provider-specific options (multi-valued)." +
+        "Expects either the key=value format or just plain key. If value is not specified, defaults to 'true'." +
+        "Supported values: spotBid=x.xxx (Amazon).", multiValued = true)
+    private List<String> providerOptions = Lists.newArrayList();
+
     @Option(name = "--port", description = "Firewall port that need to be open for any TCP traffic " +
         "(multi-valued). SSH (22) is always open by default.", multiValued = true)
     private List<Integer> ports = Lists.newArrayList();
@@ -110,6 +121,14 @@ Pool createPoolFromArgumentsAndServiceDefaults(Provisionr service) {
         checkArgument(defaultProvider.isPresent(), String.format("please configure a default provider " +
             "by editing etc/com.axemblr.provisionr.%s.cfg", id));
 
+        /* append the provider options that were passed in and rebuild the default provider */
+        // TODO: this currently does not support overriding default options, it will throw an exception
+        Map<String,String> options = ImmutableMap.<String, String>builder()
+                .putAll(defaultProvider.get().getOptions())     // default options
+                .putAll(parseProviderOptions(providerOptions))  // options added by the user
+                .build();
+        Provider provider = defaultProvider.get().toBuilder().options(options).createProvider();
+
         /* Always allow ICMP and ssh traffic by default */
         final Network network = Network.builder().addRules(
             Rule.builder().anySource().icmp().createRule(),
@@ -122,8 +141,9 @@ Pool createPoolFromArgumentsAndServiceDefaults(Provisionr service) {
 
         final Software software = Software.builder().packages(packages).createSoftware();
 
+
         final Pool pool = Pool.builder()
-            .provider(defaultProvider.get())
+            .provider(provider)
             .hardware(hardware)
             .software(software)
             .network(network)
@@ -145,6 +165,16 @@ Pool createPoolFromArgumentsAndServiceDefaults(Provisionr service) {
         return pool;
     }
 
+    private Map<String, String> parseProviderOptions(List<String> providerOptions) {
+        Map<String, String> result = Maps.newHashMap();
+        for (String option : providerOptions) {
+            String[] parts = option.split("=");
+            String value = parts.length > 1 ? parts[1] : "true";
+            result.put(parts[0], value);
+        }
+        return result;
+    }
+
     private Set<Rule> formatPortsAsIngressRules() {
         ImmutableSet.Builder<Rule> rules = ImmutableSet.builder();
         for (int port : ports) {
@@ -203,6 +233,11 @@ void setPackages(List<String> packages) {
         this.packages = ImmutableList.copyOf(packages);
     }
 
+    @VisibleForTesting
+    void setProviderOptions(List<String> providerOptions) {
+        this.providerOptions = ImmutableList.copyOf(providerOptions);
+    }
+
     @VisibleForTesting
     void setCacheBaseImage(boolean cacheBaseImage) {
         this.cacheBaseImage = cacheBaseImage;

karaf/commands/src/test/java/com/axemblr/provisionr/commands/CreatePoolCommandTest.java

@@ -16,27 +16,33 @@
 
 package com.axemblr.provisionr.commands;
 
+import static org.fest.assertions.api.Assertions.assertThat;
+import static org.mockito.Matchers.anyMapOf;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
 import com.axemblr.provisionr.api.Provisionr;
 import com.axemblr.provisionr.api.access.AdminAccess;
 import com.axemblr.provisionr.api.pool.Pool;
 import com.axemblr.provisionr.api.provider.Provider;
-import com.axemblr.provisionr.core.templates.PoolTemplate;
+import com.axemblr.provisionr.api.provider.ProviderBuilder;
 import com.axemblr.provisionr.core.templates.xml.XmlTemplate;
+import com.axemblr.provisionr.core.templates.PoolTemplate;
 import com.google.common.base.Charsets;
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
 import com.google.common.io.Resources;
+
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.NoSuchElementException;
+
 import org.apache.felix.service.command.CommandSession;
-import static org.fest.assertions.api.Assertions.assertThat;
 import org.junit.Test;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 
 public class CreatePoolCommandTest {
 
@@ -94,14 +100,39 @@ protected AdminAccess collectCurrentUserCredentialsForAdminAccess() {
         command.setTemplate(template.getId());
 
         Provisionr service = mock(Provisionr.class);
-        when(service.getDefaultProvider()).thenReturn(Optional.of(mock(Provider.class)));
+        Provider provider = mock(Provider.class);
+        ProviderBuilder providerBuilder = mock(ProviderBuilder.class);
+        when(providerBuilder.options(anyMapOf(String.class, String.class))).thenReturn(providerBuilder);
+        when(providerBuilder.createProvider()).thenReturn(provider);
+        when(provider.toBuilder()).thenReturn(providerBuilder);
+
+        when(service.getDefaultProvider()).thenReturn(Optional.of(provider));
 
         Pool pool = command.createPoolFromArgumentsAndServiceDefaults(service);
 
         assertThat(pool.getSoftware().getRepositories()).hasSize(1);
         assertThat(pool.getSoftware().getPackages()).contains("jenkins").contains("git-core");
     }
 
+    @Test
+    public void testProviderSpecificOptions() {
+        CreatePoolCommand command = new CreatePoolCommand(Collections.<Provisionr>emptyList(),
+            Collections.<PoolTemplate>emptyList());
+        command.setId("service");
+        command.setKey("key");
+        command.setProviderOptions(Lists.newArrayList("spotBid=0.07"));
+
+        Provisionr service = mock(Provisionr.class);
+        // TODO: consider refactoring this with an argument captor instead of 
+        // using an actual object
+        Provider provider = new ProviderBuilder().id("id").endpoint("endpoint")
+                .accessKey("aKey").secretKey("sKey").createProvider();
+        when(service.getDefaultProvider()).thenReturn(Optional.of(provider));
+
+        Pool pool = command.createPoolFromArgumentsAndServiceDefaults(service);
+        assertThat(pool.getProvider().getOption("spotBid")).isEqualTo("0.07");
+    }
+
     private Provisionr newProvisionrMockWithId(String id) {
         Provisionr service = mock(Provisionr.class);
         when(service.getId()).thenReturn(id);

providers/amazon-tests/src/test/java/com/axemblr/provisionr/amazon/AmazonProvisionrLiveTest.java

@@ -82,16 +82,31 @@ public Option[] configuration() throws Exception {
     }
 
     @Test
-    public void startProvisioningProcess() throws Exception {
+    public void startProvisioningProcessForOnDemandInstances() throws Exception {
+        startProvisioningProcess(null);
+    }
+
+    @Test
+    public void startProvisioningProcessForSpotInstances() throws Exception {
+        startProvisioningProcess("0.04");
+    }
+
+    private void startProvisioningProcess(String spotBid) throws Exception {
         waitForProcessDeployment(AmazonProvisionr.MANAGEMENT_PROCESS_KEY);
 
         final Provisionr provisionr = getOsgiService(Provisionr.class, 5000);
 
-        final Provider provider = collectProviderCredentialsFromSystemProperties()
+        Provider provider = collectProviderCredentialsFromSystemProperties()
             .option(ProviderOptions.REGION, getProviderProperty(
                 ProviderOptions.REGION, ProviderOptions.DEFAULT_REGION))
             .createProvider();
 
+        if (spotBid != null) {
+            provider = provider.toBuilder()
+                    .option(ProviderOptions.SPOT_BID, spotBid)
+                    .createProvider();
+        }
+
         final Network network = Network.builder().addRules(
             Rule.builder().anySource().icmp().createRule(),
             Rule.builder().anySource().port(22).protocol(Protocol.TCP).createRule()

providers/amazon/src/main/java/com/axemblr/provisionr/amazon/AmazonProvisionr.java

@@ -17,6 +17,7 @@
 package com.axemblr.provisionr.amazon;
 
 import com.axemblr.provisionr.amazon.config.DefaultProviderConfig;
+import com.axemblr.provisionr.amazon.options.ProviderOptions;
 import com.axemblr.provisionr.api.pool.Machine;
 import com.axemblr.provisionr.api.pool.Pool;
 import com.axemblr.provisionr.api.provider.Provider;
@@ -75,6 +76,9 @@ public String startPoolManagementProcess(String businessKey, Pool pool) {
         arguments.put(CoreProcessVariables.POOL, pool);
         arguments.put(CoreProcessVariables.PROVIDER, getId());
         arguments.put(CoreProcessVariables.POOL_BUSINESS_KEY, businessKey);
+
+        /* needed because the Activiti EL doesn't work as expected and properties can't be read from the pool. */
+        arguments.put(ProcessVariables.SPOT_BID, pool.getProvider().getOption(ProviderOptions.SPOT_BID));
 
         /* Authenticate as kermit to make the process visible in the Explorer UI */
         processEngine.getIdentityService().setAuthenticatedUserId(CoreConstants.ACTIVITI_EXPLORER_DEFAULT_USER);

providers/amazon/src/main/java/com/axemblr/provisionr/amazon/ProcessVariables.java

@@ -34,6 +34,48 @@ private ProcessVariables() {
      */
     public static final String RESERVATION_ID = "reservationId";
 
+    /**
+     * The amount the user is willing to pay for spot instances in the
+     * Amazon pool he's trying to start. If set, the request is for spot
+     * instances, if null the request is for on demand instances.
+     * 
+     * @see com.axemblr.provisionr.amazon.activities.RunSpotInstances
+     */
+    public static final String SPOT_BID = "spotBid";
+
+    /**
+     * Flag that gets set when the process attempts to send spot requests
+     * for the first time. Because the describe call is not consistent
+     * until a reasonable delay passes, this will be used to timeout the
+     * Activiti retries so that the requests are not resent if they were
+     * successful. 
+     * 
+     * @see com.axemblr.provisionr.amazon.activities.RunSpotInstances
+     */
+    public static final String SPOT_REQUESTS_SENT = "spotRequestsSent";
+
+    /**
+     * List of request IDs as returned by Amazon for spot instances. These need to 
+     * be followed up to get the actual instance IDs.
+     * 
+     * @see com.axemblr.provisionr.amazon.activities.RunSpotInstances
+     */
+    public static final String SPOT_INSTANCE_REQUEST_IDS = "spotInstanceRequestIds";
+
+    /**
+     * Have all spot instance requests been handled by Amazon? (none are pending)
+     * 
+     *  @see com.axemblr.provisionr.amazon.activities.CheckNoRequestsAreOpen
+     */
+    public static final String NO_SPOT_INSTANCE_REQUESTS_OPEN = "noSpotInstanceRequestsOpen";
+
+    /**
+     * Are all spot instance requests in an active state? (none cancelled, none terminated)
+     * 
+     * @see com.axemblr.provisionr.amazon.activities.CheckAllRequestsAreActive
+     */
+    public static final String ALL_SPOT_INSTANCE_REQUESTS_ACTIVE = "allSpotInstanceRequestsActive";
+
     /**
      * List of instance IDs as returned by Amazon
      *

providers/amazon/src/main/java/com/axemblr/provisionr/amazon/activities/AllInstancesMatchPredicate.java

@@ -16,23 +16,29 @@
 
 package com.axemblr.provisionr.amazon.activities;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.List;
+
+import org.activiti.engine.delegate.DelegateExecution;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import com.amazonaws.AmazonServiceException;
 import com.amazonaws.services.ec2.AmazonEC2;
 import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
 import com.amazonaws.services.ec2.model.DescribeInstancesResult;
 import com.amazonaws.services.ec2.model.Instance;
+import com.amazonaws.services.ec2.model.Reservation;
+import com.amazonaws.services.ec2.model.SpotInstanceRequest;
 import com.axemblr.provisionr.amazon.ProcessVariables;
 import com.axemblr.provisionr.amazon.core.ProviderClientCache;
 import com.axemblr.provisionr.api.pool.Pool;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Preconditions.checkState;
+import com.google.common.base.Function;
 import com.google.common.base.Predicate;
 import com.google.common.base.Throwables;
 import com.google.common.collect.Iterables;
-import java.util.List;
-import org.activiti.engine.delegate.DelegateExecution;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
 
 public abstract class AllInstancesMatchPredicate extends AmazonActivity {
 
@@ -57,9 +63,9 @@ public void execute(AmazonEC2 client, Pool pool, DelegateExecution execution) th
         try {
             DescribeInstancesResult result = client.describeInstances(new DescribeInstancesRequest()
                 .withInstanceIds(instanceIds));
-            checkState(result.getReservations().size() == 1, "the instance ids are part of multiple reservations");
 
-            List<Instance> instances = result.getReservations().get(0).getInstances();
+            List<Instance> instances = collectInstancesFromReservations(result.getReservations());
+
             if (Iterables.all(instances, predicate)) {
                 LOG.info(">> All {} instances match predicate {} ", instanceIds, predicate);
                 execution.setVariable(resultVariable, true);