Merge pull request #862 from shatterednirvana/set-max-memory-for-apps…

…ervers

Allow users to set max memory for AppServers

AppController/lib/app_controller_client.rb

@@ -170,10 +170,10 @@ def get_userappserver_ip(verbose_level="low")
     return userappserver_ip
   end
 
-  def set_parameters(locations, creds, apps_to_start)
+  def set_parameters(locations, options, apps_to_start)
     result = ""
     make_call(10, ABORT_ON_FAIL, "set_parameters") { 
-      result = conn.set_parameters(locations, creds, apps_to_start, @secret)
+      result = conn.set_parameters(locations, options, apps_to_start, @secret)
     }  
     HelperFunctions.log_and_crash(result) if result =~ /Error:/
   end

AppController/lib/app_manager_client.rb

@@ -97,6 +97,8 @@ def make_call(timeout, retry_on_except, callr)
    #   db_locations: An Array of datastore server IPs
    #   env_vars: A Hash of environemnt variables that should be passed to the
    #     application to start.
+   #   max_memory: An Integer that names the maximum amount of memory (in
+   #     megabytes) that should be used for this App Engine app.
    # Returns:
    #   The PID of the process started
    # Note:
@@ -110,14 +112,16 @@ def start_app(app_name,
                  language, 
                  xmpp_ip,
                  db_locations,
-                 env_vars)
+                 env_vars,
+                 max_memory=500)
     config = {'app_name' => app_name,
               'app_port' => app_port,
               'load_balancer_ip' => load_balancer_ip,
               'language' => language,
               'xmpp_ip' => xmpp_ip,
               'dblocations' => db_locations,
-              'env_vars' => env_vars}
+              'env_vars' => env_vars,
+              'max_memory' => max_memory}
     json_config = JSON.dump(config)
     result = ""
     make_call(MAX_TIME_OUT, false, "start_app") {

AppController/lib/helperfunctions.rb

@@ -596,10 +596,10 @@ def self.get_optimal_spot_price(instance_type)
   end
 
 
-  def self.set_creds_in_env(creds, cloud_num)
+  def self.set_options_in_env(options, cloud_num)
     ENV['EC2_JVM_ARGS'] = nil
 
-    creds.each_pair { |k, v|
+    options.each_pair { |k, v|
       next unless k =~ /\ACLOUD/
       env_key = k.scan(/\ACLOUD_(.*)\Z/).flatten.to_s
       ENV[env_key] = v
@@ -616,23 +616,23 @@ def self.set_creds_in_env(creds, cloud_num)
     Djinn.log_debug("Setting private key to #{cloud_keys_dir}/mykey.pem, cert to #{cloud_keys_dir}/mycert.pem")
   end
 
-  def self.spawn_hybrid_vms(creds, nodes)
-    info = "Spawning hybrid vms with creds #{self.obscure_creds(creds).inspect} and nodes #{nodes.inspect}"
+  def self.spawn_hybrid_vms(options, nodes)
+    info = "Spawning hybrid vms with options #{self.obscure_options(options).inspect} and nodes #{nodes.inspect}"
     Djinn.log_debug(info)
 
     cloud_info = []
 
     cloud_num = 1
     loop {
-      cloud_type = creds["CLOUD#{cloud_num}_TYPE"]
+      cloud_type = options["CLOUD#{cloud_num}_TYPE"]
       break if cloud_type.nil?
 
-      self.set_creds_in_env(creds, cloud_num)
+      self.set_options_in_env(options, cloud_num)
 
       if cloud_type == "euca"
-        machine = creds["CLOUD#{cloud_num}_EMI"]
+        machine = options["CLOUD#{cloud_num}_EMI"]
       elsif cloud_type == "ec2"
-        machine = creds["CLOUD#{cloud_num}_AMI"]
+        machine = options["CLOUD#{cloud_num}_AMI"]
       else
         self.log_and_crash("Cloud type was #{cloud_type}, which is not a supported value.")
       end
@@ -647,9 +647,9 @@ def self.spawn_hybrid_vms(creds, nodes)
       }
 
       instance_type = "m1.large"
-      keyname = creds["keyname"]
+      keyname = options["keyname"]
       cloud = "cloud#{cloud_num}"
-      group = creds["group"]
+      group = options["group"]
 
       this_cloud_info = self.spawn_vms(num_of_vms, jobs_needed, machine, 
         instance_type, keyname, cloud_type, cloud, group)
@@ -851,19 +851,19 @@ def self.terminate_vms(nodes, infrastructure)
     self.shell("#{infrastructure}-terminate-instances #{instances.join(' ')}")
   end
 
-  def self.terminate_hybrid_vms(creds)
+  def self.terminate_hybrid_vms(options)
     # TODO: kill my own cloud last
     # otherwise could orphan other clouds
 
     cloud_num = 1
     loop {
       key = "CLOUD#{cloud_num}_TYPE"
-      cloud_type = creds[key]
+      cloud_type = options[key]
       break if cloud_type.nil?
 
-      self.set_creds_in_env(creds, cloud_num)
+      self.set_options_in_env(options, cloud_num)
 
-      keyname = creds["keyname"]
+      keyname = options["keyname"]
       Djinn.log_debug("Killing Cloud#{cloud_num}'s machines, of type #{cloud_type} and with keyname #{keyname}")
       self.terminate_all_vms(cloud_type, keyname)
 
@@ -1208,18 +1208,18 @@ def self.obscure_array(array)
   # be too sensitive to log in cleartext. If any of these items are
   # found, a sanitized version of the item is returned in its place.
   # Args:
-  #   creds: The item to sanitize. As we are expecting Hashes here,
+  #   options: The item to sanitize. As we are expecting Hashes here,
   #     callers that pass in non-Hash items can expect no change to
   #     be performed on their argument.
   # Returns:
   #   A sanitized version of the given Hash, that can be safely
   #     logged via stdout or saved in log files. In the case of
   #     non-Hash items, the original item is returned.
-  def self.obscure_creds(creds)
-    return creds if creds.class != Hash
+  def self.obscure_options(options)
+    return options if options.class != Hash
 
     obscured = {}
-    creds.each { |k, v|
+    options.each { |k, v|
       if CLOUDY_CREDS.include?(k)
         obscured[k] = self.obscure_string(v)
       else

AppController/lib/infrastructure_manager_client.rb

@@ -127,34 +127,34 @@ def make_call(time, retry_on_except, callr, ok_to_fail=false)
   # Hash containing infrastructure-specific parameters.
   #
   # Args:
-  #   creds: A Hash that contains all of the credentials passed between
+  #   options: A Hash that contains all of the credentials passed between
   #     AppControllers.
   # Returns:
   #   A Hash that contains only the parameters needed to interact with AWS,
   #   Eucalyptus, or GCE.
-  def get_parameters_from_credentials(creds)
+  def get_parameters_from_credentials(options)
     return {
       "credentials" => {
         # EC2 / Eucalyptus-specific credentials
-        'EC2_ACCESS_KEY' => creds['ec2_access_key'],
-        'EC2_SECRET_KEY' => creds['ec2_secret_key'],
-        'EC2_URL' => creds['ec2_url']
+        'EC2_ACCESS_KEY' => options['ec2_access_key'],
+        'EC2_SECRET_KEY' => options['ec2_secret_key'],
+        'EC2_URL' => options['ec2_url']
       },
-      'project' => creds['project'],  # GCE-specific
-      "group" => creds['group'],
-      "image_id" => creds['machine'],
-      "infrastructure" => creds['infrastructure'],
-      "instance_type" => creds['instance_type'],
-      "keyname" => creds['keyname'],
-      "use_spot_instances" => creds['use_spot_instances'],
-      "max_spot_price" => creds['max_spot_price']
+      'project' => options['project'],  # GCE-specific
+      "group" => options['group'],
+      "image_id" => options['machine'],
+      "infrastructure" => options['infrastructure'],
+      "instance_type" => options['instance_type'],
+      "keyname" => options['keyname'],
+      "use_spot_instances" => options['use_spot_instances'],
+      "max_spot_price" => options['max_spot_price']
     }
   end
 
 
   def run_instances(parameters)
     obscured = parameters.dup
-    obscured['credentials'] = HelperFunctions.obscure_creds(obscured['credentials'])
+    obscured['credentials'] = HelperFunctions.obscure_options(obscured['credentials'])
     Djinn.log_debug("Calling run_instances with parameters " +
       "#{obscured.inspect}")
 
@@ -174,14 +174,14 @@ def describe_instances(parameters)
   end
 
 
-  def terminate_instances(creds, instance_ids)
-    parameters = get_parameters_from_credentials(creds)
+  def terminate_instances(options, instance_ids)
+    parameters = get_parameters_from_credentials(options)
 
     if instance_ids.class != Array
       instance_ids = [instance_ids]
     end
     parameters['instance_ids'] = instance_ids
-    parameters['region'] = creds['region']
+    parameters['region'] = options['region']
 
     terminate_result = make_call(NO_TIMEOUT, RETRY_ON_FAIL,
       "terminate_instances") {
@@ -191,12 +191,12 @@ def terminate_instances(creds, instance_ids)
   end
 
 
-  def spawn_vms(num_vms, creds, job, disks)
-    parameters = get_parameters_from_credentials(creds)
+  def spawn_vms(num_vms, options, job, disks)
+    parameters = get_parameters_from_credentials(options)
     parameters['num_vms'] = num_vms.to_s
     parameters['cloud'] = 'cloud1'
-    parameters['zone'] = creds['zone']
-    parameters['region'] = creds['region']
+    parameters['zone'] = options['zone']
+    parameters['region'] = options['region']
 
     run_result = run_instances(parameters)
     Djinn.log_debug("[IM] Run instances info says [#{run_result}]")

AppController/lib/zookeeper_helper.rb

@@ -53,7 +53,7 @@ def configure_zookeeper(nodes, my_index)
 
 def start_zookeeper
   Djinn.log_info("starting ZooKeeper")
-  if @creds['clear_datastore']
+  if @options['clear_datastore']
     Djinn.log_run("rm -rfv /var/lib/zookeeper")
     Djinn.log_run("rm -rfv #{DATA_LOCATION}")
   end

AppController/test/tc_djinn.rb

@@ -892,15 +892,15 @@ def test_start_new_roles_on_nodes_in_cluster
     all_nodes_serialized = JSON.dump([original_node.to_hash(),
       new_node2.to_hash(), new_node1.to_hash()])
 
-    creds = {'keyname' => 'boo', 'user_commands' => []}
-    creds_as_array = creds.to_a.flatten
+    options = {'keyname' => 'boo', 'user_commands' => []}
+    options_as_array = options.to_a.flatten
     no_apps = []
 
     # and that the appcontrollers receive the initial message to start
     # up from our appcontroller
     flexmock(AppControllerClient).new_instances { |instance|
       instance.should_receive(:set_parameters).with(all_nodes_serialized,
-        creds_as_array, no_apps).and_return("OK")
+        options_as_array, no_apps).and_return("OK")
     }
 
     # the appcontroller will update its local /etc/hosts file
@@ -935,7 +935,7 @@ def test_start_new_roles_on_nodes_in_cluster
     djinn = Djinn.new()
     djinn.nodes = [original_node]
     djinn.my_index = 0
-    djinn.creds = creds
+    djinn.options = options
     djinn.apps_loaded = ["booapp"]
     djinn.app_info_map = {
       'booapp' => {
@@ -1084,15 +1084,15 @@ def test_start_new_roles_on_nodes_in_cloud
     all_nodes_serialized = JSON.dump([original_node.to_hash(),
       new_node1.to_hash(), new_node2.to_hash()])
 
-    creds = {'keyname' => 'boo', 'user_commands' => []}
-    creds_as_array = creds.to_a.flatten
+    options = {'keyname' => 'boo', 'user_commands' => []}
+    options_as_array = options.to_a.flatten
     no_apps = []
 
     # and that the appcontrollers receive the initial message to start
     # up from our appcontroller
     flexmock(AppControllerClient).new_instances { |instance|
       instance.should_receive(:set_parameters).with(all_nodes_serialized,
-        creds_as_array, no_apps).and_return("OK")
+        options_as_array, no_apps).and_return("OK")
     }
 
     # lastly, the appcontroller will update its local /etc/hosts file
@@ -1113,7 +1113,7 @@ def test_start_new_roles_on_nodes_in_cloud
     djinn = Djinn.new()
     djinn.nodes = [original_node]
     djinn.my_index = 0
-    djinn.creds = creds
+    djinn.options = options
     actual = djinn.start_new_roles_on_nodes_in_cloud(ips_to_roles)
     assert_equal(true, actual.include?(node1_info))
     assert_equal(true, actual.include?(node2_info))
@@ -1261,7 +1261,7 @@ def test_scale_appservers_across_nodes_and_scale_up_one_app
     djinn = Djinn.new()
     djinn.nodes = [node, open_node]
     djinn.my_index = 0
-    djinn.creds = { 'keyname' => 'boo' }
+    djinn.options = { 'keyname' => 'boo' }
 
     # let's say there's one app running
     djinn.apps_loaded = ['bazapp']

AppController/test/tc_helperfunctions.rb

@@ -17,8 +17,8 @@ def setup
     @appengine_web_xml = "/var/apps/boo/app/war/WEB-INF/appengine-web.xml"
   end
 
-  def test_obscure_creds
-    creds = {
+  def test_obscure_options
+    options = {
       'ec2_access_key' => 'ABCDEFG',
       'ec2_secret_key' => 'HIJKLMN',
       'CLOUD_EC2_ACCESS_KEY' => 'OPQRSTU',
@@ -32,7 +32,7 @@ def test_obscure_creds
       'CLOUD_EC2_SECRET_KEY' => '***YZAB'
     }
 
-    actual = HelperFunctions.obscure_creds(creds)
+    actual = HelperFunctions.obscure_options(options)
     assert_equal(expected['ec2_access_key'], actual['ec2_access_key'])
     assert_equal(expected['ec2_secret_key'], actual['ec2_secret_key'])
     assert_equal(expected['CLOUD_EC2_ACCESS_KEY'],

AppController/test/tc_infrastructure_manager_client.rb

@@ -78,7 +78,7 @@ def test_spawn_one_vm_in_ec2
       and_return("127.0.0.1")
 
     imc = InfrastructureManagerClient.new("secret")
-    creds = {
+    options = {
       'group' => 'boogroup',
       'machine' => 'booid',
       'infrastructure' => 'booinfrastructure',
@@ -100,7 +100,7 @@ def test_spawn_one_vm_in_ec2
       "instance_id" => "i-id",
       "disk" => nil
     }]
-    actual = imc.spawn_vms(1, creds, "open", [nil])
+    actual = imc.spawn_vms(1, options, "open", [nil])
     assert_equal(expected, actual)
   end
 
@@ -162,7 +162,7 @@ def test_spawn_three_vms_in_ec2
       and_return("127.0.0.1")
 
     imc = InfrastructureManagerClient.new("secret")
-    creds = {
+    options = {
       'group' => 'boogroup',
       'machine' => 'booid',
       'infrastructure' => 'booinfrastructure',
@@ -196,7 +196,7 @@ def test_spawn_three_vms_in_ec2
       'instance_id' => 'i-id3',
       'disk' => nil
     }]
-    actual = imc.spawn_vms(3, creds, ["a", "b", "c"], [nil, nil, nil])
+    actual = imc.spawn_vms(3, options, ["a", "b", "c"], [nil, nil, nil])
     assert_equal(expected, actual)
   end
 
@@ -258,7 +258,7 @@ def test_spawn_three_vms_in_gce
       and_return("127.0.0.1")
 
     imc = InfrastructureManagerClient.new("secret")
-    creds = {
+    options = {
       'group' => 'boogroup',
       'machine' => 'booid',
       'infrastructure' => 'booinfrastructure',
@@ -292,7 +292,7 @@ def test_spawn_three_vms_in_gce
       'instance_id' => 'i-id3',
       'disk' => nil
     }]
-    actual = imc.spawn_vms(3, creds, ["a", "b", "c"], [nil, nil, nil])
+    actual = imc.spawn_vms(3, options, ["a", "b", "c"], [nil, nil, nil])
     assert_equal(expected, actual)
   end
 

AppDB/cassandra/cassandra_helper.rb

@@ -102,7 +102,7 @@ def setup_db_config_files(master_ip, slave_ips)
       contents.gsub!(/APPSCALE-LOCAL/, HelperFunctions.local_ip)
       contents.gsub!(/APPSCALE-MASTER/, master_ip)
       contents.gsub!(/APPSCALE-TOKEN/, "#{local_token}")
-      contents.gsub!(/REPLICATION/, @creds["replication"])
+      contents.gsub!(/REPLICATION/, @options["replication"])
       contents.gsub!(/APPSCALE-JMX-PORT/, "7070")              
       File.open(full_path_to_write, "w+") { |dest_file|
         dest_file.write(contents)
@@ -137,7 +137,7 @@ def start_db_slave()
 # Starts Cassandra, and waits for it to start the Thrift service locally.
 def start_cassandra()
   Djinn.log_run("pkill ThriftBroker")
-  if @creds['clear_datastore']
+  if @options['clear_datastore']
     Djinn.log_info("Erasing datastore contents")
     Djinn.log_run("rm -rf /opt/appscale/cassandra*")
     Djinn.log_run("rm /var/log/appscale/cassandra/system.log")