Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

EC2Architect - Various small improvements #609

Merged
merged 3 commits into from
Nov 18, 2021
Merged

EC2Architect - Various small improvements #609

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d85479b
Various small improvements
JackUrb Nov 17, 2021
3b3018a
proper env use
JackUrb Nov 17, 2021
8aec981
Also clean subdomain string
JackUrb Nov 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion mephisto/abstractions/architects/ec2/ec2_architect.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,15 @@
DEPLOY_WAIT_TIME = 3


def url_safe_string(in_string: str) -> str:
"""
Produces a domain string that is safe for use
in ec2 resources
"""
hyphenated = in_string.replace("_", "-")
return re.sub("[^0-9a-zA-Z\-]+", "", hyphenated)


@dataclass
class EC2ArchitectArgs(ArchitectArgs):
"""Additional arguments for configuring a heroku architect"""
Expand Down Expand Up @@ -85,7 +94,7 @@ def __init__(
with open(DEFAULT_FALLBACK_FILE, "r") as fallback_detail_file:
self.fallback_details = json.load(fallback_detail_file)

self.subdomain = args.architect.subdomain
self.subdomain = url_safe_string(args.architect.subdomain)
self.root_domain = self.fallback_details["domain"]
self.router_name = f"{self.subdomain}-routing-server"
self.full_domain = f"{self.subdomain}.{self.root_domain}"
Expand Down
74 changes: 56 additions & 18 deletions mephisto/abstractions/architects/ec2/ec2_helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import os
import subprocess
import json
import getpass
from mephisto.abstractions.providers.mturk.mturk_utils import setup_aws_credentials
from mephisto.abstractions.architects.router import build_router

Expand Down Expand Up @@ -44,6 +45,14 @@
MAX_RETRIES = 10


def get_owner_tag() -> Dict[str, str]:
"""
Creates a tag with the user's username
as the owner for the given resource
"""
return {"Key": "Owner", "Value": getpass.getuser()}


def check_aws_credentials(profile_name: str) -> bool:
try:
# Check existing credentials
Expand Down Expand Up @@ -285,7 +294,10 @@ def create_mephisto_vpc(session: boto3.Session) -> Dict[str, str]:
TagSpecifications=[
{
"ResourceType": "vpc",
"Tags": [{"Key": "Name", "Value": "mephisto-core-vpc"}],
"Tags": [
{"Key": "Name", "Value": "mephisto-core-vpc"},
get_owner_tag(),
],
}
],
)
Expand All @@ -296,7 +308,7 @@ def create_mephisto_vpc(session: boto3.Session) -> Dict[str, str]:
TagSpecifications=[
{
"ResourceType": "internet-gateway",
"Tags": [{"Key": "Name", "Value": "mephisto-gateway"}],
"Tags": [{"Key": "Name", "Value": "mephisto-gateway"}, get_owner_tag()],
}
],
)
Expand All @@ -311,7 +323,10 @@ def create_mephisto_vpc(session: boto3.Session) -> Dict[str, str]:
TagSpecifications=[
{
"ResourceType": "subnet",
"Tags": [{"Key": "Name", "Value": "mephisto-subnet-1"}],
"Tags": [
{"Key": "Name", "Value": "mephisto-subnet-1"},
get_owner_tag(),
],
}
],
CidrBlock="10.0.0.0/24",
Expand All @@ -324,7 +339,10 @@ def create_mephisto_vpc(session: boto3.Session) -> Dict[str, str]:
TagSpecifications=[
{
"ResourceType": "subnet",
"Tags": [{"Key": "Name", "Value": "mephisto-subnet-2"}],
"Tags": [
{"Key": "Name", "Value": "mephisto-subnet-2"},
get_owner_tag(),
],
}
],
CidrBlock="10.0.1.0/24",
Expand All @@ -338,7 +356,10 @@ def create_mephisto_vpc(session: boto3.Session) -> Dict[str, str]:
TagSpecifications=[
{
"ResourceType": "route-table",
"Tags": [{"Key": "Name", "Value": "mephisto-routes-1"}],
"Tags": [
{"Key": "Name", "Value": "mephisto-routes-1"},
get_owner_tag(),
],
}
],
VpcId=vpc_id,
Expand All @@ -349,7 +370,10 @@ def create_mephisto_vpc(session: boto3.Session) -> Dict[str, str]:
TagSpecifications=[
{
"ResourceType": "route-table",
"Tags": [{"Key": "Name", "Value": "mephisto-routes-2"}],
"Tags": [
{"Key": "Name", "Value": "mephisto-routes-2"},
get_owner_tag(),
],
}
],
VpcId=vpc_id,
Expand Down Expand Up @@ -402,7 +426,10 @@ def create_security_group(session: boto3.Session, vpc_id: str, ssh_ip: str) -> s
TagSpecifications=[
{
"ResourceType": "security-group",
"Tags": [{"Key": "Name", "Value": "mephisto-server-security-group"}],
"Tags": [
{"Key": "Name", "Value": "mephisto-server-security-group"},
get_owner_tag(),
],
}
],
)
Expand Down Expand Up @@ -516,7 +543,10 @@ def create_key_pair(
response = client.create_key_pair(
KeyName=key_name,
TagSpecifications=[
{"ResourceType": "key-pair", "Tags": [{"Key": "Name", "Value": key_name}]}
{
"ResourceType": "key-pair",
"Tags": [{"Key": "Name", "Value": key_name}, get_owner_tag()],
}
],
)
with open(target_keypair_filename, "w+") as keypair_file:
Expand Down Expand Up @@ -574,10 +604,8 @@ def create_instance(
{
"ResourceType": "instance",
"Tags": [
{
"Key": "Name",
"Value": instance_name,
},
{"Key": "Name", "Value": instance_name},
get_owner_tag(),
],
},
],
Expand Down Expand Up @@ -684,7 +712,12 @@ def register_instance_to_listener(
find_rule_response = client.describe_rules(
ListenerArn=listener_arn,
)
rule_count = len(find_rule_response["Rules"])

# Get the next available priority
priorities = set([r["Priority"] for r in find_rule_response["Rules"]])
priority = 1
while str(priority) in priorities:
priority += 1

rule_response = client.create_rule(
ListenerArn=listener_arn,
Expand All @@ -699,7 +732,7 @@ def register_instance_to_listener(
},
},
],
Priority=rule_count + 1,
Priority=priority,
Actions=[
{
"Type": "forward",
Expand Down Expand Up @@ -807,7 +840,8 @@ def get_instance_address(
{
"Key": "Name",
"Value": f"{instance_id}-ip-address",
}
},
get_owner_tag(),
],
}
],
Expand Down Expand Up @@ -861,7 +895,9 @@ def try_server_push(subprocess_args: List[str], retries=5, sleep_time=10.0):
"""
while retries > 0:
try:
subprocess.check_call(subprocess_args)
subprocess.check_call(
subprocess_args, env=dict(os.environ, SSH_AUTH_SOCK="")
)
return
except subprocess.CalledProcessError:
retries -= 1
Expand Down Expand Up @@ -919,7 +955,8 @@ def deploy_fallback_server(
remote_server,
"bash",
"/home/ec2-user/fallback_server/scripts/first_setup.sh",
]
],
env=dict(os.environ, SSH_AUTH_SOCK=""),
)
detete_instance_address(session, allocation_id, association_id)
except Exception as e:
Expand Down Expand Up @@ -966,7 +1003,8 @@ def deploy_to_routing_server(
remote_server,
"bash",
"/home/ec2-user/routing_server/setup/init_server.sh",
]
],
env=dict(os.environ, SSH_AUTH_SOCK=""),
)
detete_instance_address(session, allocation_id, association_id)
print("Server setup complete!")
Expand Down