diff --git a/Makefile b/Makefile index 78d09be1..6affc43a 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,8 @@ SOURCES := \ lambda/tea_bumper.py \ lambda/update_lambda.py -RESOURCES := $(wildcard lambda/templates/*) +HTML_TEMPLATES := $(wildcard lambda/templates/*.html) +MD_TEMPLATES := $(wildcard lambda/templates/*.md) TERRAFORM := $(wildcard terraform/*) REQUIREMENTS_IN := $(wildcard requirements/*.in) @@ -15,7 +16,9 @@ DIR := dist EMPTY := $(DIR)/empty # Temporary artifacts DIST_SOURCES := $(SOURCES:lambda/%=$(DIR)/code/%) -DIST_RESOURCES := $(RESOURCES:lambda/%=$(DIR)/code/%) +DIST_MD_RESOURCES := $(MD_TEMPLATES:lambda/%.md=$(DIR)/code/%.html) +DIST_HTML_RESOURCES := $(HTML_TEMPLATES:lambda/%=$(DIR)/code/%) +DIST_RESOURCES := $(DIST_HTML_RESOURCES) $(DIST_MD_RESOURCES) DIST_TERRAFORM := $(TERRAFORM:terraform/%=$(DIR)/terraform/%) BUCKET_MAP_OBJECT_KEY := DEFAULT @@ -105,8 +108,13 @@ $(DIR)/thin-egress-app-dependencies.zip: requirements/requirements.txt $(REQUIRE @mkdir -p $(DIR)/python $(DOCKER_LAMBDA_CI) build/dependency_builder.sh "$(DIR)/thin-egress-app-dependencies.zip" "$(DIR)" +.SECONDARY: $(DIST_MD_RESOURCES) +$(DIST_MD_RESOURCES): $(DIR)/code/%.html: lambda/%.md $(BUILD_VENV) + @mkdir -p $(@D) + $(BUILD_VENV)/bin/python scripts/render_md.py $< --output $@ + .SECONDARY: $(DIST_RESOURCES) -$(DIST_RESOURCES): $(DIR)/code/%: lambda/% +$(DIST_HTML_RESOURCES): $(DIR)/code/%: lambda/% @mkdir -p $(@D) cp $< $@ diff --git a/cloudformation/thin-egress-app.yaml.j2 b/cloudformation/thin-egress-app.yaml.j2 index 062a2a0b..5808dc74 100644 --- a/cloudformation/thin-egress-app.yaml.j2 +++ b/cloudformation/thin-egress-app.yaml.j2 @@ -716,6 +716,16 @@ Resources: PathPart: 's3credentials' RestApiId: !Ref EgressApiGateway + EgressApiResourceS3CredentialsREADME: + Type: AWS::ApiGateway::Resource + Condition: S3CredentialsEndpointIsSet + DependsOn: + - EgressApiGateway + Properties: + ParentId: !GetAtt EgressApiGateway.RootResourceId + PathPart: 's3credentialsREADME' + RestApiId: !Ref EgressApiGateway + EgressApiResourceProfile: Type: AWS::ApiGateway::Resource DependsOn: @@ -925,6 +935,23 @@ Resources: ResourceId: !Ref EgressApiResourceS3Credentials RestApiId: !Ref EgressApiGateway + EgressAPIMethodS3CredentialsREADME: + Type: AWS::ApiGateway::Method + Condition: S3CredentialsEndpointIsSet + Properties: + ApiKeyRequired: false + AuthorizationType: 'NONE' + HttpMethod: 'GET' + Integration: + IntegrationHttpMethod: 'POST' + IntegrationResponses: + - StatusCode: 200 + Type: 'AWS_PROXY' + Uri: !Sub "arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${EgressLambda.Arn}/invocations" + OperationName: 's3credentialsREADME view' + ResourceId: !Ref EgressApiResourceS3CredentialsREADME + RestApiId: !Ref EgressApiGateway + EgressAPIMethodProfile: Type: AWS::ApiGateway::Method Properties: diff --git a/docs/configuration.md b/docs/configuration.md index d91667f4..3ec6e680 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,3 +1,5 @@ +# Configuration + ## Bucket Mapping At the heart of TEA is the concept of the Bucket Map. This YAML file tells TEA how to diff --git a/docs/deploying.md b/docs/deploying.md index 4abd8634..320ad77f 100644 --- a/docs/deploying.md +++ b/docs/deploying.md @@ -1,3 +1,5 @@ +# Deploying + ## Quickstart You can either download [`deploy.sh`](https://github.com/asfadmin/thin-egress-app/blob/devel/build/deploy.sh) diff --git a/docs/index.md b/docs/index.md index 88dbafd8..7df5956c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,4 +1,5 @@ # Thin Egress App + ## Purpose of TEA TEA is a fully Earthdata Cloud (EDC) compliant application to enable diff --git a/docs/s3access.md b/docs/s3access.md index 45939977..ad5d6d42 100644 --- a/docs/s3access.md +++ b/docs/s3access.md @@ -1,83 +1,130 @@ -## S3 Direct Access +# S3 Direct Access + *NOTE: Support for S3 direct access is currently experimental* You can retrieve temporary S3 credentials at the `/s3credentials` endpoint when -authenticated via earthdata login. These credentials will be valid for 1 hour +authenticated via Earthdata Login. These credentials will only be valid for +**1 hour** due to +[role chaining](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_terms-and-concepts.html) and can be used make in-region `s3:ListBucket` and `s3:GetObject` requests. +Your code must handle expired tokens and request new ones as needed +for sessions that exceed this 1 hour limit. -### Request -Credentials are retrieved through a `GET` request to the `/s3credentials` -endpoint. An optional header `app-name` can be provided to include in the -generated role session name which will show up in EMS logs. +## Request -**Params:** -None. +Credentials are retrieved through an HTTP `GET` request to the `/s3credentials` +endpoint. The request must be authenticated with either a JWT token for TEA or +by using +[EDL Bearer Tokens](https://urs.earthdata.nasa.gov/documentation/for_users/user_token). +Unauthenticated requests will be redirected to EDL. **Headers:** - - `app-name`: A string to include in the generated role session name for - metric reporting purposes. It is recommended to include this header when - making requests on users behalf from another cloud service. The generated - role session name is `username@app-name`. + +* (optional) `app-name`: An arbitrary string to include in the generated role + session name for metric reporting purposes. It can only contain characters + that are valid in a `RoleSessionName` see the + [AssumeRole documentation](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#API_AssumeRole_RequestParameters). + It is recommended to include this header when making requests on users behalf + from another cloud service. The generated role session name is + `username@app-name`. **Example:** ```python import requests -requests.get( +resp = requests.get( "https://your-tea-host/s3credentials", headers={"app-name": "my-application"}, cookies={"asf-urs": ""} ) +print(resp.json()) ``` -### Response -The response is your temporary credentials as returned by Amazon STS. -[See the AWS Credentials reference](https://docs.aws.amazon.com/STS/latest/APIReference/API_Credentials.html") +## Response + +The response is your temporary credentials as returned by Amazon STS. See the +[AWS Credentials reference](https://docs.aws.amazon.com/STS/latest/APIReference/API_Credentials.html) for more details. **Example:** ```json { - "AccessKeyId": "AKIAIOSFODNN7EXAMPLE", - "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - "SessionToken": "LONGSTRINGOFCHARACTERS.../HJLgV91QJFCMlmY8slIEOjrOChLQYmzAqrb5U1ekoQAK6f86HKJFTT2dONzPgmJN9ZvW5DBwt6XUxC9HAQ0LDPEYEwbjGVKkzSNQh/", - "Expiration": "2021-01-27 00:50:09+00:00" + "AccessKeyId": "AKIAIOSFODNN7EXAMPLE", + "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "SessionToken": "LONGSTRINGOFCHARACTERS.../HJLgV91QJFCMlmY8slIEOjrOChLQYmzAqrb5U1ekoQAK6f86HKJFTT2dONzPgmJN9ZvW5DBwt6XUxC9HAQ0LDPEYEwbjGVKkzSNQh/", + "Expiration": "2021-01-27 00:50:09+00:00" } ``` -### Using Temporary Credentials -To use the credentials you must configure your AWS client with the returned -access key, secret and token. Note that the credentials will only work in- -region, so you will get 403 errors if you try to use them with the AWS cli. +## Using Temporary Credentials + +To use the credentials you must configure your AWS SDK library with the +returned access key, secret and token. Note that the credentials are only valid +for in-region requests, so using them with your AWS CLI will not work! You must +make your requests from an AWS service such as Lambda or EC2 in the same region +as the source bucket you are pulling from. See +[Using temporary credentials with AWS resources](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) +for more information on how to use your temporary credentials. **Example:** -```python -import boto3 -import requests -def get_client(): - resp = requests.get( - "https://your-tea-host/s3credentials", - headers={"app-name": "my-application"}, - cookies={"asf-urs": ""} - ) - resp.raise_for_status() - creds = resp.json() - - return boto3.client( - "s3", - aws_access_key_id=creds["AccessKeyId"], - aws_secret_access_key=creds["SecretAccessKey"], - aws_session_token=creds["SessionToken"] - ) +This example lambda function uses +[EDL Bearer Tokens](https://urs.earthdata.nasa.gov/documentation/for_users/user_token) +to obtain s3 credentials and stream an object from one bucket to another. The +lambda execution role will need `s3:PutObject` permissions on the destination +bucket. -``` -### Limits +```python +import boto3 +import json +import urllib.request + + +def lambda_handler(event, context): + # Get temporary download credentials + tea_url = event["CredentialsEndpoint"] + bearer_token = event["BearerToken"] + req = urllib.request.Request( + url=tea_url, + headers={"Authorization": f"Bearer {bearer_token}"} + ) + with urllib.request.urlopen(req) as f: + creds = json.loads(f.read().decode()) + + # Set up separate boto3 clients for download and upload + download_client = boto3.client( + "s3", + aws_access_key_id=creds["AccessKeyId"], + aws_secret_access_key=creds["SecretAccessKey"], + aws_session_token=creds["SessionToken"] + ) + # Lambda needs to have permission to upload to destination bucket + upload_client = boto3.client("s3") + + # Stream from the source bucket to the destination bucket + resp = download_client.get_object( + Bucket=event["Source"]["Bucket"], + Key=event["Source"]["Key"], + ) + upload_client.upload_fileobj( + resp["Body"], + event["Dest"]["Bucket"], + event["Dest"].get("Key") or event["Source"]["Key"], + ) +``` -The credentials dispensed from the `/s3credentials` endpoint are valid for -**1 hour**. Your code must handle expired tokens and request new ones as needed -for sessions that exceed this 1 hour limit. This is an AWS Limit is due to -[role chaining](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_terms-and-concepts.html) +The example can be invoked with an event payload as follows: -These credentials will have `s3:GetObject` and `s3:ListBucket` permissions on -the configured resources. +```json +{ + "CredentialsEndpoint": "https://your-tea-host/s3credentials", + "BearerToken": "your bearer token", + "Source": { + "Bucket": "S3 bucket name from CMR link", + "Key": "S3 key from CMR link" + }, + "Dest": { + "Bucket": "S3 bucket name to copy to" + } +} +``` diff --git a/docs/technical.md b/docs/technical.md index c90c9feb..14eb9753 100644 --- a/docs/technical.md +++ b/docs/technical.md @@ -1,3 +1,5 @@ +# Technical + ## System Architecture TEA is a [Python Chalice](https://github.com/aws/chalice) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 287a15ec..936f5f19 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1,3 +1,5 @@ +# Troubleshooting + ## When things go wrong There is a lot that _can_ go wrong, but we hope that if you followed this diff --git a/docs/vision.md b/docs/vision.md index 6e25a84e..d594b89f 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -1,3 +1,5 @@ +# Vision + We the **TEA**m strive to create the simplest, most reliable, most feature-rich S3 distribution app. Our intent is not to provide EVERY feature imaginable, but to maintain the **THIN** core of broadly applicable features. We want to continue to engage the user diff --git a/lambda/app.py b/lambda/app.py index 714da6f2..18d3ee30 100644 --- a/lambda/app.py +++ b/lambda/app.py @@ -1032,6 +1032,12 @@ def get_s3_credentials(user_id: str, role_session_name: str, policy: dict): return response["Credentials"] +@app.route('/s3credentialsREADME', methods=['GET']) +@with_trace(context={}) +def s3credentials_readme(): + return make_html_response({}, {}, 200, "s3credentials_readme.html") + + @app.route('/profile') @with_trace(context={}) def profile(): diff --git a/lambda/templates/base.html b/lambda/templates/base.html index 13ccc3c3..d0f6e7e8 100644 --- a/lambda/templates/base.html +++ b/lambda/templates/base.html @@ -28,6 +28,7 @@ font-size: 85%; } + {% block head %}{% endblock %} diff --git a/lambda/templates/s3access.md b/lambda/templates/s3access.md new file mode 120000 index 00000000..8ab38ce0 --- /dev/null +++ b/lambda/templates/s3access.md @@ -0,0 +1 @@ +../../docs/s3access.md \ No newline at end of file diff --git a/lambda/templates/s3credentials_readme.html b/lambda/templates/s3credentials_readme.html new file mode 100644 index 00000000..40e41038 --- /dev/null +++ b/lambda/templates/s3credentials_readme.html @@ -0,0 +1,99 @@ +{% extends "base.html" %} + +{% block pagetitle %} +S3 Credentials Readme +{% endblock %} + +{% block head %} + +{% raw %} + +{% endraw %} + + +{% endblock %} + +{% block content %} +
+{% include "s3access.html" %} +
+{% endblock %} diff --git a/requirements/requirements-make.in b/requirements/requirements-make.in index 7f7afbf3..67700c12 100644 --- a/requirements/requirements-make.in +++ b/requirements/requirements-make.in @@ -1 +1,4 @@ jinja2 +markdown +pygments +pymdown-extensions diff --git a/requirements/requirements-make.txt b/requirements/requirements-make.txt index d9c25897..6e757c07 100644 --- a/requirements/requirements-make.txt +++ b/requirements/requirements-make.txt @@ -4,7 +4,19 @@ # # pip-compile requirements/requirements-make.in # +importlib-metadata==5.0.0 + # via markdown jinja2==3.1.2 # via -r requirements/requirements-make.in +markdown==3.4.1 + # via + # -r requirements/requirements-make.in + # pymdown-extensions markupsafe==2.1.1 # via jinja2 +pygments==2.13.0 + # via -r requirements/requirements-make.in +pymdown-extensions==9.7 + # via -r requirements/requirements-make.in +zipp==3.10.0 + # via importlib-metadata diff --git a/scripts/render_md.py b/scripts/render_md.py new file mode 100644 index 00000000..4015a30d --- /dev/null +++ b/scripts/render_md.py @@ -0,0 +1,34 @@ +import argparse +from pathlib import Path +from typing import Optional + +import markdown + + +def render_markdown(inpath: Path, outpath: Optional[Path] = None): + rendered = markdown.markdown( + inpath.read_text(), + extensions=[ + "markdown.extensions.sane_lists", + "markdown.extensions.tables", + "pymdownx.betterem", + "pymdownx.highlight", + "pymdownx.superfences", + ] + ) + outpath = outpath or inpath.with_suffix(".html") + outpath.write_text(rendered) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("input", type=Path) + parser.add_argument("--output", "-o", type=Path) + + args = parser.parse_args() + + render_markdown(args.input, args.output) + + +if __name__ == "__main__": + main()