Merge pull request #55 from GoogleCloudPlatform/sfan-sts03

Add storagetransfer samples

storage/storage_transfer/README.md

@@ -0,0 +1,51 @@
+# Transfer Service sample using Python
+
+This app creates two types of transfers using the Transfer Service tool.
+
+## Prerequisites
+
+1. Set up a project on Google Developers Console.
+  1. Go to the [Developers Console](https://cloud.google.com/console) and create or select your project.
+     You will need the project ID later.
+1. Within Developers Console, select APIs & auth > Credentials.
+  1. Add a new JSON key credential for a service account.
+  1. Set the environment variable GOOGLE_APPLICATION_CREDENTIALS to point to your JSON key.
+1. Add the Storage Transfer service account, [email protected] as an
+   editor of your project.
+1. Set up gcloud for application default credentials.
+  1. `gcloud components update`
+  1. `gcloud auth login`
+  1. `gcloud config set project PROJECT_ID`
+1. Install [Google API Client Library for Python](https://developers.google.com/api-client-library/python/start/installation).
+
+## Transfer from Amazon S3 to Google Cloud Storage
+
+Creating a one-time transfer from Amazon S3 to Google Cloud Storage.
+1. Set up data sink.
+  1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser.
+1. Set up data source.
+  1. Go to AWS Management Console and create a bucket.
+  1. Under Security Credentials, create an IAM User with access to the bucket.
+  1. Create an Access Key for the user. Note the Access Key ID and Secret Access Key.
+1. In aws_request.py, fill in the Transfer Job JSON template with relevant values.
+1. Run with `python aws_request.py`
+  1. Note the job ID in the returned Transfer Job.
+
+## Transfer data from a standard Cloud Storage bucket to a Cloud Storage Nearline bucket
+
+Creating a daily transfer from a standard Cloud Storage bucket to a Cloud Storage Nearline
+bucket for files untouched for 30 days.
+1. Set up data sink.
+  1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser.
+  1. Select Nearline for Storage Class.
+1. Set up data source.
+  1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser.
+1. In nearline_request.py, fill in the Transfer Job JSON template with relevant values.
+1. Run with `python nearline_request.py`
+  1. Note the job ID in the returned Transfer Job.
+
+## Checking the status of a transfer
+
+1. In transfer_check.py, fill in the Transfer Job JSON template with relevant values.
+   Use the Job Name you recorded earlier.
+1. Run with `python transfer_check.py`

storage/storage_transfer/aws_request.py

@@ -0,0 +1,68 @@
+# Copyright 2015, Google, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import json
+import logging
+
+import create_client
+
+
+def main():
+    """Create a one-off transfer from Amazon S3 to GCS."""
+    logging.getLogger().setLevel(logging.DEBUG)
+    transfer_service_client = create_client.create_transfer_client()
+
+    # Edit this template with desired parameters.
+    # Specify times below using US Pacific Time Zone.
+    transfer_job = '''
+    {
+        "description": "YOUR DESCRIPTION",
+        "status": "ENABLED",
+        "projectId": "YOUR_PROJECT_ID",
+        "schedule": {
+            "scheduleStartDate": {
+                "day": 1,
+                "month": 1,
+                "year": 2015
+            },
+            "scheduleEndDate": {
+                "day": 1,
+                "month": 1,
+                "year": 2015
+            },
+            "startTimeOfDay": {
+                "hours": 0,
+                "minutes": 0
+            }
+        },
+        "transferSpec": {
+            "awsS3DataSource": {
+                "bucketName": "YOUR_SOURCE_BUCKET",
+                "awsAccessKey": {
+                    "accessKeyId": "YOUR_ACCESS_KEY_ID",
+                    "secretAccessKey": "YOUR_SECRET_ACCESS_KEY"
+                }
+            },
+            "gcsDataSink": {
+                "bucketName": "YOUR_SINK_BUCKET"
+            }
+        }
+    }
+    '''
+
+    result = transfer_service_client.transferJobs().create(body=json.loads(
+        transfer_job)).execute()
+    logging.info('Returned transferJob: %s', json.dumps(result, indent=4))
+
+if __name__ == '__main__':
+    main()

storage/storage_transfer/create_client.py

@@ -0,0 +1,27 @@
+# Copyright 2015, Google, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import logging
+
+from apiclient import discovery
+from oauth2client.client import GoogleCredentials
+
+CLOUD_SCOPES = 'https://www.googleapis.com/auth/cloud-platform'
+
+
+def create_transfer_client():
+    """Create a transfer client."""
+
+    logging.getLogger().setLevel(logging.DEBUG)
+    credentials = GoogleCredentials.get_application_default()
+    return discovery.build('storagetransfer', 'v1', credentials=credentials)

storage/storage_transfer/nearline_request.py

@@ -0,0 +1,64 @@
+# Copyright 2015, Google, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import json
+import logging
+
+import create_client
+
+
+def main():
+    """Transfer from standard Cloud Storage to Cloud Storage Nearline."""
+    logging.getLogger().setLevel(logging.DEBUG)
+    transfer_service_client = create_client.create_transfer_client()
+
+    # Edit this template with desired parameters.
+    # Specify times below using US Pacific Time Zone.
+    transfer_job = '''
+    {
+        "description": "YOUR DESCRIPTION",
+        "status": "ENABLED",
+        "projectId": "YOUR_PROJECT_ID",
+        "schedule": {
+            "scheduleStartDate": {
+                "day": 1,
+                "month": 1,
+                "year": 2015
+            },
+            "startTimeOfDay": {
+                "hours": 1,
+                "minutes": 1
+            }
+        },
+        "transferSpec": {
+            "gcsDataSource": {
+                "bucketName": "YOUR_SOURCE_BUCKET"
+            },
+            "gcsDataSink": {
+                "bucketName": "YOUR_SINK_BUCKET"
+            },
+            "objectConditions": {
+                "minTimeElapsedSinceLastModification": "2592000s"
+            },
+            "transferOptions": {
+                "deleteObjectsFromSourceAfterTransfer": true
+            }
+        }
+    }
+    '''
+    result = transfer_service_client.transferJobs().create(body=json.loads(
+        transfer_job)).execute()
+    logging.info('Returned transferJob: %s', json.dumps(result, indent=4))
+
+if __name__ == '__main__':
+    main()

storage/storage_transfer/test_aws_request.py

@@ -0,0 +1,39 @@
+# Copyright 2015, Google, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import unittest
+
+from aws_request import main
+
+from mock import Mock
+from mock import patch
+
+
+class AwsRequestTestCase(unittest.TestCase):
+    """A test case for creating a TransferJob from AWS S3."""
+
+    def setUp(self):
+        patcher1 = patch(
+            'storage.storage_transfer.aws_request.create_client')
+        self.mock_create_client = patcher1.start()
+        self.addCleanup(patcher1.stop)
+        self.mock_client = Mock(spec=['transferJobs'])
+        self.mock_create_client.create_transfer_client.return_value = \
+            self.mock_client
+
+    def test_create_aws_request(self):
+        execute = self.mock_client.transferJobs.return_value.create.return_value \
+            .execute
+        execute.return_value = ""
+        main()
+        execute.assert_called_with()

storage/storage_transfer/test_create_client.py

@@ -0,0 +1,44 @@
+# Copyright 2015, Google, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import unittest
+
+import create_client
+
+from mock import Mock
+from mock import patch
+from oauth2client.client import GoogleCredentials
+
+
+class CheckCreateClientTestCase(unittest.TestCase):
+    """A test case for client creation."""
+
+    def setUp(self):
+        patcher1 = patch(
+            'storage.storage_transfer.create_client.GoogleCredentials')
+        patcher2 = patch(
+            'storage.storage_transfer.create_client.discovery.build')
+        self.mock_google_credentials = patcher1.start()
+        self.mock_discovery = patcher2.start()
+        self.addCleanup(patcher1.stop)
+        self.addCleanup(patcher2.stop)
+
+        self.mock_credentials = Mock(spec=GoogleCredentials)
+        self.mock_google_credentials.get_application_default.return_value = \
+            self.mock_credentials
+
+    def test_create_client(self):
+        create_client.create_transfer_client()
+        self.mock_discovery.assert_called_with(
+            'storagetransfer', 'v1',
+            credentials=self.mock_credentials)

storage/storage_transfer/test_nearline_request.py

@@ -0,0 +1,39 @@
+# Copyright 2015, Google, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import unittest
+
+from mock import Mock
+from mock import patch
+
+from nearline_request import main
+
+
+class NearlineRequestTestCase(unittest.TestCase):
+    """A test case for creating a TransferJob to Nearline for old files."""
+
+    def setUp(self):
+        patcher1 = patch(
+            'storage.storage_transfer.nearline_request.create_client')
+        self.mock_create_client = patcher1.start()
+        self.addCleanup(patcher1.stop)
+        self.mock_client = Mock(spec=['transferJobs'])
+        self.mock_create_client.create_transfer_client.return_value = \
+            self.mock_client
+
+    def test_create_nearline_request(self):
+        execute = self.mock_client.transferJobs.return_value.create.return_value \
+            .execute
+        execute.return_value = ""
+        main()
+        execute.assert_called_with()

storage/storage_transfer/test_transfer_check.py

@@ -0,0 +1,31 @@
+# Copyright 2015, Google, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import unittest
+
+from mock import Mock
+
+from transfer_check import check_operation
+
+
+class CheckTransferTestCase(unittest.TestCase):
+    """A test case for querying transfer job completion."""
+
+    def test_check_operation(self):
+        mock_client = Mock(spec=['transferOperations'])
+        execute = mock_client.transferOperations.return_value.list.return_value \
+            .execute
+        project_id = ""
+        job_name = ""
+        check_operation(mock_client, project_id, job_name)
+        execute.assert_called_with()