[In- review] Improving pageable extension adding autopagination (#1211)

Improving pageable extension adding autopagination

Samples/azure-storage/Azure.Ruby/generated/azure_storage/storage_accounts.rb

@@ -141,12 +141,8 @@ def check_name_availability_async(account_name, custom_headers = nil)
     # characters in length and use numbers and lower-case letters only.
     # @param parameters [StorageAccountCreateParameters] The parameters to provide
     # for the created account.
-    # @param @client.api_version [String] Client Api Version.
-    # @param @client.subscription_id [String] Gets subscription credentials which
-    # uniquely identify Microsoft Azure subscription. The subscription ID forms
-    # part of the URI for every service call.
-    # @param @client.accept_language [String] Gets or sets the preferred language
-    # for the response.
+    # @param custom_headers [Hash{String => String}] A hash of custom headers that
+    # will be added to the HTTP request.
     #
     # @return [Concurrent::Promise] promise which provides async access to http
     # response.

src/generator/AutoRest.Ruby.Azure.Tests/RspecTests/paging_spec.rb

@@ -19,102 +19,146 @@
 
   # Paging happy path tests
   it 'should get single pages' do
-    result = @client.paging.get_single_pages_async().value!
-    expect(result.response.status).to eq(200)
-    expect(result.body.next_link).to be_nil
+    all_items = @client.paging.get_single_pages()
+    expect(all_items.count).to eq(1)
+    expect(all_items.is_a? Array)
+    expect(all_items.first.is_a? PagingModule::Models::Product)
   end
 
   it 'should get multiple pages' do
-    result = @client.paging.get_multiple_pages_async().value!
-    expect(result.response.status).to eq(200)
-    expect(result.body.next_link).not_to be_nil
-
-    count = 1
-    while result.body.next_link != nil do
-      result = @client.paging.get_multiple_pages_next_async(result.body.next_link).value!
-      count += 1
-    end
+    all_items = @client.paging.get_multiple_pages() #returns items from all the pages
+    items_count = all_items.count
+    expect(items_count).to eq(10)
+    expect(all_items.is_a? Array)
+    expect(all_items.first.is_a? PagingModule::Models::Product)
+  end
 
-    expect(count).to eq(10)
+  it 'should get multiple pages - lazy' do
+    page = @client.paging.get_multiple_pages_as_lazy()
+    expect(page.is_a? PagingModule::Models::ProductResult)
+    expect(page.next_link).not_to be_nil
+    items = page.values
+    while page.next_link != nil do
+      page = page.get_next_page
+      items.concat(page.values)
+    end
+    expect(items.count).to eq(10)
   end
 
-  it 'should get multiple pages with odata kind nextLink' do
-    result = @client.paging.get_odata_multiple_pages_async().value!
-    expect(result.response.status).to eq(200)
-    expect(result.body.odatanext_link).not_to be_nil
+  it 'should get multiple pages' do
+    all_items = @client.paging.get_odata_multiple_pages() #returns items from all the pages
+    items_count = all_items.count
+    expect(items_count).to eq(10)
+    expect(all_items.is_a? Array)
+    expect(all_items.first.is_a? PagingModule::Models::Product)
+  end
 
-    count = 1
-    while result.body.odatanext_link != nil do
-      result = @client.paging.get_odata_multiple_pages_next_async(result.body.odatanext_link).value!
-      count += 1
+  it 'should get multiple pages with odata - lazy ' do
+    page = @client.paging.get_odata_multiple_pages_as_lazy()
+    expect(page.is_a? PagingModule::Models::OdataProductResult)
+    expect(page.odatanext_link).not_to be_nil
+    items = page.values
+    while page.odatanext_link != nil do
+      page = page.get_next_page
+      items.concat(page.values)
     end
-
-    expect(count).to eq(10)
+    expect(items.count).to eq(10)
   end
 
   it 'should get multiple pages with offset' do
     options = PagingModule::Models::PagingGetMultiplePagesWithOffsetOptions.new
     options.offset = 100
-    result = @client.paging.get_multiple_pages_with_offset_async(options).value!
-    expect(result.response.status).to eq(200)
-    expect(result.body.next_link).not_to be_nil
-
-    count = 1
-    while result.body.next_link != nil do
-      result = @client.paging.get_multiple_pages_with_offset_next_async(result.body.next_link).value!
-      count += 1
-    end
+    all_items = @client.paging.get_multiple_pages_with_offset(options)
+    expect(all_items.count).to eq(10)
+    expect(all_items.is_a? Array)
+    expect(all_items.first.is_a? PagingModule::Models::Product)
+    expect(all_items.last.properties.id).to eq (110)
+  end
 
-    expect(count).to eq(10)
-    expect(result.body.values.last.properties.id).to eq (110)
+  it 'should get multiple pages with offset - lazy' do
+    options = PagingModule::Models::PagingGetMultiplePagesWithOffsetOptions.new
+    options.offset = 100
+    page = @client.paging.get_multiple_pages_with_offset_as_lazy(options)
+    expect(page.is_a? PagingModule::Models::ProductResult)
+    expect(page.next_link).not_to be_nil
+    items = page.values
+    while page.next_link != nil do
+      page = page.get_next_page
+      items.concat(page.values)
+    end
+    expect(items.count).to eq(10)
+    expect(page.values.last.properties.id).to eq (110)
   end
 
   it 'should get multiple pages retry first' do
-    result = @client.paging.get_multiple_pages_retry_first_async().value!
-    expect(result.response.status).to eq(200)
-    expect(result.body.next_link).not_to be_nil
-
-    count = 1
-    while result.body.next_link != nil do
-      result = @client.paging.get_multiple_pages_retry_first_next_async(result.body.next_link).value!
-      count += 1
+    all_items = @client.paging.get_multiple_pages_retry_first()
+    expect(all_items.count).to eq(10)
+    expect(all_items.is_a? Array)
+    expect(all_items.first.is_a? PagingModule::Models::Product)
+  end
+
+  it 'should get multiple pages retry first - lazy' do
+    page = @client.paging.get_multiple_pages_retry_first_as_lazy
+    expect(page.is_a? PagingModule::Models::ProductResult)
+    expect(page.next_link).not_to be_nil
+    items = page.values
+    while page.next_link != nil do
+      page = page.get_next_page
+      items.concat(page.values)
     end
+    expect(items.count).to eq(10)
 
-    expect(count).to eq(10)
   end
 
   it 'should get multiple pages retry second' do
-    result = @client.paging.get_multiple_pages_retry_second_async().value!
-    expect(result.response.status).to eq(200)
-    expect(result.body.next_link).not_to be_nil
-
-    count = 1
-    while result.body.next_link != nil do
-      result = @client.paging.get_multiple_pages_retry_second_next_async(result.body.next_link).value!
-      count += 1
-    end
+    all_items = @client.paging.get_multiple_pages_retry_second()
+    expect(all_items.count).to eq(10)
+    expect(all_items.is_a? Array)
+    expect(all_items.first.is_a? PagingModule::Models::Product)
+  end
 
-    expect(count).to eq(10)
+  it 'should get multiple pages retry second - lazy' do
+    page = @client.paging.get_multiple_pages_retry_second_as_lazy
+    expect(page.is_a? PagingModule::Models::ProductResult)
+    expect(page.next_link).not_to be_nil
+    items = page.values
+    while page.next_link != nil do
+      page = page.get_next_page
+      items.concat(page.values)
+    end
+    expect(items.count).to eq(10)
   end
 
   # Paging sad path tests
   it 'should get single pages failure' do
-    expect { @client.paging.get_single_pages_failure_async().value! }.to raise_exception(MsRest::HttpOperationError)
+    expect { @client.paging.get_single_pages_failure_as_lazy }.to raise_exception(MsRest::HttpOperationError)
+  end
+
+  it 'should get single pages failure' do
+    expect { @client.paging.get_single_pages_failure() }.to raise_exception(MsRest::HttpOperationError)
   end
 
   it 'should get multiple pages failure' do
-    result = @client.paging.get_multiple_pages_failure_async().value!
-    expect(result.response.status).to eq(200)
-    expect(result.body.next_link).not_to be_nil
+    expect {  @client.paging.get_multiple_pages_failure()}.to raise_exception(MsRest::HttpOperationError)
+  end
+
+  it 'should get multiple pages failure - lazy' do
+    page = @client.paging.get_multiple_pages_failure_as_lazy
+    expect(page.is_a? PagingModule::Models::ProductResult)
+    expect(page.next_link).not_to be_nil
 
-    expect { @client.paging.get_multiple_pages_failure_next_async(result.body.next_link).value! }.to raise_exception(MsRest::HttpOperationError)
+    expect { page.get_next_page }.to raise_exception(MsRest::HttpOperationError)
   end
 
   it 'should get multiple pages failure URI' do
-    result = @client.paging.get_multiple_pages_failure_uri_async().value!
-    expect(result.response.status).to eq(200)
-    expect(result.body.next_link).not_to be_nil
+    expect {  @client.paging.get_multiple_pages_failure_uri()}.to raise_exception(MsRest::HttpOperationError)
+  end
+
+  it 'should get multiple pages failure URI - lazy' do
+    page = @client.paging.get_multiple_pages_failure_uri_as_lazy
+    expect(page.is_a? PagingModule::Models::ProductResult)
+    expect(page.next_link).not_to be_nil
 
-    expect { @client.paging.get_multiple_pages_failure_uri_next_async(result.body.next_link).value! }.to raise_exception(MsRest::HttpOperationError)
+    expect { page.get_next_page  }.to raise_exception(MsRest::HttpOperationError)
   end
-end
+end

src/generator/AutoRest.Ruby.Azure/AzureRubyCodeGenerator.cs

@@ -1,16 +1,20 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
+using System;
 using System.IO;
 using System.Linq;
+using System.Collections.Generic;
 using System.Threading.Tasks;
 using AutoRest.Core;
 using AutoRest.Core.ClientModel;
 using AutoRest.Extensions.Azure;
+using AutoRest.Extensions.Azure.Model;
+using AutoRest.Ruby.Templates;
 using AutoRest.Ruby.Azure.TemplateModels;
 using AutoRest.Ruby.Azure.Templates;
+using Newtonsoft.Json;
 using AutoRest.Ruby.TemplateModels;
-using AutoRest.Ruby.Templates;
 
 namespace AutoRest.Ruby.Azure
 {
@@ -19,12 +23,17 @@ namespace AutoRest.Ruby.Azure
     /// </summary>
     public class AzureRubyCodeGenerator : RubyCodeGenerator
     {
+
+        // List of models with paging extensions.
+        private IList<PageTemplateModel> pageModels;
+
         /// <summary>
         /// Initializes a new instance of the class AzureRubyCodeGenerator.
         /// </summary>
         /// <param name="settings">The settings.</param>
         public AzureRubyCodeGenerator(Settings settings) : base(settings)
         {
+            pageModels = new List<PageTemplateModel>();
         }
 
         /// <summary>
@@ -61,6 +70,79 @@ public override void NormalizeClientModel(ServiceClient serviceClient)
             AzureExtensions.NormalizeAzureClientModel(serviceClient, Settings, CodeNamer);
             CorrectFilterParameters(serviceClient);
             base.NormalizeClientModel(serviceClient);
+            AddRubyPageableMethod(serviceClient);
+            ApplyPagination(serviceClient);
+        }
+
+        /// <summary>
+        /// Adds method to use for autopagination.
+        /// </summary>
+        /// <param name="serviceClient">The service client.</param>
+        private void AddRubyPageableMethod(ServiceClient serviceClient)
+        {
+            if (serviceClient == null)
+            {
+                throw new ArgumentNullException(nameof(serviceClient));
+            }
+
+            for (int i = 0; i < serviceClient.Methods.Count; i++)
+            {
+                Method method = serviceClient.Methods[i];
+                if (method.Extensions.ContainsKey(AzureExtensions.PageableExtension))
+                {
+                    PageableExtension pageableExtension = JsonConvert.DeserializeObject<PageableExtension>(method.Extensions[AzureExtensions.PageableExtension].ToString());
+                    if (pageableExtension != null && !method.Extensions.ContainsKey("nextLinkMethod") && !string.IsNullOrWhiteSpace(pageableExtension.NextLinkName))
+                    {
+                        serviceClient.Methods.Insert(i, (Method)method.Clone());
+                        if (serviceClient.Methods[i].Extensions.ContainsKey("nextMethodName"))
+                        {
+                            serviceClient.Methods[i].Extensions["nextMethodName"] = CodeNamer.GetMethodName((string)method.Extensions["nextMethodName"]);
+                        }
+                        i++;
+                    }
+                    serviceClient.Methods[i].Extensions.Remove(AzureExtensions.PageableExtension);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Changes paginated method signatures to return Page type.
+        /// </summary>
+        /// <param name="serviceClient">The service client.</param>
+        private void ApplyPagination(ServiceClient serviceClient)
+        {
+            if (serviceClient == null)
+            {
+                throw new ArgumentNullException(nameof(serviceClient));
+            }
+
+            foreach (Method method in serviceClient.Methods.Where(m => m.Extensions.ContainsKey(AzureExtensions.PageableExtension)))
+            {
+                string nextLinkName = null;
+                var ext = method.Extensions[AzureExtensions.PageableExtension] as Newtonsoft.Json.Linq.JContainer;
+                if (ext == null)
+                {
+                    continue;
+                }
+                nextLinkName = CodeNamer.GetPropertyName((string)ext["nextLinkName"]);
+                string itemName = CodeNamer.GetPropertyName((string)ext["itemName"] ?? "value");
+                foreach (var responseStatus in method.Responses.Where(r => r.Value.Body is CompositeType).Select(s => s.Key).ToArray())
+                {
+                    CompositeType compositeType = (CompositeType)method.Responses[responseStatus].Body;
+                    SequenceType sequenceType = compositeType.Properties.Select(p => p.Type).FirstOrDefault(t => t is SequenceType) as SequenceType;
+
+                    // if the type is a wrapper over page-able response
+                    if (sequenceType != null)
+                    {
+                        compositeType.Extensions[AzureExtensions.PageableExtension] = true;
+                        PageTemplateModel pageTemplateModel = new PageTemplateModel(compositeType, serviceClient.ModelTypes, nextLinkName, itemName);
+                        if (!pageModels.Contains(pageTemplateModel))
+                        {
+                            pageModels.Add(pageTemplateModel);
+                        }
+                    }
+                }
+            }
         }
 
         /// <summary>
@@ -84,16 +166,17 @@ public static void CorrectFilterParameters(ServiceClient serviceClient)
         }
 
         /// <summary>
-        /// Generates C# code for service client.
+        /// Generates Ruby code for Azure service client.
         /// </summary>
         /// <param name="serviceClient">The service client.</param>
         /// <returns>Async tasks which generates SDK files.</returns>
         public override async Task Generate(ServiceClient serviceClient)
         {
+            var serviceClientTemplateModel = new AzureServiceClientTemplateModel(serviceClient);
             // Service client
             var serviceClientTemplate = new ServiceClientTemplate
             {
-                Model = new AzureServiceClientTemplateModel(serviceClient),
+                Model = serviceClientTemplateModel
             };
             await Write(serviceClientTemplate, Path.Combine(sdkPath, RubyCodeNamer.UnderscoreCase(serviceClient.Name) + ImplementationFileExtension));
 
@@ -109,7 +192,7 @@ public override async Task Generate(ServiceClient serviceClient)
             }
 
             // Models
-            foreach (var model in serviceClient.ModelTypes)
+            foreach (var model in serviceClientTemplateModel.ModelTypes)
             {
                 if ((model.Extensions.ContainsKey(AzureExtensions.ExternalExtension) &&
                     (bool)model.Extensions[AzureExtensions.ExternalExtension])
@@ -122,9 +205,17 @@ public override async Task Generate(ServiceClient serviceClient)
                 {
                     Model = new AzureModelTemplateModel(model, serviceClient.ModelTypes),
                 };
-
                 await Write(modelTemplate, Path.Combine(modelsPath, RubyCodeNamer.UnderscoreCase(model.Name) + ImplementationFileExtension));
             }
+            // Paged Models
+            foreach (var pageModel in pageModels)
+            {
+                var pageTemplate = new PageModelTemplate
+                {
+                    Model = pageModel
+                };
+                await Write(pageTemplate, Path.Combine(modelsPath, RubyCodeNamer.UnderscoreCase(pageModel.Name) + ImplementationFileExtension));
+            }
 
             // Enums
             foreach (var enumType in serviceClient.EnumTypes)