Use skiptoken to retrieve complete list of resources

balag0 · balag0 · commit 526106d2690b · 2017-10-19T10:53:05.000-07:00
Currently all calls to ARM APIs ignore skiptoken link. This meant only partial list of resources would be retrieved in some cases. This change adds support to walk the skiptoken url when retrieving  GET /subscriptions/{subscriptionId}/resources.
Note: Other API calls still have this limitation. Resource Search uses the same API but has not been updated to make use of this new functionality yet (only looks at top 100 resources in a subscription)
diff --git a/ARMExplorer.csproj b/ARMExplorer.csproj
@@ -376,6 +376,8 @@
     <Compile Include="Controllers\OperationController.cs" />
     <Compile Include="Controllers\OperationInfo.cs" />
     <Compile Include="Controllers\ArmRepository.cs" />
+    <Compile Include="Model\ArmResource.cs" />
+    <Compile Include="Model\ArmResourceListResult.cs" />
     <Compile Include="Modules\Extensions.cs" />
     <Compile Include="SwaggerParser\Model\CodeGenerationSettings.cs" />
     <Compile Include="SwaggerParser\Model\ComparisonContext.cs" />
diff --git a/Controllers/ArmRepository.cs b/Controllers/ArmRepository.cs
@@ -6,6 +6,7 @@
 using System.Net.Http.Headers;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+using ARMExplorer.Model;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 
@@ -14,6 +15,7 @@ namespace ARMExplorer.Controllers
     public class ArmRepository : IArmRepository 
     {
         private readonly IHttpClientWrapper _clientWrapper;
+        private readonly int _maxNextLinkDepth = 5;
 
         public ArmRepository(IHttpClientWrapper clientWrapper)
         {
@@ -40,17 +42,63 @@ public async Task<IList<string>> GetSubscriptionIdsAsync(HttpRequestMessage requ
             return subscriptionIds;
         }
 
+        private static bool AddResourceToList(IEnumerable<ArmResource> resources, ISet<ArmResource> allResources)
+        {
+            var initalCount = allResources.Count;
+
+            foreach (var resource in resources)
+            {
+                allResources.Add(resource);
+            }
+
+            var updatedCount = allResources.Count;
+
+            return updatedCount > initalCount;
+        }
+
+        private async Task<HashSet<ArmResource>> GetResources(HttpRequestMessage requestMessage, string getResourcesUrl)
+        {
+            var allResources = new HashSet<ArmResource>();
+            var currentNextLinkDepth = 0;
+
+            while (!string.IsNullOrEmpty(getResourcesUrl))
+            {
+                var response = await GetAsync(requestMessage, getResourcesUrl);
+                response.EnsureSuccessStatusCode();
+                var armResourceListResult = await response.Content.ReadAsAsync<ArmResourceListResult>();
+
+                var newResourceFound = AddResourceToList(armResourceListResult.Value, allResources);
+
+                // ARM API returns the same skiptoken and resources repeatedly when there are no more resources. To avoid infinite cycle break when
+                // 1. No new resource was found in the current response or
+                // 2. Limit the max number of links to follow to _maxNextLinkDepth or
+                // 3. When nextLink is empty
+
+                if (!newResourceFound)
+                {
+                    break;
+                }
+
+                if (currentNextLinkDepth++ > _maxNextLinkDepth)
+                {
+                    break;
+                }
+
+                getResourcesUrl = armResourceListResult.NextLink;
+            }
+
+            return allResources;
+        }
+
         public async Task<HashSet<string>> GetProviderNamesFor(HttpRequestMessage requestMessage, string subscriptionId)
         {
-            var response = await GetResourcesForAsync(requestMessage, subscriptionId);
-            response.EnsureSuccessStatusCode();
-            dynamic resources = await response.Content.ReadAsAsync<JObject>();
-            JArray values = resources.value;
+            var initialGetResourcesUrl = string.Format(Utils.ResourcesTemplate, HyakUtils.CSMUrl, subscriptionId, Utils.CSMApiVersion);
+            var resources = await GetResources(requestMessage, initialGetResourcesUrl);
             var uniqueProviders = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
-            foreach (dynamic value in values)
+
+            foreach (var resource in resources)
             {
-                string id = value.id;
-                var match = Regex.Match(id, "/subscriptions/.*?/resourceGroups/(.*?)/providers/(.*?)/(.*?)/");
+                var match = Regex.Match(resource.Id, "/subscriptions/.*?/resourceGroups/(.*?)/providers/(.*?)/(.*?)/");
                 if (match.Success)
                 {
                     var provider = match.Groups[2].Value.ToUpperInvariant();
@@ -63,15 +111,13 @@ public async Task<HashSet<string>> GetProviderNamesFor(HttpRequestMessage reques
 
         public async Task<Dictionary<string, Dictionary<string, HashSet<string>>>> GetProvidersFor(HttpRequestMessage requestMessage, string subscriptionId)
         {
-            var response = await GetResourcesForAsync(requestMessage, subscriptionId);
-            response.EnsureSuccessStatusCode();
-
-            dynamic resources = await response.Content.ReadAsAsync<JObject>();
-            JArray values = resources.value;
+            var initialGetResourcesUrl = string.Format(Utils.ResourcesTemplate, HyakUtils.CSMUrl, subscriptionId, Utils.CSMApiVersion);
+            var resources = await GetResources(requestMessage, initialGetResourcesUrl);
             var result = new Dictionary<string, Dictionary<string, HashSet<string>>>();
-            foreach (dynamic value in values)
+
+            foreach (var resource in resources)
             {
-                string id = value.id;
+                string id = resource.Id;
                 var match = Regex.Match(id, "/subscriptions/.*?/resourceGroups/(.*?)/providers/(.*?)/(.*?)/");
                 if (match.Success)
                 {
@@ -116,10 +162,10 @@ private async Task<HttpResponseMessage> GetSubscriptionsAsync(HttpRequestMessage
             return await _clientWrapper.SendAsync(requestMessage, sendRequest);
         }
 
-        private async Task<HttpResponseMessage> GetResourcesForAsync(HttpRequestMessage requestMessage, string subscriptionId)
+        private async Task<HttpResponseMessage> GetAsync(HttpRequestMessage requestMessage, string url)
         {
-            var sendRequest = new HttpRequestMessage(HttpMethod.Get, string.Format(Utils.ResourcesTemplate, HyakUtils.CSMUrl, subscriptionId, Utils.CSMApiVersion));
-            return await _clientWrapper.SendAsync(requestMessage, sendRequest);
+            var sendRequest = new HttpRequestMessage(HttpMethod.Get, url);
+            return await _clientWrapper.ExecuteAsync(requestMessage, sendRequest);
         }
     }
 }
diff --git a/Model/ArmResource.cs b/Model/ArmResource.cs
@@ -0,0 +1,31 @@
+﻿using System;
+
+namespace ARMExplorer.Model
+{
+    public class ArmResource : IEquatable<ArmResource>
+    {
+        [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public string Id { get; set; }
+        // other fields ignored
+
+        public bool Equals(ArmResource other)
+        {
+            if (ReferenceEquals(null, other)) return false;
+            if (ReferenceEquals(this, other)) return true;
+            return string.Equals(Id, other.Id, StringComparison.OrdinalIgnoreCase);
+        }
+
+        public override bool Equals(object obj)
+        {
+            if (ReferenceEquals(null, obj)) return false;
+            if (ReferenceEquals(this, obj)) return true;
+            if (obj.GetType() != this.GetType()) return false;
+            return Equals((ArmResource) obj);
+        }
+
+        public override int GetHashCode()
+        {
+            return Id != null ? Id.GetHashCode() : 0;
+        }
+    }
+}
diff --git a/Model/ArmResourceListResult.cs b/Model/ArmResourceListResult.cs
@@ -0,0 +1,13 @@
+﻿using System.Collections.ObjectModel;
+
+namespace ARMExplorer.Model
+{
+    public class ArmResourceListResult
+    {
+        [Newtonsoft.Json.JsonProperty("value", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public Collection<ArmResource> Value { get; set; }
+
+        [Newtonsoft.Json.JsonProperty("nextLink", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public string NextLink { get; set; }
+    }
+}
diff --git a/Tests/WebApiTests/MockHttpClientWrapper.cs b/Tests/WebApiTests/MockHttpClientWrapper.cs
@@ -30,7 +30,20 @@ public Task<HttpResponseMessage> SendAsync(HttpRequestMessage requestMessage, Ht
 
         public Task<HttpResponseMessage> ExecuteAsync(HttpRequestMessage requestMessage, HttpRequestMessage executeRequest)
         {
-            throw new NotImplementedException();
+            var responseMessage = new HttpResponseMessage(HttpStatusCode.OK);
+            string filePath;
+            if (executeRequest.RequestUri.ToString().Contains("resources"))
+            {
+                filePath = Path.Combine(new DirectoryInfo(Directory.GetCurrentDirectory()).FullName, Path.Combine("WebApiTests", "data", "resourcesForsubscription.json"));
+            }
+            else
+            {
+                filePath = Path.Combine(new DirectoryInfo(Directory.GetCurrentDirectory()).FullName, Path.Combine("WebApiTests", "data", "subscriptions.json"));
+            }
+            responseMessage.Content = new StringContent(File.ReadAllText(filePath), Encoding.UTF8, "application/json");
+            var response = new TaskCompletionSource<HttpResponseMessage>();
+            response.SetResult(responseMessage);
+            return response.Task;
         }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -30,7 +30,20 @@ public Task<HttpResponseMessage> SendAsync(HttpRequestMessage requestMessage, Ht`
`30`	`30`
`31`	`31`	`public Task<HttpResponseMessage> ExecuteAsync(HttpRequestMessage requestMessage, HttpRequestMessage executeRequest)`
`32`	`32`	`{`
`33`		`- throw new NotImplementedException();`
	`33`	`+ var responseMessage = new HttpResponseMessage(HttpStatusCode.OK);`
	`34`	`+ string filePath;`
	`35`	`+ if (executeRequest.RequestUri.ToString().Contains("resources"))`
	`36`	`+ {`
	`37`	`+ filePath = Path.Combine(new DirectoryInfo(Directory.GetCurrentDirectory()).FullName, Path.Combine("WebApiTests", "data", "resourcesForsubscription.json"));`
	`38`	`+ }`
	`39`	`+ else`
	`40`	`+ {`
	`41`	`+ filePath = Path.Combine(new DirectoryInfo(Directory.GetCurrentDirectory()).FullName, Path.Combine("WebApiTests", "data", "subscriptions.json"));`
	`42`	`+ }`
	`43`	`+ responseMessage.Content = new StringContent(File.ReadAllText(filePath), Encoding.UTF8, "application/json");`
	`44`	`+ var response = new TaskCompletionSource<HttpResponseMessage>();`
	`45`	`+ response.SetResult(responseMessage);`
	`46`	`+ return response.Task;`
`34`	`47`	`}`
`35`	`48`	`}`
`36`	`49`	`}`