From afabd8688d9fa8a1e0a07596b7715a52f7b1561a Mon Sep 17 00:00:00 2001 From: Varun Puranik Date: Mon, 11 Mar 2019 13:30:56 -0700 Subject: [PATCH 1/4] EdgeAgent: Add support for uploading blobs to Azure storage container (#914) * Add support for AzureBlobLogsUploader * Cleanup and add tests * Fix build * Rename file correctly --- .../AgentEventIds.cs | 1 + .../logs/ILogsUploader.cs | 10 ++ .../logs/LogsContentEncoding.cs | 9 + .../logs/LogsContentType.cs | 9 + .../blob/AzureBlob.cs | 23 +++ .../blob/AzureBlobLogsUploader.cs | 155 ++++++++++++++++++ .../blob/AzureBlobUploader.cs | 20 +++ .../blob/IAzureBlob.cs | 15 ++ .../blob/IAzureBlobUploader.cs | 13 ++ .../blob/AzureBlobLogsUploaderTest.cs | 113 +++++++++++++ 10 files changed, 368 insertions(+) create mode 100644 edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ILogsUploader.cs create mode 100644 edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsContentEncoding.cs create mode 100644 edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsContentType.cs create mode 100644 edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlob.cs create mode 100644 edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobLogsUploader.cs create mode 100644 edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobUploader.cs create mode 100644 edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/IAzureBlob.cs create mode 100644 edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/IAzureBlobUploader.cs create mode 100644 edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/blob/AzureBlobLogsUploaderTest.cs diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/AgentEventIds.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/AgentEventIds.cs index 2e149fd02d1..92f252773e7 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/AgentEventIds.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/AgentEventIds.cs @@ -21,6 +21,7 @@ public struct AgentEventIds public const int ModuleManagementHttpClient = EventIdStart + 1500; public const int ModuleIdentityLifecycleManager = EventIdStart + 1600; public const int RequestManager = EventIdStart + 1700; + public const int AzureBlobLogsUploader = EventIdStart + 1800; const int EventIdStart = 100000; } } diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ILogsUploader.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ILogsUploader.cs new file mode 100644 index 00000000000..189590aa49b --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/ILogsUploader.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Core.Logs +{ + using System.Threading.Tasks; + + public interface ILogsUploader + { + Task Upload(string uri, string module, byte[] payload, LogsContentEncoding logsContentEncoding, LogsContentType logsContentType); + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsContentEncoding.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsContentEncoding.cs new file mode 100644 index 00000000000..3473305cf04 --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsContentEncoding.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Core.Logs +{ + public enum LogsContentEncoding + { + None, + Gzip + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsContentType.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsContentType.cs new file mode 100644 index 00000000000..3c345a15b84 --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/logs/LogsContentType.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Core.Logs +{ + public enum LogsContentType + { + Json, + Text + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlob.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlob.cs new file mode 100644 index 00000000000..36d0aca08dd --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlob.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.IoTHub.Blob +{ + using System.Threading.Tasks; + using Microsoft.Azure.Devices.Edge.Util; + using Microsoft.WindowsAzure.Storage.Blob; + + class AzureBlob : IAzureBlob + { + readonly CloudBlockBlob blockBlob; + + public AzureBlob(CloudBlockBlob blockBlob) + { + this.blockBlob = Preconditions.CheckNotNull(blockBlob, nameof(blockBlob)); + } + + public string Name => this.blockBlob.Name; + + public BlobProperties BlobProperties => this.blockBlob.Properties; + + public Task UploadFromByteArrayAsync(byte[] bytes) => this.blockBlob.UploadFromByteArrayAsync(bytes, 0, bytes.Length); + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobLogsUploader.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobLogsUploader.cs new file mode 100644 index 00000000000..90c6f8be7ff --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobLogsUploader.cs @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.IoTHub.Blob +{ + using System; + using System.Globalization; + using System.Threading.Tasks; + using Microsoft.Azure.Devices.Edge.Agent.Core; + using Microsoft.Azure.Devices.Edge.Agent.Core.Logs; + using Microsoft.Azure.Devices.Edge.Util; + using Microsoft.Azure.Devices.Edge.Util.TransientFaultHandling; + using Microsoft.Extensions.Logging; + using Microsoft.WindowsAzure.Storage.Blob; + + public class AzureBlobLogsUploader : ILogsUploader + { + const int RetryCount = 2; + + static readonly ITransientErrorDetectionStrategy TransientErrorDetectionStrategy = new ErrorDetectionStrategy(); + + static readonly RetryStrategy TransientRetryStrategy = + new ExponentialBackoff(RetryCount, TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(20), TimeSpan.FromSeconds(4)); + + readonly string iotHubName; + readonly string deviceId; + readonly IAzureBlobUploader azureBlobUploader; + + public AzureBlobLogsUploader(string iotHubName, string deviceId, IAzureBlobUploader azureBlobUploader) + { + this.iotHubName = Preconditions.CheckNonWhiteSpace(iotHubName, nameof(iotHubName)); + this.deviceId = Preconditions.CheckNonWhiteSpace(deviceId, nameof(deviceId)); + this.azureBlobUploader = Preconditions.CheckNotNull(azureBlobUploader, nameof(azureBlobUploader)); + } + + public async Task Upload(string uri, string id, byte[] payload, LogsContentEncoding logsContentEncoding, LogsContentType logsContentType) + { + Preconditions.CheckNonWhiteSpace(uri, nameof(uri)); + Preconditions.CheckNonWhiteSpace(id, nameof(id)); + Preconditions.CheckNotNull(payload, nameof(payload)); + + try + { + var containerUri = new Uri(uri); + string blobName = this.GetBlobName(id, logsContentEncoding, logsContentType); + var container = new CloudBlobContainer(containerUri); + Events.Uploading(blobName, container.Name); + await ExecuteWithRetry( + () => + { + IAzureBlob blob = this.azureBlobUploader.GetBlob(containerUri, blobName); + SetContentEncoding(blob, logsContentEncoding); + SetContentType(blob, logsContentType); + return blob.UploadFromByteArrayAsync(payload); + }, + r => Events.UploadErrorRetrying(blobName, container.Name, r)); + Events.UploadSuccess(blobName, container.Name); + } + catch (Exception e) + { + Events.UploadError(e, id); + throw; + } + } + + internal static string GetExtension(LogsContentEncoding logsContentEncoding, LogsContentType logsContentType) + { + if (logsContentEncoding == LogsContentEncoding.Gzip) + { + return "gz"; + } + + if (logsContentType == LogsContentType.Json) + { + return "json"; + } + + return "log"; + } + + internal string GetBlobName(string id, LogsContentEncoding logsContentEncoding, LogsContentType logsContentType) + { + string extension = GetExtension(logsContentEncoding, logsContentType); + string blobName = $"{id}-{DateTime.UtcNow.ToString("yyyy-MM-dd--HH-mm-ss", CultureInfo.InvariantCulture)}"; + return $"{this.iotHubName}/{this.deviceId}/{blobName}.{extension}"; + } + + static void SetContentType(IAzureBlob blob, LogsContentType logsContentType) + { + switch (logsContentType) + { + case LogsContentType.Json: + blob.BlobProperties.ContentType = "application/json"; + break; + case LogsContentType.Text: + blob.BlobProperties.ContentType = "text/plain"; + break; + } + } + + static void SetContentEncoding(IAzureBlob blob, LogsContentEncoding logsContentEncoding) + { + switch (logsContentEncoding) + { + case LogsContentEncoding.Gzip: + blob.BlobProperties.ContentEncoding = "gzip"; + break; + } + } + + static Task ExecuteWithRetry(Func func, Action onRetry) + { + var transientRetryPolicy = new RetryPolicy(TransientErrorDetectionStrategy, TransientRetryStrategy); + transientRetryPolicy.Retrying += (_, args) => onRetry(args); + return transientRetryPolicy.ExecuteAsync(func); + } + + class ErrorDetectionStrategy : ITransientErrorDetectionStrategy + { + public bool IsTransient(Exception ex) => !ex.IsFatal(); + } + + static class Events + { + const int IdStart = AgentEventIds.AzureBlobLogsUploader; + static readonly ILogger Log = Logger.Factory.CreateLogger(); + + enum EventIds + { + Uploading = IdStart + 1, + UploadSuccess, + UploadErrorRetrying, + UploadError + } + + public static void Uploading(string blobName, string container) + { + Log.LogInformation((int)EventIds.Uploading, $"Uploading blob {blobName} to container {container}"); + } + + public static void UploadSuccess(string blobName, string container) + { + Log.LogDebug((int)EventIds.UploadSuccess, $"Successfully uploaded blob {blobName} to container {container}"); + } + + public static void UploadErrorRetrying(string blobName, string container, RetryingEventArgs retryingEventArgs) + { + Log.LogDebug((int)EventIds.UploadErrorRetrying, retryingEventArgs.LastException, $"Error uploading {blobName} to container {container}. Retry count - {retryingEventArgs.CurrentRetryCount}"); + } + + public static void UploadError(Exception ex, string module) + { + Log.LogDebug((int)EventIds.UploadError, ex, $"Error uploading logs for {module}"); + } + } + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobUploader.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobUploader.cs new file mode 100644 index 00000000000..79777ed6e23 --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobUploader.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.IoTHub.Blob +{ + using System; + using System.Threading.Tasks; + using Microsoft.WindowsAzure.Storage.Blob; + + class AzureBlobUploader : IAzureBlobUploader + { + public IAzureBlob GetBlob(Uri containerUri, string blobName) + { + var container = new CloudBlobContainer(containerUri); + CloudBlockBlob blob = container.GetBlockBlobReference(blobName); + var azureBlob = new AzureBlob(blob); + return azureBlob; + } + + public Task UploadBlob(IAzureBlob blob, byte[] bytes) => blob.UploadFromByteArrayAsync(bytes); + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/IAzureBlob.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/IAzureBlob.cs new file mode 100644 index 00000000000..1219791f548 --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/IAzureBlob.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.IoTHub.Blob +{ + using System.Threading.Tasks; + using Microsoft.WindowsAzure.Storage.Blob; + + public interface IAzureBlob + { + string Name { get; } + + BlobProperties BlobProperties { get; } + + Task UploadFromByteArrayAsync(byte[] bytes); + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/IAzureBlobUploader.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/IAzureBlobUploader.cs new file mode 100644 index 00000000000..9fbc4496775 --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/IAzureBlobUploader.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.IoTHub.Blob +{ + using System; + using System.Threading.Tasks; + + public interface IAzureBlobUploader + { + IAzureBlob GetBlob(Uri containerUri, string blobName); + + Task UploadBlob(IAzureBlob blob, byte[] bytes); + } +} diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/blob/AzureBlobLogsUploaderTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/blob/AzureBlobLogsUploaderTest.cs new file mode 100644 index 00000000000..9f52e74aaa4 --- /dev/null +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/blob/AzureBlobLogsUploaderTest.cs @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test.Blob +{ + using System; + using System.Globalization; + using System.Text; + using System.Text.RegularExpressions; + using System.Threading.Tasks; + using Microsoft.Azure.Devices.Edge.Agent.Core.Logs; + using Microsoft.Azure.Devices.Edge.Agent.IoTHub.Blob; + using Microsoft.Azure.Devices.Edge.Util.Test.Common; + using Microsoft.WindowsAzure.Storage.Blob; + using Moq; + using Xunit; + using Match = System.Text.RegularExpressions.Match; + + [Unit] + public class AzureBlobLogsUploaderTest + { + const string BlobNameRegexPattern = @"(?.*)/(?.*)/(?.*)-(?\d{4}-\d{2}-\d{2}--\d{2}-\d{2}-\d{2}).(?\w{2})"; + + [Theory] + [InlineData(LogsContentEncoding.Gzip, LogsContentType.Json, "gz")] + [InlineData(LogsContentEncoding.Gzip, LogsContentType.Text, "gz")] + [InlineData(LogsContentEncoding.None, LogsContentType.Json, "json")] + [InlineData(LogsContentEncoding.None, LogsContentType.Text, "log")] + public void GetExtensionTest(LogsContentEncoding contentEncoding, LogsContentType contentType, string expectedExtension) + { + Assert.Equal(expectedExtension, AzureBlobLogsUploader.GetExtension(contentEncoding, contentType)); + } + + [Fact] + public void GetBlobNameTest() + { + // Arrange + string iotHub = "foo.azure-devices.net"; + string deviceId = "abcd"; + string id = "pqr"; + string extension = "gz"; + + var regex = new Regex(BlobNameRegexPattern); + + var azureBlobLogsUploader = new AzureBlobLogsUploader(iotHub, deviceId, Mock.Of()); + + // Act + string blobName = azureBlobLogsUploader.GetBlobName(id, LogsContentEncoding.Gzip, LogsContentType.Json); + + // Assert + Assert.NotNull(blobName); + Match match = regex.Match(blobName); + Assert.True(match.Success); + string receivedIotHub = match.Groups["iothub"].Value; + string receivedDeviceId = match.Groups["deviceid"].Value; + string receivedId = match.Groups["id"].Value; + string receivedTimestamp = match.Groups["timestamp"].Value; + string receivedExtension = match.Groups["extension"].Value; + Assert.Equal(id, receivedId); + Assert.Equal(iotHub, receivedIotHub); + Assert.Equal(deviceId, receivedDeviceId); + Assert.Equal(extension, receivedExtension); + Assert.True(DateTime.UtcNow - DateTime.ParseExact(receivedTimestamp, "yyyy-MM-dd--HH-mm-ss", CultureInfo.InvariantCulture) < TimeSpan.FromSeconds(10)); + } + + [Fact] + public async Task UploadTest() + { + // Arrange + string iotHub = "foo.azure-devices.net"; + string deviceId = "abcd"; + string id = "pqr"; + string sasUri = @"http://testuri/"; + var regex = new Regex(BlobNameRegexPattern); + + string receivedBlobName = null; + Uri receivedSasUri = null; + byte[] receivedPayload = null; + + byte[] payload = Encoding.UTF8.GetBytes("Test payload string"); + + var azureBlob = new Mock(); + azureBlob.Setup(a => a.BlobProperties) + .Returns(new BlobProperties()); + azureBlob.Setup(a => a.Name) + .Returns(() => receivedBlobName); + azureBlob.Setup(a => a.UploadFromByteArrayAsync(payload)) + .Callback(b => receivedPayload = b) + .Returns(Task.CompletedTask); + + var azureBlobUploader = new Mock(); + azureBlobUploader.Setup(a => a.GetBlob(It.IsAny(), It.IsAny())) + .Callback((u, b) => + { + receivedSasUri = u; + receivedBlobName = b; + }) + .Returns(azureBlob.Object); + + var azureBlobLogsUploader = new AzureBlobLogsUploader(iotHub, deviceId, azureBlobUploader.Object); + + // Act + await azureBlobLogsUploader.Upload(sasUri, id, payload, LogsContentEncoding.Gzip, LogsContentType.Json); + + // Assert + Assert.NotNull(receivedBlobName); + Match match = regex.Match(receivedBlobName); + Assert.True(match.Success); + Assert.NotNull(receivedSasUri); + Assert.Equal(sasUri, receivedSasUri.ToString()); + Assert.NotNull(receivedPayload); + Assert.Equal(payload, receivedPayload); + } + } +} From bb6c3271b035579ffb4e30af5fa4ab3637cf49f0 Mon Sep 17 00:00:00 2001 From: Varun Puranik Date: Mon, 11 Mar 2019 14:04:34 -0700 Subject: [PATCH 2/4] Fix Sasl Plain Authenticator (#903) * Fix Sasl Plain Authenticator * Add comment --- .../EdgeSaslPlainAuthenticator.cs | 8 +++-- .../EdgeSaslPlainAuthenticatorTest.cs | 31 +++++++++++++++++-- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Amqp/EdgeSaslPlainAuthenticator.cs b/edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Amqp/EdgeSaslPlainAuthenticator.cs index d94a31c6bbd..6d640a0dc43 100644 --- a/edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Amqp/EdgeSaslPlainAuthenticator.cs +++ b/edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Amqp/EdgeSaslPlainAuthenticator.cs @@ -38,9 +38,13 @@ public async Task AuthenticateAsync(string identity, string password throw new EdgeHubConnectionException("Identity does not contain device ID."); } - if (!this.iotHubHostName.Equals(iotHubName)) + // iotHubName can be a segment of the full iotHubHostName. + // For example, if iotHubHostName = testhub1.azure-devices.net, + // then iotHubName = testhub1 is valid. + if (!this.iotHubHostName.StartsWith(iotHubName, StringComparison.OrdinalIgnoreCase) || + this.iotHubHostName[iotHubName.Length] != '.') { - throw new EdgeHubConnectionException($"Identity contains an invalid IotHubHostName {iotHubName}, expected value {this.iotHubHostName}."); + throw new EdgeHubConnectionException($"Identity contains an invalid IotHubHostName {iotHubName}."); } // TODO: Figure out where the device client type parameter value should come from. diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Amqp.Test/EdgeSaslPlainAuthenticatorTest.cs b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Amqp.Test/EdgeSaslPlainAuthenticatorTest.cs index 93d2d1d88f5..0a88463f6bd 100644 --- a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Amqp.Test/EdgeSaslPlainAuthenticatorTest.cs +++ b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Amqp.Test/EdgeSaslPlainAuthenticatorTest.cs @@ -85,11 +85,11 @@ public async void TestAuthFailed() [Fact] [Unit] - public async void TestAuthSucceeds() + public async void TestAuthSucceeds_Module() { var authenticator = Mock.Of(); var clientCredentialsFactory = Mock.Of(); - var saslAuthenticator = new EdgeSaslPlainAuthenticator(authenticator, clientCredentialsFactory, "hub1"); + var saslAuthenticator = new EdgeSaslPlainAuthenticator(authenticator, clientCredentialsFactory, "hub1.azure-devices.net"); var identity = new ModuleIdentity("hub1", "dev1", "mod1"); var clientCredentials = Mock.Of(c => c.Identity == identity); const string UserId = "dev1/modules/mod1@sas.hub1"; @@ -109,5 +109,32 @@ public async void TestAuthSucceeds() bool isAuthenticated = await amqpAuthenticator.AuthenticateAsync("dev1/mod1"); Assert.True(isAuthenticated); } + + [Fact] + [Unit] + public async void TestAuthSucceeds_Device() + { + var authenticator = Mock.Of(); + var clientCredentialsFactory = Mock.Of(); + var saslAuthenticator = new EdgeSaslPlainAuthenticator(authenticator, clientCredentialsFactory, "hub1.azure-devices.net"); + var identity = new DeviceIdentity("hub1", "dev1"); + var clientCredentials = Mock.Of(c => c.Identity == identity); + const string UserId = "dev1@sas.hub1"; + const string Password = "pwd"; + + Mock.Get(clientCredentialsFactory).Setup(f => f.GetWithSasToken("dev1", string.Empty, string.Empty, Password, false)) + .Returns(clientCredentials); + Mock.Get(authenticator).Setup(a => a.AuthenticateAsync(clientCredentials)) + .ReturnsAsync(true); + + IPrincipal principal = await saslAuthenticator.AuthenticateAsync(UserId, Password); + Assert.NotNull(principal); + + var amqpAuthenticator = principal as IAmqpAuthenticator; + Assert.NotNull(amqpAuthenticator); + + bool isAuthenticated = await amqpAuthenticator.AuthenticateAsync("dev1"); + Assert.True(isAuthenticated); + } } } From a8cfd5c790a77e51202cf0bc443cdceb2a4ccad5 Mon Sep 17 00:00:00 2001 From: Angelo Ribeiro Date: Mon, 11 Mar 2019 14:29:05 -0700 Subject: [PATCH 3/4] Generating ARM64 Images (#941) Changes are: 1-Adding a new job on our Yaml Image (Since we have to use a different version of .NET Core; 2-Adding a line on our Manifest.yaml files to add arm64; 3-Change our Docker files to have a logic to handle a different .NET Core for ARM64; 4-Some changes on our build script to pass .NET Core Runtime configuration on build and publish. --- builds/misc/images.yaml | 118 ++++++++++++++++++ builds/misc/templates/image-linux.yaml | 2 +- edge-agent/docker/manifest.yaml.template | 5 + ...ft.Azure.Devices.Edge.Agent.Service.csproj | 4 +- ....Azure.Devices.Edge.Agent.Core.Test.csproj | 5 +- ....Devices.Edge.Agent.Docker.E2E.Test.csproj | 5 +- ...zure.Devices.Edge.Agent.Docker.Test.csproj | 5 +- ...ices.Edge.Agent.Edgelet.Docker.Test.csproj | 5 +- ...ure.Devices.Edge.Agent.Edgelet.Test.csproj | 14 ++- ...zure.Devices.Edge.Agent.IoTHub.Test.csproj | 5 +- edge-hub/docker/linux/arm64v8/base/Dockerfile | 1 + edge-hub/docker/manifest.yaml.template | 5 + ...soft.Azure.Devices.Edge.Hub.Service.csproj | 4 +- ...ft.Azure.Devices.Edge.Hub.Amqp.Test.csproj | 7 +- ...re.Devices.Edge.Hub.CloudProxy.Test.csproj | 5 +- ...ft.Azure.Devices.Edge.Hub.Core.Test.csproj | 5 +- ...oft.Azure.Devices.Edge.Hub.E2E.Test.csproj | 5 +- ...ft.Azure.Devices.Edge.Hub.Http.Test.csproj | 5 +- ...ft.Azure.Devices.Edge.Hub.Mqtt.Test.csproj | 5 +- ...Azure.Devices.Edge.Hub.Service.Test.csproj | 5 +- ...oft.Azure.Devices.Routing.Core.Test.csproj | 5 +- .../DirectMethodCloudSender.csproj | 5 +- .../DirectMethodReceiver.csproj | 5 +- .../docker/linux/arm64v8/Dockerfile | 2 +- .../DirectMethodSender.csproj | 5 +- .../MessagesAnalyzer/MessagesAnalyzer.csproj | 5 +- ...osoft.Azure.Devices.Edge.ModuleUtil.csproj | 5 +- .../SimulatedTemperatureSensor.csproj | 5 +- .../docker/manifest.yaml.template | 5 + .../TemperatureFilter.csproj | 5 +- ...ft.Azure.WebJobs.Extensions.EdgeHub.csproj | 4 +- .../EdgeHubTriggerCSharp.csproj | 6 +- edge-modules/load-gen/load-gen.csproj | 5 +- ...e.Devices.Edge.Storage.RocksDb.Test.csproj | 7 +- ...oft.Azure.Devices.Edge.Storage.Test.csproj | 7 +- ...rosoft.Azure.Devices.Edge.Util.Test.csproj | 14 ++- edgelet/Cross.toml | 5 + .../docker/manifest.yaml.template | 5 + scripts/linux/buildBranch.sh | 40 ++++-- scripts/linux/buildDiagnostics.sh | 5 +- scripts/linux/docker/manifest.yaml.template | 5 + .../IotEdgeQuickstart.csproj | 4 +- smoke/LeafDevice/LeafDevice.csproj | 4 +- 43 files changed, 272 insertions(+), 101 deletions(-) diff --git a/builds/misc/images.yaml b/builds/misc/images.yaml index 09c94ad9bbb..3c35110db12 100644 --- a/builds/misc/images.yaml +++ b/builds/misc/images.yaml @@ -5,7 +5,124 @@ trigger: - master pr: none jobs: + +################################################################################ + - job: linuxARM64 +################################################################################ + displayName: LinuxARM64 + pool: + name: Hosted Ubuntu 1604 + vmImage: 'ubuntu-16.04' + variables: + NetCorePackageUri: https://download.visualstudio.microsoft.com/download/pr/efa6dde9-a5ee-4322-b13c-a2a02d3980f0/dad445eba341c1d806bae5c8afb47015/dotnet-sdk-3.0.100-preview-010184-linux-x64.tar.gz + steps: + - script: scripts/linux/installPrereqs.sh -u $(NetCorePackageUri) + name: install + displayName: Install dependencies + - bash: 'docker login $(registry.address) --username $(registry.user) --password $(registry.password)' + displayName: 'Docker Login' + + - script: edgelet/build/linux/install.sh --package-arm + displayName: Install Rust + + - bash: 'echo "##vso[task.setvariable variable=PATH;]$HOME/.cargo/bin:$PATH"' + displayName: Modify path + + - bash: 'cargo install --git https://github.com/arsing/cross.git --branch set-path' + displayName: 'Install cross (fork with docker fix)' + + - script: scripts/linux/buildBranch.sh -c Release --no-rocksdb-bin --os Unix --dotnet_runtime netcoreapp3.0 + name: build + displayName: Build (release) + + - script: scripts/linux/buildDiagnostics.sh + displayName: Build iotedge-diagnostics + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifacts' + inputs: + PathtoPublish: '$(Build.BinariesDirectory)/publish' + ArtifactName: 'core-linux' + + # azureiotedge-diagnostics - Not Using Template for ARM64 because we have 2 different .NET Core. + - task: Bash@3 + displayName: Build Image azureiotedge-diagnostics - aarch64 + inputs: + filePath: scripts/linux/buildImage.sh + arguments: -r $(registry.address) -u $(registry.user) -p $(registry.password) -i azureiotedge-diagnostics -n microsoft -P azureiotedge-diagnostics --target-arch aarch64 + + # Edge Agent - Not Using Template for ARM64 because we have 2 different .NET Core. + - task: Bash@3 + displayName: Build Edge Agent Image - aarch64 + inputs: + filePath: scripts/linux/buildImage.sh + arguments: -r $(registry.address) -u $(registry.user) -p $(registry.password) -i azureiotedge-agent -n microsoft -P Microsoft.Azure.Devices.Edge.Agent.Service --target-arch aarch64 + + # Edge Hub - Not Using Template for ARM64 because we have 2 different .NET Core. + - task: Bash@3 + displayName: Build Edge Hub Image - aarch64 + inputs: + filePath: scripts/linux/buildImage.sh + arguments: -r $(registry.address) -u $(registry.user) -p $(registry.password) -i azureiotedge-hub -n microsoft -P Microsoft.Azure.Devices.Edge.Hub.Service --target-arch aarch64 + + # Simulated Temperature Sensor - Not Using Template for ARM64 because we have 2 different .NET Core. + - task: Bash@3 + displayName: Build Image Temperature Sensor - aarch64 + inputs: + filePath: scripts/linux/buildImage.sh + arguments: -r $(registry.address) -u $(registry.user) -p $(registry.password) -i azureiotedge-simulated-temperature-sensor -n microsoft -P SimulatedTemperatureSensor --target-arch aarch64 + + # Temperature Filter - Not Using Template for ARM64 because we have 2 different .NET Core. + - task: Bash@3 + displayName: Build Image Temperature Filter - aarch64 + inputs: + filePath: scripts/linux/buildImage.sh + arguments: -r $(registry.address) -u $(registry.user) -p $(registry.password) -i azureiotedge-temperature-filter -n microsoft -P TemperatureFilter --target-arch aarch64 + + # Load Gen - Not Using Template for ARM64 because we have 2 different .NET Core. + - task: Bash@3 + displayName: Build Image Load Gen - aarch64 + inputs: + filePath: scripts/linux/buildImage.sh + arguments: -r $(registry.address) -u $(registry.user) -p $(registry.password) -i azureiotedge-load-gen -n microsoft -P load-gen --target-arch aarch64 + + # Messages Analyzer - Not Using Template for ARM64 because we have 2 different .NET Core. + - task: Bash@3 + displayName: Build Image Messages Analyzer - aarch64 + inputs: + filePath: scripts/linux/buildImage.sh + arguments: -r $(registry.address) -u $(registry.user) -p $(registry.password) -i azureiotedge-analyzer -n microsoft -P MessagesAnalyzer --target-arch aarch64 + + # Functions Sample - Not Using Template for ARM64 because we have 2 different .NET Core.//TODO: Enable this once Functions supports arm64v8. Right now they are not ready. + #- task: Bash@3 + # displayName: Build Image Functions Sample - aarch64 + # inputs: + # filePath: scripts/linux/buildImage.sh + # arguments: -r $(registry.address) -u $(registry.user) -p $(registry.password) -i azureiotedge-functions-filter -n azureiotedge -P EdgeHubTriggerCSharp --target-arch aarch64 + + # Direct Method Sender - Not Using Template for ARM64 because we have 2 different .NET Core. + - task: Bash@3 + displayName: Build Image Direct Method Sender - aarch64 + inputs: + filePath: scripts/linux/buildImage.sh + arguments: -r $(registry.address) -u $(registry.user) -p $(registry.password) -i azureiotedge-direct-method-sender -n microsoft -P DirectMethodSender --target-arch aarch64 + + # Direct Method Receiver - Not Using Template for ARM64 because we have 2 different .NET Core. + - task: Bash@3 + displayName: Build Image Direct Method Receiver - aarch64 + inputs: + filePath: scripts/linux/buildImage.sh + arguments: -r $(registry.address) -u $(registry.user) -p $(registry.password) -i azureiotedge-direct-method-receiver -n microsoft -P DirectMethodReceiver --target-arch aarch64 + + # Direct Method Cloud Sender - Not Using Template for ARM64 because we have 2 different .NET Core. + - task: Bash@3 + displayName: Build Image Direct Method Cloud Sender - aarch64 + inputs: + filePath: scripts/linux/buildImage.sh + arguments: -r $(registry.address) -u $(registry.user) -p $(registry.password) -i azureiotedge-direct-method-cloud-sender -n microsoft -P DirectMethodCloudSender --target-arch aarch64 + + ################################################################################ - job: linux ################################################################################ @@ -245,6 +362,7 @@ jobs: dependsOn: - linux - windows + - linuxARM64 variables: tags: "['latest']" steps: diff --git a/builds/misc/templates/image-linux.yaml b/builds/misc/templates/image-linux.yaml index 503cbd6b44c..12850b7600f 100644 --- a/builds/misc/templates/image-linux.yaml +++ b/builds/misc/templates/image-linux.yaml @@ -14,4 +14,4 @@ steps: displayName: Build Image - ${{ parameters.name }} - arm32 inputs: filePath: scripts/linux/buildImage.sh - arguments: -r $(registry.address) -u $(registry.user) -p $(registry.password) -i ${{ parameters.imageName }} -n ${{ parameters.namespace }} -P ${{ parameters.project }} --target-arch armv7l + arguments: -r $(registry.address) -u $(registry.user) -p $(registry.password) -i ${{ parameters.imageName }} -n ${{ parameters.namespace }} -P ${{ parameters.project }} --target-arch armv7l \ No newline at end of file diff --git a/edge-agent/docker/manifest.yaml.template b/edge-agent/docker/manifest.yaml.template index 95693e50594..0e4c600e9c2 100644 --- a/edge-agent/docker/manifest.yaml.template +++ b/edge-agent/docker/manifest.yaml.template @@ -6,6 +6,11 @@ manifests: platform: architecture: amd64 os: linux + - + image: __REGISTRY__/__NAMESPACE__/azureiotedge-agent:__VERSION__-linux-arm64v8 + platform: + architecture: arm64 + os: linux - image: __REGISTRY__/__NAMESPACE__/azureiotedge-agent:__VERSION__-linux-arm32v7 platform: diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/Microsoft.Azure.Devices.Edge.Agent.Service.csproj b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/Microsoft.Azure.Devices.Edge.Agent.Service.csproj index 2b7a7db7305..6e4cc86d12b 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/Microsoft.Azure.Devices.Edge.Agent.Service.csproj +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/Microsoft.Azure.Devices.Edge.Agent.Service.csproj @@ -1,9 +1,9 @@  - + netcoreapp2.1 - + netcoreapp3.0 linux-arm64 diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/Microsoft.Azure.Devices.Edge.Agent.Core.Test.csproj b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/Microsoft.Azure.Devices.Edge.Agent.Core.Test.csproj index bafed6395ad..6310a2d138e 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/Microsoft.Azure.Devices.Edge.Agent.Core.Test.csproj +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/Microsoft.Azure.Devices.Edge.Agent.Core.Test.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.E2E.Test/Microsoft.Azure.Devices.Edge.Agent.Docker.E2E.Test.csproj b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.E2E.Test/Microsoft.Azure.Devices.Edge.Agent.Docker.E2E.Test.csproj index bb34fb5efe0..0a9f0abb7cd 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.E2E.Test/Microsoft.Azure.Devices.Edge.Agent.Docker.E2E.Test.csproj +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.E2E.Test/Microsoft.Azure.Devices.Edge.Agent.Docker.E2E.Test.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test.csproj b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test.csproj index 45669c6ac9f..0fec64a05c8 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test.csproj +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test/Microsoft.Azure.Devices.Edge.Agent.Docker.Test.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Docker.Test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Docker.Test.csproj b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Docker.Test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Docker.Test.csproj index b45ca8dae7c..86ea14cd674 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Docker.Test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Docker.Test.csproj +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Docker.Test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Docker.Test.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test.csproj b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test.csproj index e011be89c9a..14bb37c9741 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test.csproj +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test/Microsoft.Azure.Devices.Edge.Agent.Edgelet.Test.csproj @@ -1,11 +1,10 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 @@ -28,7 +27,6 @@ - @@ -40,6 +38,14 @@ + + + + + + + + diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test.csproj b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test.csproj index b286d8f7dea..b40d9982399 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test.csproj +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test.csproj @@ -1,11 +1,10 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-hub/docker/linux/arm64v8/base/Dockerfile b/edge-hub/docker/linux/arm64v8/base/Dockerfile index f2c7f4c7c61..dc76f113d47 100644 --- a/edge-hub/docker/linux/arm64v8/base/Dockerfile +++ b/edge-hub/docker/linux/arm64v8/base/Dockerfile @@ -8,6 +8,7 @@ ENV EdgeHubUser=edgehubuser RUN apt-get update && \ apt-get install -y libsnappy1v5 && \ + apt-get install -y libcap2-bin && \ rm -rf /var/lib/apt/lists/* && \ ln -s /lib/aarch64-linux-gnu/libdl.so.2 /usr/lib/aarch64-linux-gnu/libdl.so.2 && \ ln -s /lib/aarch64-linux-gnu/libdl.so.2 /lib/aarch64-linux-gnu/libdl.so && \ diff --git a/edge-hub/docker/manifest.yaml.template b/edge-hub/docker/manifest.yaml.template index f6bc9cbb826..6830ee9df30 100644 --- a/edge-hub/docker/manifest.yaml.template +++ b/edge-hub/docker/manifest.yaml.template @@ -11,6 +11,11 @@ manifests: platform: architecture: arm os: linux + - + image: __REGISTRY__/__NAMESPACE__/azureiotedge-hub:__VERSION__-linux-arm64v8 + platform: + architecture: arm64 + os: linux - image: __REGISTRY__/__NAMESPACE__/azureiotedge-hub:__VERSION__-windows-amd64 platform: diff --git a/edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Service/Microsoft.Azure.Devices.Edge.Hub.Service.csproj b/edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Service/Microsoft.Azure.Devices.Edge.Hub.Service.csproj index 283b6eee2a4..aef725eb3e8 100644 --- a/edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Service/Microsoft.Azure.Devices.Edge.Hub.Service.csproj +++ b/edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Service/Microsoft.Azure.Devices.Edge.Hub.Service.csproj @@ -1,9 +1,9 @@  - + netcoreapp2.1 - + netcoreapp3.0 linux-arm64 diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Amqp.Test/Microsoft.Azure.Devices.Edge.Hub.Amqp.Test.csproj b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Amqp.Test/Microsoft.Azure.Devices.Edge.Hub.Amqp.Test.csproj index 14e19d077c0..80e26d46d6c 100644 --- a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Amqp.Test/Microsoft.Azure.Devices.Edge.Hub.Amqp.Test.csproj +++ b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Amqp.Test/Microsoft.Azure.Devices.Edge.Hub.Amqp.Test.csproj @@ -1,12 +1,11 @@ - + - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test.csproj b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test.csproj index 4cab5b2f4de..558169180fe 100644 --- a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test.csproj +++ b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Core.Test/Microsoft.Azure.Devices.Edge.Hub.Core.Test.csproj b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Core.Test/Microsoft.Azure.Devices.Edge.Hub.Core.Test.csproj index c24698ed23a..59fbf32a529 100644 --- a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Core.Test/Microsoft.Azure.Devices.Edge.Hub.Core.Test.csproj +++ b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Core.Test/Microsoft.Azure.Devices.Edge.Hub.Core.Test.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test.csproj b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test.csproj index a315746b62d..94d42f55c87 100644 --- a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test.csproj +++ b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Http.Test/Microsoft.Azure.Devices.Edge.Hub.Http.Test.csproj b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Http.Test/Microsoft.Azure.Devices.Edge.Hub.Http.Test.csproj index ddbd1cad18c..f189bf1fc44 100644 --- a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Http.Test/Microsoft.Azure.Devices.Edge.Hub.Http.Test.csproj +++ b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Http.Test/Microsoft.Azure.Devices.Edge.Hub.Http.Test.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Mqtt.Test/Microsoft.Azure.Devices.Edge.Hub.Mqtt.Test.csproj b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Mqtt.Test/Microsoft.Azure.Devices.Edge.Hub.Mqtt.Test.csproj index b88f0e5c87a..f969094d725 100644 --- a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Mqtt.Test/Microsoft.Azure.Devices.Edge.Hub.Mqtt.Test.csproj +++ b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Mqtt.Test/Microsoft.Azure.Devices.Edge.Hub.Mqtt.Test.csproj @@ -1,11 +1,10 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Service.Test/Microsoft.Azure.Devices.Edge.Hub.Service.Test.csproj b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Service.Test/Microsoft.Azure.Devices.Edge.Hub.Service.Test.csproj index 50243cc4642..dac6c649cd4 100644 --- a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Service.Test/Microsoft.Azure.Devices.Edge.Hub.Service.Test.csproj +++ b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Service.Test/Microsoft.Azure.Devices.Edge.Hub.Service.Test.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-hub/test/Microsoft.Azure.Devices.Routing.Core.Test/Microsoft.Azure.Devices.Routing.Core.Test.csproj b/edge-hub/test/Microsoft.Azure.Devices.Routing.Core.Test/Microsoft.Azure.Devices.Routing.Core.Test.csproj index f3c6bafdcee..7cc368b4458 100644 --- a/edge-hub/test/Microsoft.Azure.Devices.Routing.Core.Test/Microsoft.Azure.Devices.Routing.Core.Test.csproj +++ b/edge-hub/test/Microsoft.Azure.Devices.Routing.Core.Test/Microsoft.Azure.Devices.Routing.Core.Test.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-modules/DirectMethodCloudSender/DirectMethodCloudSender.csproj b/edge-modules/DirectMethodCloudSender/DirectMethodCloudSender.csproj index 3a64fbc5b3c..3ed67791baa 100644 --- a/edge-modules/DirectMethodCloudSender/DirectMethodCloudSender.csproj +++ b/edge-modules/DirectMethodCloudSender/DirectMethodCloudSender.csproj @@ -1,11 +1,10 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-modules/DirectMethodReceiver/DirectMethodReceiver.csproj b/edge-modules/DirectMethodReceiver/DirectMethodReceiver.csproj index cf57348733c..2b123b6c721 100644 --- a/edge-modules/DirectMethodReceiver/DirectMethodReceiver.csproj +++ b/edge-modules/DirectMethodReceiver/DirectMethodReceiver.csproj @@ -1,11 +1,10 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-modules/DirectMethodReceiver/docker/linux/arm64v8/Dockerfile b/edge-modules/DirectMethodReceiver/docker/linux/arm64v8/Dockerfile index 7614410dcd2..8f7b62fd800 100644 --- a/edge-modules/DirectMethodReceiver/docker/linux/arm64v8/Dockerfile +++ b/edge-modules/DirectMethodReceiver/docker/linux/arm64v8/Dockerfile @@ -1,4 +1,4 @@ -ARG base_tag=1.0.0-preview009-linux-arm64v8 +ARG base_tag=1.0.0-preview010-linux-arm64v8 FROM azureiotedge/azureiotedge-module-base:${base_tag} ARG EXE_DIR=. diff --git a/edge-modules/DirectMethodSender/DirectMethodSender.csproj b/edge-modules/DirectMethodSender/DirectMethodSender.csproj index 36497388341..3d414cfbd3e 100644 --- a/edge-modules/DirectMethodSender/DirectMethodSender.csproj +++ b/edge-modules/DirectMethodSender/DirectMethodSender.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-modules/MessagesAnalyzer/MessagesAnalyzer.csproj b/edge-modules/MessagesAnalyzer/MessagesAnalyzer.csproj index 2badc59e664..f7068973664 100644 --- a/edge-modules/MessagesAnalyzer/MessagesAnalyzer.csproj +++ b/edge-modules/MessagesAnalyzer/MessagesAnalyzer.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-modules/ModuleLib/Microsoft.Azure.Devices.Edge.ModuleUtil.csproj b/edge-modules/ModuleLib/Microsoft.Azure.Devices.Edge.ModuleUtil.csproj index 40e036f1c83..3317b9163c1 100644 --- a/edge-modules/ModuleLib/Microsoft.Azure.Devices.Edge.ModuleUtil.csproj +++ b/edge-modules/ModuleLib/Microsoft.Azure.Devices.Edge.ModuleUtil.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-modules/SimulatedTemperatureSensor/SimulatedTemperatureSensor.csproj b/edge-modules/SimulatedTemperatureSensor/SimulatedTemperatureSensor.csproj index 36497388341..3d414cfbd3e 100644 --- a/edge-modules/SimulatedTemperatureSensor/SimulatedTemperatureSensor.csproj +++ b/edge-modules/SimulatedTemperatureSensor/SimulatedTemperatureSensor.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-modules/SimulatedTemperatureSensor/docker/manifest.yaml.template b/edge-modules/SimulatedTemperatureSensor/docker/manifest.yaml.template index f445f51b790..312210a0a41 100644 --- a/edge-modules/SimulatedTemperatureSensor/docker/manifest.yaml.template +++ b/edge-modules/SimulatedTemperatureSensor/docker/manifest.yaml.template @@ -11,6 +11,11 @@ manifests: platform: architecture: arm os: linux + - + image: __REGISTRY__/__NAMESPACE__/azureiotedge-simulated-temperature-sensor:__VERSION__-linux-arm64v8 + platform: + architecture: arm64 + os: linux - image: __REGISTRY__/__NAMESPACE__/azureiotedge-simulated-temperature-sensor:__VERSION__-windows-amd64 platform: diff --git a/edge-modules/TemperatureFilter/TemperatureFilter.csproj b/edge-modules/TemperatureFilter/TemperatureFilter.csproj index 1086bdfb50d..895d028a888 100644 --- a/edge-modules/TemperatureFilter/TemperatureFilter.csproj +++ b/edge-modules/TemperatureFilter/TemperatureFilter.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-modules/functions/binding/src/Microsoft.Azure.WebJobs.Extensions.EdgeHub/Microsoft.Azure.WebJobs.Extensions.EdgeHub.csproj b/edge-modules/functions/binding/src/Microsoft.Azure.WebJobs.Extensions.EdgeHub/Microsoft.Azure.WebJobs.Extensions.EdgeHub.csproj index 1af9cef8f29..af70e74e1a8 100644 --- a/edge-modules/functions/binding/src/Microsoft.Azure.WebJobs.Extensions.EdgeHub/Microsoft.Azure.WebJobs.Extensions.EdgeHub.csproj +++ b/edge-modules/functions/binding/src/Microsoft.Azure.WebJobs.Extensions.EdgeHub/Microsoft.Azure.WebJobs.Extensions.EdgeHub.csproj @@ -1,5 +1,7 @@  - + + true + netstandard2.0 Microsoft.Azure.WebJobs.Extensions.EdgeHub diff --git a/edge-modules/functions/samples/EdgeHubTrigger-Csharp/EdgeHubTriggerCSharp.csproj b/edge-modules/functions/samples/EdgeHubTrigger-Csharp/EdgeHubTriggerCSharp.csproj index 60f14ce9d94..f92e2097ee4 100644 --- a/edge-modules/functions/samples/EdgeHubTrigger-Csharp/EdgeHubTriggerCSharp.csproj +++ b/edge-modules/functions/samples/EdgeHubTrigger-Csharp/EdgeHubTriggerCSharp.csproj @@ -15,9 +15,13 @@ %(RecursiveDir)%(Filename)%(Extension) + + + + + - diff --git a/edge-modules/load-gen/load-gen.csproj b/edge-modules/load-gen/load-gen.csproj index 5f7882ab48e..9c402c9334d 100644 --- a/edge-modules/load-gen/load-gen.csproj +++ b/edge-modules/load-gen/load-gen.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.RocksDb.Test/Microsoft.Azure.Devices.Edge.Storage.RocksDb.Test.csproj b/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.RocksDb.Test/Microsoft.Azure.Devices.Edge.Storage.RocksDb.Test.csproj index 3f121cc597c..1b319d85978 100644 --- a/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.RocksDb.Test/Microsoft.Azure.Devices.Edge.Storage.RocksDb.Test.csproj +++ b/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.RocksDb.Test/Microsoft.Azure.Devices.Edge.Storage.RocksDb.Test.csproj @@ -1,12 +1,11 @@ - + - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.Test/Microsoft.Azure.Devices.Edge.Storage.Test.csproj b/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.Test/Microsoft.Azure.Devices.Edge.Storage.Test.csproj index e659192fd2e..c65238979fc 100644 --- a/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.Test/Microsoft.Azure.Devices.Edge.Storage.Test.csproj +++ b/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.Test/Microsoft.Azure.Devices.Edge.Storage.Test.csproj @@ -1,12 +1,11 @@ - + - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 diff --git a/edge-util/test/Microsoft.Azure.Devices.Edge.Util.Test/Microsoft.Azure.Devices.Edge.Util.Test.csproj b/edge-util/test/Microsoft.Azure.Devices.Edge.Util.Test/Microsoft.Azure.Devices.Edge.Util.Test.csproj index c76cea0981b..f065fb76833 100644 --- a/edge-util/test/Microsoft.Azure.Devices.Edge.Util.Test/Microsoft.Azure.Devices.Edge.Util.Test.csproj +++ b/edge-util/test/Microsoft.Azure.Devices.Edge.Util.Test/Microsoft.Azure.Devices.Edge.Util.Test.csproj @@ -1,12 +1,11 @@  - + netcoreapp2.1 - + netcoreapp3.0 - linux-arm64 @@ -29,7 +28,6 @@ - @@ -41,6 +39,14 @@ + + + + + + + + diff --git a/edgelet/Cross.toml b/edgelet/Cross.toml index 045c5ee96e7..746235ca0f0 100644 --- a/edgelet/Cross.toml +++ b/edgelet/Cross.toml @@ -12,3 +12,8 @@ image = "azureiotedge/gcc-linaro-7.2.1-2017.11-x86_64_arm-linux-gnueabihf:0.2" # Built from rust-embedded/cross#718a19cd68fb09428532d1317515fe7303692b47 with `./build-docker-image.sh armv7-unknown-linux-musleabihf` # because the image in Docker hub is outdated and broken image = "azureiotedge/armv7-unknown-linux-musleabihf:0.1" + +[target.aarch64-unknown-linux-musl] +# Built from rust-embedded/cross#718a19cd68fb09428532d1317515fe7303692b47 with `./build-docker-image.sh aarch64-unknown-linux-musl` +# because the image in Docker hub is outdated and broken +image = "azureiotedge/aarch64-unknown-linux-musl:0.1" \ No newline at end of file diff --git a/edgelet/iotedge-diagnostics/docker/manifest.yaml.template b/edgelet/iotedge-diagnostics/docker/manifest.yaml.template index 07132a47bb5..992de0d6e95 100644 --- a/edgelet/iotedge-diagnostics/docker/manifest.yaml.template +++ b/edgelet/iotedge-diagnostics/docker/manifest.yaml.template @@ -11,6 +11,11 @@ manifests: platform: architecture: arm os: linux + - + image: __REGISTRY__/__NAMESPACE__/azureiotedge-diagnostics:__VERSION__-linux-arm64v8 + platform: + architecture: arm64 + os: linux - image: __REGISTRY__/__NAMESPACE__/azureiotedge-diagnostics:__VERSION__-windows-amd64 platform: diff --git a/scripts/linux/buildBranch.sh b/scripts/linux/buildBranch.sh index d217e408fdd..8d5d12eb618 100755 --- a/scripts/linux/buildBranch.sh +++ b/scripts/linux/buildBranch.sh @@ -25,7 +25,7 @@ SRC_E2E_TEST_FILES_DIR=$ROOT_FOLDER/e2e_test_files SRC_CERT_TOOLS_DIR=$ROOT_FOLDER/tools/CACertificates FUNCTIONS_SAMPLE_DIR=$ROOT_FOLDER/edge-modules/functions/samples VERSIONINFO_FILE_PATH=$BUILD_REPOSITORY_LOCALPATH/versionInfo.json - +DOTNETBUILD_OS= DOTNET_RUNTIME=netcoreapp2.1 usage() @@ -35,6 +35,8 @@ usage() echo "options" echo " -c, --config Product binary configuration: Debug [default] or Release" echo " --no-rocksdb-bin Do not copy the RocksDB binaries into the project's output folders" + echo " --os Sets OS Variable for dotnet build command (Used to build for .NET Core 3.0 - Linux ARM64)" + echo " --dotnet_runtime Set the dotnet_runtime version to build. (Default netcoreapp2.1)" exit 1; } @@ -47,16 +49,24 @@ print_help_and_exit() process_args() { local save_next_arg=0 - for arg in "$@" + for arg in "$@"; do if [ $save_next_arg -eq 1 ]; then CONFIGURATION="$arg" save_next_arg=0 + elif [ $save_next_arg -eq 2 ]; then + DOTNETBUILD_OS="$arg" + save_next_arg=0 + elif [ $save_next_arg -eq 3 ]; then + DOTNET_RUNTIME="$arg" + save_next_arg=0 else case "$arg" in "-h" | "--help" ) usage;; "-c" | "--config" ) save_next_arg=1;; "--no-rocksdb-bin" ) MSBUILD_OPTIONS="-p:RocksDbAsPackage=false";; + "--os" ) save_next_arg=2;; + "--dotnet_runtime" ) save_next_arg=3;; * ) usage;; esac fi @@ -130,7 +140,7 @@ publish_project() fi echo "Publishing $type '$name'" - $DOTNET_ROOT_PATH/dotnet publish -f $framework -c $config $option -o $output $path + $DOTNET_ROOT_PATH/dotnet publish -f $framework -p:DotNet_Runtime=$DOTNET_RUNTIME -c $config $option -o $output $path if [ $? -gt 0 ]; then RES=1 fi @@ -156,6 +166,7 @@ publish_quickstart() $DOTNET_ROOT_PATH/dotnet publish \ -c $CONFIGURATION \ -f $DOTNET_RUNTIME \ + -p:DotNet_Runtime=$DOTNET_RUNTIME \ -r $rid \ $ROOT_FOLDER/smoke/IotEdgeQuickstart if [ $? -gt 0 ]; then @@ -175,6 +186,7 @@ publish_leafdevice() $DOTNET_ROOT_PATH/dotnet publish \ -c $CONFIGURATION \ -f $DOTNET_RUNTIME \ + -p:DotNet_Runtime=$DOTNET_RUNTIME \ -r $rid \ $ROOT_FOLDER/smoke/LeafDevice if [ $? -gt 0 ]; then @@ -190,10 +202,20 @@ publish_leafdevice() build_solution() { echo "Building IoT Edge solution" - $DOTNET_ROOT_PATH/dotnet build \ - -c $CONFIGURATION \ - -o "$BUILD_BINARIESDIRECTORY" \ - "$ROOT_FOLDER/Microsoft.Azure.Devices.Edge.sln" + dotnet --version + + build_command="$DOTNET_ROOT_PATH/dotnet build -c $CONFIGURATION" + + if [ -n "$DOTNETBUILD_OS" ]; then + build_command="$build_command -p:OS=$DOTNETBUILD_OS" + fi + + if [ -n "$DOTNET_RUNTIME" ]; then + build_command="$build_command -p:DotNet_Runtime=$DOTNET_RUNTIME" + fi + build_command="$build_command $ROOT_FOLDER/Microsoft.Azure.Devices.Edge.sln" + + eval ${build_command} if [ $? -gt 0 ]; then RES=1 fi @@ -239,7 +261,9 @@ publish_files $SRC_CERT_TOOLS_DIR $PUBLISH_FOLDER publish_quickstart linux-arm publish_quickstart linux-x64 +publish_quickstart linux-arm64 publish_leafdevice linux-arm publish_leafdevice linux-x64 +publish_leafdevice linux-arm64 -exit $RES +exit $RES \ No newline at end of file diff --git a/scripts/linux/buildDiagnostics.sh b/scripts/linux/buildDiagnostics.sh index d71ae494621..1d983151fb2 100755 --- a/scripts/linux/buildDiagnostics.sh +++ b/scripts/linux/buildDiagnostics.sh @@ -1,7 +1,7 @@ #!/bin/bash # This script builds the iotedge-diagnostics binary that goes into the azureiotedge-diagnostics image, -# for each supported arch (x86_64, arm32v7). +# for each supported arch (x86_64, arm32v7, arm64v8). # It then publishes the binaries along with their corresponding dockerfiles to the publish directory, # so that buildImage.sh can build the container image. @@ -21,3 +21,6 @@ cp $BUILD_REPOSITORY_LOCALPATH/edgelet/target/x86_64-unknown-linux-musl/release/ cross build -p iotedge-diagnostics --release --target armv7-unknown-linux-musleabihf cp $BUILD_REPOSITORY_LOCALPATH/edgelet/target/armv7-unknown-linux-musleabihf/release/iotedge-diagnostics $PUBLISH_FOLDER/azureiotedge-diagnostics/docker/linux/arm32v7/ + +cross build -p iotedge-diagnostics --release --target aarch64-unknown-linux-musl +cp $BUILD_REPOSITORY_LOCALPATH/edgelet/target/aarch64-unknown-linux-musl/release/iotedge-diagnostics $PUBLISH_FOLDER/azureiotedge-diagnostics/docker/linux/arm64v8/ diff --git a/scripts/linux/docker/manifest.yaml.template b/scripts/linux/docker/manifest.yaml.template index 8d1878620f7..74721965011 100644 --- a/scripts/linux/docker/manifest.yaml.template +++ b/scripts/linux/docker/manifest.yaml.template @@ -11,6 +11,11 @@ manifests: platform: architecture: arm os: linux + - + image: __REGISTRY__/__NAMESPACE__/__NAME__:__VERSION__-linux-arm64v8 + platform: + architecture: aarch64 + os: linux - image: __REGISTRY__/__NAMESPACE__/__NAME__:__VERSION__-windows-amd64 platform: diff --git a/smoke/IotEdgeQuickstart/IotEdgeQuickstart.csproj b/smoke/IotEdgeQuickstart/IotEdgeQuickstart.csproj index d26d2a1b477..68464af4380 100644 --- a/smoke/IotEdgeQuickstart/IotEdgeQuickstart.csproj +++ b/smoke/IotEdgeQuickstart/IotEdgeQuickstart.csproj @@ -1,10 +1,10 @@  - + netcoreapp2.1 - + netcoreapp3.0 linux-arm64 diff --git a/smoke/LeafDevice/LeafDevice.csproj b/smoke/LeafDevice/LeafDevice.csproj index e7fa0d20559..d458ce60091 100644 --- a/smoke/LeafDevice/LeafDevice.csproj +++ b/smoke/LeafDevice/LeafDevice.csproj @@ -1,10 +1,10 @@  - + netcoreapp2.1 - + netcoreapp3.0 linux-arm64 From 8d32a995320ef8e16e71f662d81cce781a2e31e4 Mon Sep 17 00:00:00 2001 From: Varun Puranik Date: Mon, 11 Mar 2019 15:33:48 -0700 Subject: [PATCH 4/4] Edgelet: Add support for "since" parameter when getting module logs (#932) * Add support for since in logs * Add iotedge parameter * Connect logs since option * Add logs since option to Management 2019-1-30 * Add and fix tests * Remove println * format changes * Fix clippy --- edgelet/api/managementVersion_2019_01_30.yaml | 5 ++++ edgelet/edgelet-core/src/module.rs | 11 ++++++++ edgelet/edgelet-docker/src/runtime.rs | 10 +++++++- edgelet/edgelet-docker/tests/runtime.rs | 6 ++++- .../edgelet-http-mgmt/src/client/module.rs | 8 +++++- .../src/server/module/logs.rs | 25 +++++++++++++++++-- edgelet/iotedge/src/main.rs | 17 ++++++++++++- edgelet/management/src/apis/module_api.rs | 3 +++ 8 files changed, 79 insertions(+), 6 deletions(-) diff --git a/edgelet/api/managementVersion_2019_01_30.yaml b/edgelet/api/managementVersion_2019_01_30.yaml index d2d81f5b5c8..fecb218dc07 100644 --- a/edgelet/api/managementVersion_2019_01_30.yaml +++ b/edgelet/api/managementVersion_2019_01_30.yaml @@ -296,6 +296,11 @@ paths: description: Only return this number of lines from the end of the logs. type: string default: "all" + - in: query + name: since + description: Only return logs since this time, as a UNIX timestamp. + type: integer + default: 0 responses: '101': description: Logs returned as a stream diff --git a/edgelet/edgelet-core/src/module.rs b/edgelet/edgelet-core/src/module.rs index ff8109c8df7..1ce8ff1c687 100644 --- a/edgelet/edgelet-core/src/module.rs +++ b/edgelet/edgelet-core/src/module.rs @@ -267,6 +267,7 @@ impl ToString for LogTail { pub struct LogOptions { follow: bool, tail: LogTail, + since: i32, } impl LogOptions { @@ -274,6 +275,7 @@ impl LogOptions { LogOptions { follow: false, tail: LogTail::All, + since: 0, } } @@ -287,6 +289,11 @@ impl LogOptions { self } + pub fn with_since(mut self, since: i32) -> Self { + self.since = since; + self + } + pub fn follow(&self) -> bool { self.follow } @@ -294,6 +301,10 @@ impl LogOptions { pub fn tail(&self) -> &LogTail { &self.tail } + + pub fn since(&self) -> i32 { + self.since + } } pub trait Module { diff --git a/edgelet/edgelet-docker/src/runtime.rs b/edgelet/edgelet-docker/src/runtime.rs index a084def6a3f..8f771813f63 100644 --- a/edgelet/edgelet-docker/src/runtime.rs +++ b/edgelet/edgelet-docker/src/runtime.rs @@ -678,7 +678,15 @@ impl ModuleRuntime for DockerModuleRuntime { let result = self .client .container_api() - .container_logs(&id, options.follow(), true, true, 0, false, tail) + .container_logs( + &id, + options.follow(), + true, + true, + options.since(), + false, + tail, + ) .then(|result| match result { Ok(logs) => { info!("Successfully got logs for module {}", id); diff --git a/edgelet/edgelet-docker/tests/runtime.rs b/edgelet/edgelet-docker/tests/runtime.rs index 43edc16881d..8c54f0881c2 100644 --- a/edgelet/edgelet-docker/tests/runtime.rs +++ b/edgelet/edgelet-docker/tests/runtime.rs @@ -935,6 +935,7 @@ fn container_logs_handler( assert!(query_map.contains_key("tail")); assert_eq!("true", query_map["follow"]); assert_eq!("all", query_map["tail"]); + assert_eq!("100000", query_map["since"]); let body = vec![ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x52, 0x6f, 0x73, 0x65, 0x73, 0x20, 0x61, @@ -955,7 +956,10 @@ fn container_logs_succeeds() { DockerModuleRuntime::new(&Url::parse(&format!("http://localhost:{}/", port)).unwrap()) .unwrap(); - let options = LogOptions::new().with_follow(true).with_tail(LogTail::All); + let options = LogOptions::new() + .with_follow(true) + .with_tail(LogTail::All) + .with_since(100_000); let task = mri.logs("mod1", &options); let expected_body = [ diff --git a/edgelet/edgelet-http-mgmt/src/client/module.rs b/edgelet/edgelet-http-mgmt/src/client/module.rs index 87c93e09bbf..da0e0badd09 100644 --- a/edgelet/edgelet-http-mgmt/src/client/module.rs +++ b/edgelet/edgelet-http-mgmt/src/client/module.rs @@ -313,7 +313,13 @@ impl ModuleRuntime for ModuleClient { let result = self .client .module_api() - .module_logs(&API_VERSION.to_string(), &id, options.follow(), tail) + .module_logs( + &API_VERSION.to_string(), + &id, + options.follow(), + tail, + options.since(), + ) .then(|logs| match logs { Ok(logs) => Ok(Logs(id, logs)), Err(err) => Err(Error::from_mgmt_error( diff --git a/edgelet/edgelet-http-mgmt/src/server/module/logs.rs b/edgelet/edgelet-http-mgmt/src/server/module/logs.rs index 3299604fb7a..ad08b77b17b 100644 --- a/edgelet/edgelet-http-mgmt/src/server/module/logs.rs +++ b/edgelet/edgelet-http-mgmt/src/server/module/logs.rs @@ -79,7 +79,15 @@ fn parse_options(query: &str) -> Result { .find(|&(ref key, _)| key == "follow") .map_or_else(|| Ok(false), |(_, val)| val.parse::()) .context(ErrorKind::MalformedRequestParameter("follow"))?; - let options = LogOptions::new().with_follow(follow).with_tail(tail); + let since = parse + .iter() + .find(|&(ref key, _)| key == "since") + .map_or_else(|| Ok(0), |(_, val)| val.parse::()) + .context(ErrorKind::MalformedRequestParameter("since"))?; + let options = LogOptions::new() + .with_follow(follow) + .with_tail(tail) + .with_since(since); Ok(options) } @@ -97,10 +105,11 @@ mod tests { #[test] fn correct_logoptions() { - let query = "follow=true&tail=6"; + let query = "follow=true&tail=6&since=1551885923"; let options = parse_options(&query).unwrap(); assert_eq!(LogTail::Num(6), *options.tail()); assert_eq!(true, options.follow()); + assert_eq!(1_551_885_923, options.since()); } #[test] @@ -109,6 +118,7 @@ mod tests { let options = parse_options(&query).unwrap(); assert_eq!(LogTail::default(), *options.tail()); assert_eq!(false, options.follow()); + assert_eq!(0, options.since()); } #[test] @@ -133,6 +143,17 @@ mod tests { ); } + #[test] + fn logoption_since_error() { + let query = "follow=true&tail=6&since=15abc"; + let options = parse_options(&query); + assert!(options.is_err()); + assert_eq!( + "The request parameter `since` is malformed", + options.err().unwrap().to_string() + ); + } + #[test] fn test_success() { let state = ModuleRuntimeState::default() diff --git a/edgelet/iotedge/src/main.rs b/edgelet/iotedge/src/main.rs index 10af3cccb0e..db0cdab85b7 100644 --- a/edgelet/iotedge/src/main.rs +++ b/edgelet/iotedge/src/main.rs @@ -123,6 +123,14 @@ fn run() -> Result<(), Error> { .value_name("NUM") .default_value("all"), ) + .arg( + Arg::with_name("since") + .help("Only return logs since this time, as a UNIX timestamp") + .long("since") + .takes_value(true) + .value_name("NUM") + .default_value("0"), + ) .arg( Arg::with_name("follow") .help("Follow output log") @@ -180,7 +188,14 @@ fn run() -> Result<(), Error> { .value_of("tail") .and_then(|a| a.parse::().ok()) .unwrap_or_default(); - let options = LogOptions::new().with_follow(follow).with_tail(tail); + let since = args + .value_of("since") + .and_then(|a| a.parse::().ok()) + .unwrap_or_default(); + let options = LogOptions::new() + .with_follow(follow) + .with_tail(tail) + .with_since(since); tokio_runtime.block_on(Logs::new(id, options, runtime).execute()) } ("version", Some(_args)) => tokio_runtime.block_on(Version::new().execute()), diff --git a/edgelet/management/src/apis/module_api.rs b/edgelet/management/src/apis/module_api.rs index 260d3febad9..04dc2a13b3e 100644 --- a/edgelet/management/src/apis/module_api.rs +++ b/edgelet/management/src/apis/module_api.rs @@ -55,6 +55,7 @@ pub trait ModuleApi: Send + Sync { name: &str, follow: bool, tail: &str, + since: i32, ) -> Box> + Send>; fn restart_module( &self, @@ -316,6 +317,7 @@ where name: &str, follow: bool, tail: &str, + since: i32, ) -> Box> + Send> { let configuration: &configuration::Configuration = self.configuration.borrow(); @@ -325,6 +327,7 @@ where .append_pair("api-version", &api_version.to_string()) .append_pair("follow", &follow.to_string()) .append_pair("tail", &tail.to_string()) + .append_pair("since", &since.to_string()) .finish(); let uri_str = format!("/modules/{name}/logs?{}", query, name = name);