From c6d51246bb849186f0fe5ce16ad23ab4e5b03944 Mon Sep 17 00:00:00 2001 From: Justin Baur <19896123+justindbaur@users.noreply.github.com> Date: Wed, 26 Feb 2025 16:23:35 -0500 Subject: [PATCH 1/4] Move Scaffolding into C# --- src/Api/Startup.cs | 11 ++ .../Infrastructure/AzureScaffolder.cs | 142 ++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 src/Core/Platform/Infrastructure/AzureScaffolder.cs diff --git a/src/Api/Startup.cs b/src/Api/Startup.cs index a34125725932..b4d657e46f1f 100644 --- a/src/Api/Startup.cs +++ b/src/Api/Startup.cs @@ -31,6 +31,8 @@ using Bit.Core.Auth.Identity.TokenProviders; using Bit.Core.Tools.ImportFeatures; using Bit.Core.Tools.ReportFeatures; +using Bit.Core.Platform.Infrastructure; + #if !OSS @@ -66,6 +68,15 @@ public void ConfigureServices(IServiceCollection services) services.Configure(Configuration.GetSection("IpRateLimitPolicies")); } + // TODO: Be more selective about adding this scaffolder + if (Environment.IsDevelopment()) + { + // If this scaffolder is going to be registered, we want it registered + // pretty early in case any other hosted services need to use the + // things this hosted service will create. + services.AddHostedService(); + } + // Data Protection services.AddCustomDataProtectionServices(Environment, globalSettings); diff --git a/src/Core/Platform/Infrastructure/AzureScaffolder.cs b/src/Core/Platform/Infrastructure/AzureScaffolder.cs new file mode 100644 index 000000000000..ece8480c5005 --- /dev/null +++ b/src/Core/Platform/Infrastructure/AzureScaffolder.cs @@ -0,0 +1,142 @@ +#nullable enable + +using Azure; +using Azure.Data.Tables; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Queues; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Bit.Core.Platform.Infrastructure; + +internal class BrokenDevelepmentEnvironmentException : Exception +{ + public BrokenDevelepmentEnvironmentException(string message) + : base (message) + { + + } + + public BrokenDevelepmentEnvironmentException(string message, Exception innerException) + : base(message, innerException) + { + + } +} + +public class AzureScaffolder : IHostedService +{ + private static readonly IEnumerable _containers = + [ "attachments", "sendfiles", "misc" ]; + + private static readonly IEnumerable _queues = + [ "event", "notifications", "reference-events", "mail" ]; + + private static readonly IEnumerable _tables = + [ "event", "metadata", "installationdevice"]; + + private static readonly BlobCorsRule _corsRule = new() + { + AllowedHeaders = "*", + ExposedHeaders = "*", + AllowedOrigins = "*", + MaxAgeInSeconds = 30, + AllowedMethods = "GET,PUT" + }; + + private static readonly EqualityComparer _corsComparer = EqualityComparer.Create((a, b) => + { + if (a == null) + { + return b == null; + } + + if (b == null) + { + return false; + } + + return a.AllowedOrigins == b.AllowedOrigins + && a.AllowedMethods == b.AllowedMethods + && a.AllowedHeaders == b.AllowedHeaders + && a.ExposedHeaders == b.ExposedHeaders + && a.MaxAgeInSeconds == b.MaxAgeInSeconds; + }); + + private static readonly string _developmentUri = "UseDevelopmentStorage=true"; + private readonly ILogger _logger; + + public AzureScaffolder(ILogger logger) + { + _logger = logger; + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + try + { + _logger.LogInformation("Scaffolding Azurite Infrastrucure."); + await ScaffoldAsync(cancellationToken); + } + // TODO: Handle certain errors with instructions on how to fix, like API version problems + catch (RequestFailedException requestedFailedEx) when (requestedFailedEx.ErrorCode == "InvalidHeaderValue") + { + // Rethrow with more explicit exception + throw new BrokenDevelepmentEnvironmentException( + "The version of Azurite being ran is incompitable with our Azure packages. Read more https://contributing.bitwarden.com/link-here", + requestedFailedEx + ); + } + catch (Exception ex) + { + _logger.LogError(ex, "Unknown error while scaffolding Azure infrastructure in Azurite."); + throw; + } + } + + private static async Task ScaffoldAsync(CancellationToken cancellationToken) + { + var blobServiceClient = new BlobServiceClient(_developmentUri); + + foreach (var container in _containers) + { + var blobContainer = blobServiceClient.GetBlobContainerClient(container); + if (!await blobContainer.ExistsAsync(cancellationToken)) + { + await blobContainer.CreateAsync(cancellationToken: cancellationToken); + } + } + + + // Check if our cors rule is already added, if not, add it. + var properties = await blobServiceClient.GetPropertiesAsync(cancellationToken); + var existingCorsRules = properties.Value.Cors; + if (!existingCorsRules.Contains(_corsRule, _corsComparer)) + { + properties.Value.Cors.Add(_corsRule); + await blobServiceClient.SetPropertiesAsync(properties.Value, cancellationToken); + } + + var queueServiceClient = new QueueServiceClient(_developmentUri); + + foreach (var queue in _queues) + { + var queueClient = queueServiceClient.GetQueueClient(queue); + if (!await queueClient.ExistsAsync(cancellationToken)) + { + await queueClient.CreateAsync(cancellationToken: cancellationToken); + } + } + + var tableServiceClient = new TableServiceClient(_developmentUri); + + foreach (var table in _tables) + { + var tableClient = tableServiceClient.GetTableClient(table); + await tableClient.CreateIfNotExistsAsync(cancellationToken: cancellationToken); + } + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; +} From 647aa516da6a4eafaa78050b825a92b93f4eedbf Mon Sep 17 00:00:00 2001 From: Justin Baur <19896123+justindbaur@users.noreply.github.com> Date: Wed, 26 Feb 2025 16:26:05 -0500 Subject: [PATCH 2/4] Delete existing setup relating to Azurite script --- .../internal_dev/postCreateCommand.sh | 3 -- dev/setup_azurite.ps1 | 47 ------------------- 2 files changed, 50 deletions(-) delete mode 100755 dev/setup_azurite.ps1 diff --git a/.devcontainer/internal_dev/postCreateCommand.sh b/.devcontainer/internal_dev/postCreateCommand.sh index 668b776447c9..eac03e643fe4 100755 --- a/.devcontainer/internal_dev/postCreateCommand.sh +++ b/.devcontainer/internal_dev/postCreateCommand.sh @@ -52,9 +52,6 @@ Proceed? [y/N] " response Press to continue." remove_comments ./dev/secrets.json configure_other_vars - echo "Installing Az module. This will take ~a minute..." - pwsh -Command "Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -Force" - pwsh ./dev/setup_azurite.ps1 dotnet tool install dotnet-certificate-tool -g >/dev/null diff --git a/dev/setup_azurite.ps1 b/dev/setup_azurite.ps1 deleted file mode 100755 index ad9808f6c335..000000000000 --- a/dev/setup_azurite.ps1 +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env pwsh -# Script for configuring the initial state of Azurite Storage account -# Can be run multiple times without negative impact - -# Start configuration -$corsRules = (@{ - AllowedHeaders = @("*"); - ExposedHeaders = @("*"); - AllowedOrigins = @("*"); - MaxAgeInSeconds = 30; - AllowedMethods = @("Get", "PUT"); - }); -$containers = "attachments", "sendfiles", "misc"; -$queues = "event", "notifications", "reference-events", "mail"; -$tables = "event", "metadata", "installationdevice"; -# End configuration - -$context = New-AzStorageContext -Local - -foreach ($container in $containers) { - if (Get-AzStorageContainer -Name $container -Context $context -ErrorAction SilentlyContinue) { - Write-Host -ForegroundColor Magenta "Container already exists:" $container - } - else { - New-AzStorageContainer -Name $container -Context $context - } -} - -foreach ($queue in $queues) { - if (Get-AzStorageQueue -Name $queue -Context $context -ErrorAction SilentlyContinue) { - Write-Host -ForegroundColor Magenta "Queue already exists:" $queue - } - else { - New-AzStorageQueue -Name $queue -Context $context - } -} - -foreach ($table in $tables) { - if (Get-AzStorageTable -Name $table -Context $context -ErrorAction SilentlyContinue) { - Write-Host -ForegroundColor Magenta "Table already exists:" $table - } - else { - New-AzStorageTable -Name $table -Context $context - } -} - -Set-AzStorageCORSRule -ServiceType Blob -CorsRules $corsRules -Context $context From 5dff881ef4133a6159dfb1296907972787872eea Mon Sep 17 00:00:00 2001 From: Justin Baur <19896123+justindbaur@users.noreply.github.com> Date: Wed, 26 Feb 2025 16:51:29 -0500 Subject: [PATCH 3/4] Formatting --- .../Infrastructure/AzureScaffolder.cs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Core/Platform/Infrastructure/AzureScaffolder.cs b/src/Core/Platform/Infrastructure/AzureScaffolder.cs index ece8480c5005..4e3b0cbbdf7f 100644 --- a/src/Core/Platform/Infrastructure/AzureScaffolder.cs +++ b/src/Core/Platform/Infrastructure/AzureScaffolder.cs @@ -1,4 +1,4 @@ -#nullable enable +#nullable enable using Azure; using Azure.Data.Tables; @@ -13,9 +13,9 @@ namespace Bit.Core.Platform.Infrastructure; internal class BrokenDevelepmentEnvironmentException : Exception { public BrokenDevelepmentEnvironmentException(string message) - : base (message) + : base(message) { - + } public BrokenDevelepmentEnvironmentException(string message, Exception innerException) @@ -28,13 +28,13 @@ public BrokenDevelepmentEnvironmentException(string message, Exception innerExce public class AzureScaffolder : IHostedService { private static readonly IEnumerable _containers = - [ "attachments", "sendfiles", "misc" ]; + ["attachments", "sendfiles", "misc"]; private static readonly IEnumerable _queues = - [ "event", "notifications", "reference-events", "mail" ]; + ["event", "notifications", "reference-events", "mail"]; private static readonly IEnumerable _tables = - [ "event", "metadata", "installationdevice"]; + ["event", "metadata", "installationdevice"]; private static readonly BlobCorsRule _corsRule = new() { @@ -84,7 +84,7 @@ public async Task StartAsync(CancellationToken cancellationToken) { // Rethrow with more explicit exception throw new BrokenDevelepmentEnvironmentException( - "The version of Azurite being ran is incompitable with our Azure packages. Read more https://contributing.bitwarden.com/link-here", + "The version of Azurite being ran is incompitable with our Azure packages. Read more https://contributing.bitwarden.com/link-here", requestedFailedEx ); } @@ -98,7 +98,7 @@ public async Task StartAsync(CancellationToken cancellationToken) private static async Task ScaffoldAsync(CancellationToken cancellationToken) { var blobServiceClient = new BlobServiceClient(_developmentUri); - + foreach (var container in _containers) { var blobContainer = blobServiceClient.GetBlobContainerClient(container); @@ -128,9 +128,9 @@ private static async Task ScaffoldAsync(CancellationToken cancellationToken) await queueClient.CreateAsync(cancellationToken: cancellationToken); } } - + var tableServiceClient = new TableServiceClient(_developmentUri); - + foreach (var table in _tables) { var tableClient = tableServiceClient.GetTableClient(table); From 6a8aec1e440bb8a09c364f20da406a95d0f2b0db Mon Sep 17 00:00:00 2001 From: Justin Baur <19896123+justindbaur@users.noreply.github.com> Date: Thu, 27 Feb 2025 05:22:48 -0500 Subject: [PATCH 4/4] Fix Api Integration Tests --- .../Factories/ApiApplicationFactory.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/Api.IntegrationTest/Factories/ApiApplicationFactory.cs b/test/Api.IntegrationTest/Factories/ApiApplicationFactory.cs index 230f0bcf0812..af468298739d 100644 --- a/test/Api.IntegrationTest/Factories/ApiApplicationFactory.cs +++ b/test/Api.IntegrationTest/Factories/ApiApplicationFactory.cs @@ -1,4 +1,5 @@ -using Bit.Identity.Models.Request.Accounts; +using Bit.Core.Platform.Infrastructure; +using Bit.Identity.Models.Request.Accounts; using Bit.IntegrationTestCommon.Factories; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.TestHost; @@ -32,6 +33,16 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) var jobService = services.First(sd => sd.ServiceType == typeof(IHostedService) && sd.ImplementationType == typeof(Jobs.JobsHostedService)); services.Remove(jobService); + // Integration tests are not configured to use Azurite yet + var azureScaffolder = services.FirstOrDefault( + sd => sd.ServiceType == typeof(IHostedService) && sd.ImplementationType == typeof(AzureScaffolder) + ); + + if (azureScaffolder != null) + { + services.Remove(azureScaffolder); + } + services.Configure(JwtBearerDefaults.AuthenticationScheme, options => { options.BackchannelHttpHandler = _identityApplicationFactory.Server.CreateHandler();