Skip to content

Commit 332fc80

Browse files
leeoadesleeoades
committed
Add new API end point that returns a searchable HTML version of a latest release with the associated metadata
Move ReleaseSearchViewModel to own file. Code tidy. WIP Putting together the new Content Processor for Search Rename releaseId to releaseVersionId rename releaseId to releaseVersionId in Content Processor code tidy Ensure the summary is made safe for upload Ensure Title is also made ascii and no spaces. Also, replace the magic strings Code tidy Add Azure Blob Storage Client integration tests. Also added a blob download and delete methods. Add a couple more tests for Download and Delete non-existent blobs Add tests for HostBuilder. Flow cancellation token through everywhere. Add tests for SearchableDocumentCreator Rename the search Processor to Search.FunctionApp Moved projects to Search/Function App solution folder Mask connection string Code tidy Getting the function app set up Test config and container registrations set up If service is not configured, fail on startup instead of waiting for a message to arrive Fix config Severed the ref to viewmodels. Added dto. Code Tidy. Removed unused refs. Remove the hardcoded queue names from the trigger function and move them to config Add the local.settings.json file Rename the function Add ReleaseId to the searchable view model to use as the blob name Fix Fix
1 parent 8fa0813 commit 332fc80

File tree

44 files changed

+1751
-20
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1751
-20
lines changed

‎src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/ReleaseController.cs

+1-2
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,7 @@ private Task<Either<ActionResult, ReleaseViewModel>> GetReleaseViewModel(
9595
);
9696
});
9797

98-
private Task<Either<ActionResult, ReleaseSearchViewModel>> GetLatestReleaseSearchViewModel(
99-
string publicationSlug) =>
98+
private Task<Either<ActionResult, ReleaseSearchViewModel>> GetLatestReleaseSearchViewModel(string publicationSlug) =>
10099
_publicationCacheService.GetPublication(publicationSlug)
101100
.OnSuccessCombineWith(_ => _releaseCacheService.GetRelease(publicationSlug))
102101
.OnSuccess(tuple =>

‎src/GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Tests/Builders/AzureBlobStorageClientBuilder.cs

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
using GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Clients;
2+
using GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Clients.AzureBlobStorage;
3+
using Moq;
4+
using Blob = GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Clients.AzureBlobStorage.Blob;
5+
6+
namespace GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Tests.Builders;
7+
8+
internal class AzureBlobStorageClientBuilder
9+
{
10+
private readonly Mock<IAzureBlobStorageClient> _mock = new(MockBehavior.Strict);
11+
12+
public AzureBlobStorageClientBuilder()
13+
{
14+
Assert = new(_mock);
15+
16+
_mock.Setup(mock => mock.UploadAsync(
17+
It.IsAny<string>(),
18+
It.IsAny<string>(),
19+
It.IsAny<Blob>(),
20+
It.IsAny<CancellationToken>()))
21+
.Returns(Task.CompletedTask);
22+
}
23+
24+
public IAzureBlobStorageClient Build() => _mock.Object;
25+
26+
public Asserter Assert { get; }
27+
public class Asserter(Mock<IAzureBlobStorageClient> mock)
28+
{
29+
public void BlobWasUploaded(
30+
string? containerName = null,
31+
string? blobName = null,
32+
Func<Blob, bool>? whereBlob = null)
33+
{
34+
mock.Verify(m => m.UploadAsync(
35+
It.Is<string>(actualContainerName => containerName == null || actualContainerName == containerName),
36+
It.Is<string>(actualBlobName => blobName == null || actualBlobName == blobName),
37+
It.Is<Blob>(blob => whereBlob == null || whereBlob(blob)),
38+
It.IsAny<CancellationToken>()), Times.Once);
39+
}
40+
}
41+
}

‎src/GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Tests/Builders/ContentApiClientBuilder.cs

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
using GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Clients;
2+
using GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Clients.ContentApi;
3+
using Moq;
4+
5+
namespace GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Tests.Builders;
6+
7+
internal class ContentApiClientBuilder
8+
{
9+
private readonly Mock<IContentApiClient> _mock = new Mock<IContentApiClient>(MockBehavior.Strict);
10+
private readonly ReleaseSearchViewModelBuilder _releaseSearchViewModelBuilder = new();
11+
private ReleaseSearchViewModelDto? _releaseSearchViewModel;
12+
13+
public ContentApiClientBuilder()
14+
{
15+
Assert = new Asserter(_mock);
16+
17+
_mock
18+
.Setup(m => m.GetPublicationLatestReleaseSearchViewModelAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
19+
.ReturnsAsync(_releaseSearchViewModel ?? _releaseSearchViewModelBuilder.Build());
20+
}
21+
22+
public IContentApiClient Build()
23+
{
24+
return _mock.Object;
25+
}
26+
27+
public ContentApiClientBuilder WhereReleaseSearchViewModelIs(ReleaseSearchViewModelDto releaseSearchViewModel)
28+
{
29+
_releaseSearchViewModel = releaseSearchViewModel;
30+
return this;
31+
}
32+
33+
public ContentApiClientBuilder WhereReleaseSearchViewModelIs(Func<ReleaseSearchViewModelBuilder, ReleaseSearchViewModelBuilder> modifyReleaseSearchViewModel)
34+
{
35+
modifyReleaseSearchViewModel(_releaseSearchViewModelBuilder);
36+
return this;
37+
}
38+
39+
public Asserter Assert { get; }
40+
public class Asserter(Mock<IContentApiClient> mock)
41+
{
42+
public void ContentWasLoadedFor(string publicationSlug)
43+
{
44+
mock.Verify(m => m.GetPublicationLatestReleaseSearchViewModelAsync(publicationSlug, It.IsAny<CancellationToken>()), Times.Once);
45+
}
46+
}
47+
}

‎src/GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Tests/Builders/ReleaseSearchViewModelBuilder.cs

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
using GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Clients;
2+
using GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Clients.ContentApi;
3+
4+
namespace GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Tests.Builders;
5+
6+
public class ReleaseSearchViewModelBuilder
7+
{
8+
private string? _summary;
9+
private string? _title;
10+
private Guid? _releaseVersionId;
11+
private Guid? _releaseId;
12+
13+
public ReleaseSearchViewModelDto Build() => new()
14+
{
15+
ReleaseId = _releaseId ?? new Guid("11223344-5566-7788-9900-123456789abc"),
16+
ReleaseVersionId = _releaseVersionId ?? new Guid("12345678-1234-1234-1234-123456789abc"),
17+
Published = new DateTimeOffset(2025, 02, 21, 09, 24, 01, TimeSpan.Zero),
18+
PublicationTitle = _title ?? "Publication Title",
19+
Summary = _summary ?? "This is a summary.",
20+
Theme = "Theme",
21+
Type = "Official Statistics",
22+
TypeBoost = 10,
23+
PublicationSlug = "publication-slug",
24+
ReleaseSlug = "release-slug",
25+
HtmlContent = "<p>This is some Html Content</p>",
26+
};
27+
28+
public ReleaseSearchViewModelBuilder WithSummary(string summary)
29+
{
30+
_summary = summary;
31+
return this;
32+
}
33+
34+
public ReleaseSearchViewModelBuilder WithTitle(string title)
35+
{
36+
_title = title;
37+
return this;
38+
}
39+
40+
public ReleaseSearchViewModelBuilder WithReleaseVersionId(Guid releaseVersionId)
41+
{
42+
_releaseVersionId = releaseVersionId;
43+
return this;
44+
}
45+
46+
public ReleaseSearchViewModelBuilder WithReleaseId(Guid releaseId)
47+
{
48+
_releaseId = releaseId;
49+
return this;
50+
}
51+
}

‎src/GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Tests/Clients/AzureBlobStorageClientTests.cs

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
using Azure.Storage.Blobs;
2+
using GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Clients;
3+
using GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Clients.AzureBlobStorage;
4+
using GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Exceptions;
5+
using Blob = GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Clients.AzureBlobStorage.Blob;
6+
7+
namespace GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Tests.Clients;
8+
9+
public class AzureBlobStorageClientTests
10+
{
11+
private IAzureBlobStorageClient GetSut(string connectionString)
12+
{
13+
var blobServiceClient = new BlobServiceClient(connectionString);
14+
return new AzureBlobStorageClient(blobServiceClient);
15+
}
16+
17+
public class IntegrationTests
18+
{
19+
/// <summary>
20+
/// Integration Tests
21+
/// </summary>
22+
public class HiveITAzureAccount : AzureBlobStorageClientTests
23+
{
24+
private string _integrationTestStorageAccountConnectionString = "** add Azure connection string here **";
25+
private string _integrationTestContainerName = "integration-tests";
26+
27+
private IAzureBlobStorageClient GetSut() => base.GetSut(_integrationTestStorageAccountConnectionString);
28+
29+
[Fact(Skip = "This integration test creates a blob in an Azure Storage Account and retrieves it again.")]
30+
public async Task CanUploadBlob()
31+
{
32+
// ARRANGE
33+
var uniqueBlobName = Guid.NewGuid().ToString();
34+
var sut = GetSut();
35+
var blob = new Blob("This is a test", new Dictionary<string, string>
36+
{
37+
{"key1", "value1"},
38+
{"key2", "value2"},
39+
{"timestamp", DateTimeOffset.Now.ToString("u")}
40+
});
41+
42+
// ACT
43+
await sut.UploadAsync(_integrationTestContainerName, uniqueBlobName, blob);
44+
45+
// ASSERT
46+
var actual = await sut.DownloadAsync(_integrationTestContainerName, uniqueBlobName);
47+
Assert.Equal(blob, actual);
48+
await sut.DeleteAsync(_integrationTestContainerName, uniqueBlobName);
49+
}
50+
51+
[Fact(Skip = "This integration test gets a non-existent blob from Azure Storage Account.")]
52+
public async Task DownloadBlob_WhenBlobDoesNotExist_ThenThrows()
53+
{
54+
// ARRANGE
55+
var uniqueBlobName = Guid.NewGuid().ToString();
56+
var sut = GetSut();
57+
58+
// ACT
59+
var actual = await Record.ExceptionAsync(() => sut.DownloadAsync(_integrationTestContainerName, uniqueBlobName));
60+
61+
// ASSERT
62+
Assert.NotNull(actual);
63+
var azureBlobStorageNotFoundException = Assert.IsType<AzureBlobStorageNotFoundException>(actual);
64+
Assert.Equal(uniqueBlobName, azureBlobStorageNotFoundException.BlobName);
65+
Assert.Equal(_integrationTestContainerName, azureBlobStorageNotFoundException.ContainerName);
66+
}
67+
68+
[Fact(Skip = "This integration test deletes a non-existent blob from Azure Storage Account.")]
69+
public async Task DeleteBlob_WhenBlobDoesNotExist_ThenDoesNotThrow()
70+
{
71+
// ARRANGE
72+
var uniqueBlobName = Guid.NewGuid().ToString();
73+
var sut = GetSut();
74+
75+
// ACT
76+
await sut.DeleteAsync(_integrationTestContainerName, uniqueBlobName);
77+
}
78+
}
79+
}
80+
}

‎src/GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Tests/Clients/ContentApiClientTests.cs

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
using System.Reflection;
2+
using GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Clients;
3+
using GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Clients.ContentApi;
4+
using GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Exceptions;
5+
6+
namespace GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Tests.Clients;
7+
8+
public class ContentApiClientTests
9+
{
10+
private IContentApiClient GetSut(Action<HttpClient>? modifyHttpClient = null)
11+
{
12+
var httpClient = new HttpClient { Timeout = TimeSpan.FromSeconds(10) };
13+
modifyHttpClient?.Invoke(httpClient);
14+
return new ContentApiClient(httpClient);
15+
}
16+
17+
/// <summary>
18+
/// Separately assert that each of the public properties of two instances of an object are equal.
19+
/// This provides a finer grained explanation of where two objects differ in equality.
20+
/// </summary>
21+
private void AssertAllPropertiesMatch<T>(T expected, T actual)
22+
{
23+
AssertAll(typeof(T)
24+
.GetProperties(BindingFlags.Public | BindingFlags.Instance)
25+
.Select(propertyInfo => (Action)(() => Assert.Equal(propertyInfo.GetValue(expected), propertyInfo.GetValue(actual)))));
26+
Assert.Equal(expected, actual); // Belt and braces
27+
}
28+
29+
private void AssertAll(params IEnumerable<Action>[] assertions) =>
30+
Assert.All(assertions.SelectMany(a => a), assertion => assertion());
31+
32+
public class BasicTests : ContentApiClientTests
33+
{
34+
[Fact]
35+
public void Can_instantiate_SUT() => Assert.NotNull(GetSut());
36+
}
37+
38+
public abstract class LocalDevelopmentIntegrationTests : ContentApiClientTests
39+
{
40+
/// <summary>
41+
/// Ensure ContentAPI is running locally on port 5010
42+
/// </summary>
43+
public class CallLocalService : LocalDevelopmentIntegrationTests
44+
{
45+
private IContentApiClient GetSut() =>
46+
base.GetSut(httpClient => httpClient.BaseAddress = new Uri("http://localhost:5010"));
47+
48+
[Fact(Skip = "This test is only for local development")]
49+
public async Task GetExampleSeedDocument()
50+
{
51+
// ARRANGE
52+
var sut = GetSut();
53+
var publicationSlug = "seed-publication-permanent-and-fixed-period-exclusions-in-england";
54+
55+
// ACT
56+
var actual = await sut.GetPublicationLatestReleaseSearchViewModelAsync(publicationSlug);
57+
58+
// ASSERT
59+
Assert.NotNull(actual);
60+
var expected = new ReleaseSearchViewModelDto
61+
{
62+
ReleaseVersionId = new Guid("46c5d916-ee40-49bd-cfdc-08dc1c5c621e"),
63+
Published = DateTimeOffset.Parse("2018-07-18T23:00:00Z"),
64+
PublicationTitle = "Seed publication - Permanent and fixed-period exclusions in England",
65+
Summary = "Read national statistical summaries, view charts and tables and download data files.",
66+
Theme = "Seed theme - Pupils and schools",
67+
Type = "OfficialStatistics",
68+
TypeBoost = 5,
69+
PublicationSlug = "seed-publication-permanent-and-fixed-period-exclusions-in-england",
70+
ReleaseSlug = "2016-17",
71+
HtmlContent = "<html>\n <head>\n <title>Seed publication - Permanent and fixed-period exclusions in England</title>\n </head>\n <body>\n<h1>Seed publication - Permanent and fixed-period exclusions in England</h1>\n<h2>Academic year 2016/17</h2>\n<h3>Summary</h3>\n<p>Read national statistical summaries, view charts and tables and download data files.</p>\n<h3>Headlines</h3>\n<p>The rate of permanent exclusions has increased since last year from 0.08 per cent of pupil enrolments in 2015/16 to 0.10 per cent in 2016/17.</p>\n<h3>About this release</h3>\n<p>The statistics and data cover permanent and fixed period exclusions and school-level exclusions during the 2016/17 academic year in the following state-funded school types as reported in the school census.</p>\n<h3>Permanent exclusions</h3>\n<p>The number of permanent exclusions has increased across all state-funded primary, secondary and special schools to 7,720 - up from 6,685 in 2015/16.</p>\n<h3>Fixed-period exclusions</h3>\n<p>The number of fixed-period exclusions has increased across all state-funded primary, secondary and special schools to 381,865 - up from 339,360 in 2015/16.</p>\n<h3>Number and length of fixed-period exclusions</h3>\n<p>The number of pupils with one or more fixed-period exclusion has increased across state-funded primary, secondary and special schools to 183,475 (2.29% of pupils) up from 167,125 (2.11% of pupils) in 2015/16.</p>\n<h3>Reasons for exclusions</h3>\n<p>All reasons (except bullying and theft) saw an increase in permanent exclusions since 2015/16.</p>\n<h3>Exclusions by pupil characteristics</h3>\n<p>There was a similar pattern to previous years where the following groups (where higher exclusion rates are expected) showed an increase in exclusions since 2015/16.</p>\n<h3>Independent exclusion reviews</h3>\n<p>There were 560 reviews lodged with independent review panels in maintained primary, secondary and special schools and academies of which 525 (93.4%) were determined and 45 (8.0%) resulted in an offer of reinstatement.</p>\n<h3>Pupil referral units exclusions</h3>\n<p>The permanent exclusion rate in pupil referral units decreased to 0.13 - down from 0.14% in 2015/16.</p>\n<h3>Regional and local authority (LA) breakdown</h3>\n<p>There's considerable variation in the permanent exclusion and fixed-period exclusion rate at the LA level.</p>\n </body>\n</html>\n"
72+
};
73+
AssertAllPropertiesMatch(expected, actual);
74+
}
75+
}
76+
77+
public class CallUnknownService : LocalDevelopmentIntegrationTests
78+
{
79+
private IContentApiClient GetSut() =>
80+
base.GetSut(httpClient => httpClient.BaseAddress = new Uri("http://localhost:8123")); // Cause a 404
81+
82+
[Fact(Skip = "This test is only for local development")]
83+
public async Task UnknownEndpointShouldThrow()
84+
{
85+
// ARRANGE
86+
var sut = GetSut();
87+
var publicationSlug = "seed-publication-permanent-and-fixed-period-exclusions-in-england";
88+
89+
// ACT
90+
var exception = await Record.ExceptionAsync(() => sut.GetPublicationLatestReleaseSearchViewModelAsync(publicationSlug));
91+
92+
// ASSERT
93+
Assert.NotNull(exception);
94+
var unableToGetPublicationLatestReleaseSearchViewModelException = Assert.IsType<UnableToGetPublicationLatestReleaseSearchViewModelException>(exception);
95+
Assert.Contains(publicationSlug, unableToGetPublicationLatestReleaseSearchViewModelException.Message);
96+
}
97+
}
98+
99+
}
100+
}

‎src/GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Tests/Extensions/ConfigurationBuilderExtensions.cs

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
using System.Text;
2+
using Microsoft.Extensions.Configuration;
3+
4+
namespace GovUk.Education.ExploreEducationStatistics.Content.Search.FunctionApp.Tests.Extensions;
5+
6+
public static class ConfigurationBuilderExtensions
7+
{
8+
public static IConfigurationBuilder AddJsonString(this IConfigurationBuilder builder, string json) => builder.AddJsonStream(new MemoryStream(Encoding.UTF8.GetBytes(json)));
9+
}

0 commit comments

Comments
 (0)