Skip to content

Commit e84b4f3

Browse files
AldavivaAldaviva
committed
Increased CDN cache duration from 90 to 180 days. Use beta releases of Unfucked and DataSizeUnits libraries instead of local builds. Update some MIME types for PGP files.
1 parent e2de7a1 commit e84b4f3

File tree

9 files changed

+210
-189
lines changed

9 files changed

+210
-189
lines changed

‎RaspberryPiDotnetRepository/Azure/BlobStorageClient.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ public interface BlobStorageClient {
2222
public class BlobStorageClientImpl(BlobContainerClient container, UploadProgressFactory uploadProgress, IOptions<Options> options, ILogger<BlobStorageClientImpl> logger)
2323
: BlobStorageClient, IDisposable {
2424

25-
private static readonly TimeSpan CACHE_DURATION = TimeSpan.FromDays(90);
25+
private static readonly TimeSpan CACHE_DURATION = TimeSpan.FromDays(180); // the max Azure CDN cache duration is 366 days
2626

2727
private readonly SemaphoreSlim uploadSemaphore = new(options.Value.storageParallelUploads);
2828

‎RaspberryPiDotnetRepository/Azure/CdnClient.cs

Lines changed: 36 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,37 @@
1-
using Azure;
2-
using Azure.ResourceManager.Cdn;
3-
using Azure.ResourceManager.Cdn.Models;
4-
5-
namespace RaspberryPiDotnetRepository.Azure;
6-
7-
/*
8-
* There seems to be no upper limit to the duration of a certificate that Azure apps can use, unlike client secrets, which must be rotated at least once every 2 years.
9-
*
10-
* You can generate a new certificate using PowerShell:
11-
*
12-
* New-SelfSignedCertificate -KeyAlgorithm RSA -KeyLength 2048 -CertStoreLocation "Cert:\CurrentUser\My" -KeyExportPolicy Exportable -KeySpec Signature -Subject "CN=RaspberryPiDotnetRepository" -NotAfter (Get-Date).AddYears(100)
13-
*
14-
* Then, use certmgr.msc to export this certificate as a CER file without the private key, and upload it to portal.azure.com > App registrations > your app > Certificates & secrets.
15-
*
16-
* Next, export the same certificate as a PFX file with the private key and a password, and pass its absolute path as certFilePath.
17-
*
18-
* You may now delete the cert from the Personal store if you want.
19-
*/
20-
public interface CdnClient {
21-
22-
Task purge();
23-
24-
}
25-
26-
public class CdnClientImpl(CdnEndpointResource? cdnEndpoint, ILogger<CdnClientImpl> logger): CdnClient {
27-
28-
public async Task purge() {
29-
if (cdnEndpoint != null) {
30-
await cdnEndpoint.PurgeContentAsync(WaitUntil.Started, new PurgeContent(["/dists/*", "/badges/*"]));
31-
logger.LogInformation("Starting CDN purge, will finish asynchronously later");
32-
} else {
33-
logger.LogInformation("No CDN configured, not purging");
34-
}
35-
}
36-
1+
using Azure;
2+
using Azure.ResourceManager.Cdn;
3+
using Azure.ResourceManager.Cdn.Models;
4+
5+
namespace RaspberryPiDotnetRepository.Azure;
6+
7+
/*
8+
* There seems to be no upper limit to the duration of a certificate that Azure apps can use, unlike client secrets, which must be rotated at least once every 2 years.
9+
*
10+
* You can generate a new certificate using PowerShell:
11+
*
12+
* New-SelfSignedCertificate -KeyAlgorithm RSA -KeyLength 2048 -CertStoreLocation "Cert:\CurrentUser\My" -KeyExportPolicy Exportable -KeySpec Signature -Subject "CN=RaspberryPiDotnetRepository" -NotAfter (Get-Date).AddYears(100)
13+
*
14+
* Then, use certmgr.msc to export this certificate as a CER file without the private key, and upload it to portal.azure.com > App registrations > your app > Certificates & secrets.
15+
*
16+
* Next, export the same certificate as a PFX file with the private key and a password, and pass its absolute path as certFilePath.
17+
*
18+
* You may now delete the cert from the Personal store if you want.
19+
*/
20+
public interface CdnClient {
21+
22+
Task purge(IEnumerable<string> paths);
23+
24+
}
25+
26+
public class CdnClientImpl(CdnEndpointResource? cdnEndpoint, ILogger<CdnClientImpl> logger): CdnClient {
27+
28+
public async Task purge(IEnumerable<string> paths) {
29+
if (cdnEndpoint != null) {
30+
await cdnEndpoint.PurgeContentAsync(WaitUntil.Started, new PurgeContent(paths));
31+
logger.LogInformation("Starting CDN purge, will finish asynchronously later");
32+
} else {
33+
logger.LogInformation("No CDN configured, not purging");
34+
}
35+
}
36+
3737
}

‎RaspberryPiDotnetRepository/Debian/Package/PackageBuilder.cs

Lines changed: 81 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -1,93 +1,82 @@
1-
using LibObjectFile.Ar;
2-
using RaspberryPiDotnetRepository.Data.ControlMetadata;
3-
using SharpCompress.Common;
4-
using SharpCompress.Compressors;
5-
using SharpCompress.Compressors.Deflate;
6-
using SharpCompress.IO;
7-
using SharpCompress.Writers;
8-
using SharpCompress.Writers.GZip;
9-
using SharpCompress.Writers.Tar;
10-
using TarWriter = Unfucked.Compression.Writers.Tar.TarWriter;
11-
12-
namespace RaspberryPiDotnetRepository.Debian.Package;
13-
14-
//TODO update comments for new Control type
15-
/// <summary>
16-
/// Create a Debian package with given files to install and control metadata.
17-
///
18-
/// <list type="number">
19-
/// <item><description>Construct a new <see cref="PackageBuilderImpl"/> instance</description></item>
20-
/// <item><description>Set <see cref="control"/> to be the control metadata (see <see href="https://www.debian.org/doc/debian-policy/ch-controlfields.html"/>)</description></item>
21-
/// <item><description>Add files to install by getting <see cref="data"/> and calling <see cref="TarWriter.WriteFile"/>, <see cref="TarWriter.WriteDirectory"/>, or <see cref="TarWriter.WriteSymLink"/> as many times as you want on it</description></item>
22-
/// <item><description>Create a destination stream (like a <see cref="FileStream"/>) and call <see cref="build"/> to save the package to a .deb file</description></item>
23-
/// </list>
24-
/// </summary>
25-
public interface PackageBuilder: IAsyncDisposable, IDisposable {
26-
27-
CompressionLevel gzipCompressionLevel { get; set; }
28-
TarWriter data { get; }
29-
30-
Task build(Control control, Stream output);
31-
32-
}
33-
34-
public class PackageBuilderImpl: PackageBuilder {
35-
36-
public const string CONTROL_ARCHIVE_FILENAME = "control.tar.gz";
37-
38-
public CompressionLevel gzipCompressionLevel { get; set; } = CompressionLevel.Default;
39-
public TarWriter data { get; }
40-
41-
private readonly Stream dataArchiveStream = new MemoryStream();
42-
private readonly GZipStream dataGzipStream;
43-
44-
public PackageBuilderImpl() {
45-
dataGzipStream = new GZipStream(NonDisposingStream.Create(dataArchiveStream), CompressionMode.Compress, gzipCompressionLevel);
46-
data = new TarWriter(dataGzipStream, new TarWriterOptions(CompressionType.None, true));
47-
}
48-
49-
public async Task build(Control control, Stream output) {
50-
data.Dispose();
51-
await dataGzipStream.DisposeAsync();
52-
dataArchiveStream.Position = 0;
53-
54-
await using Stream controlArchiveStream = new MemoryStream();
55-
using (IWriter controlArchiveWriter = WriterFactory.Open(controlArchiveStream, ArchiveType.Tar, new GZipWriterOptions { CompressionLevel = gzipCompressionLevel })) {
56-
await using Stream controlFileBuffer = control.serialize().ToStream();
57-
controlArchiveWriter.Write("./control", controlFileBuffer);
58-
}
59-
60-
controlArchiveStream.Position = 0;
61-
62-
ArArchiveFile debArchive = new() { Kind = ArArchiveKind.Common };
63-
debArchive.AddFile(new ArBinaryFile {
64-
Name = "debian-binary",
65-
Stream = "2.0\n".ToStream()
66-
});
67-
debArchive.AddFile(new ArBinaryFile {
68-
Name = CONTROL_ARCHIVE_FILENAME,
69-
Stream = controlArchiveStream
70-
});
71-
debArchive.AddFile(new ArBinaryFile {
72-
Name = "data.tar.gz",
73-
Stream = dataArchiveStream
74-
});
75-
76-
debArchive.Write(output);
77-
}
78-
79-
public void Dispose() {
80-
data.Dispose();
81-
dataGzipStream.Dispose();
82-
dataArchiveStream.Dispose();
83-
GC.SuppressFinalize(this);
84-
}
85-
86-
public async ValueTask DisposeAsync() {
87-
data.Dispose();
88-
await dataGzipStream.DisposeAsync();
89-
await dataArchiveStream.DisposeAsync();
90-
GC.SuppressFinalize(this);
91-
}
92-
1+
using LibObjectFile.Ar;
2+
using RaspberryPiDotnetRepository.Data.ControlMetadata;
3+
using SharpCompress.Common;
4+
using SharpCompress.Compressors;
5+
using SharpCompress.Compressors.Deflate;
6+
using SharpCompress.IO;
7+
using SharpCompress.Writers;
8+
using SharpCompress.Writers.GZip;
9+
using SharpCompress.Writers.Tar;
10+
using TarWriter = Unfucked.Compression.Writers.Tar.TarWriter;
11+
12+
namespace RaspberryPiDotnetRepository.Debian.Package;
13+
14+
public interface PackageBuilder: IAsyncDisposable, IDisposable {
15+
16+
CompressionLevel gzipCompressionLevel { get; set; }
17+
TarWriter data { get; }
18+
19+
Task build(Control control, Stream output);
20+
21+
}
22+
23+
public class PackageBuilderImpl: PackageBuilder {
24+
25+
public const string CONTROL_ARCHIVE_FILENAME = "control.tar.gz";
26+
27+
public CompressionLevel gzipCompressionLevel { get; set; } = CompressionLevel.Default;
28+
public TarWriter data { get; }
29+
30+
private readonly Stream dataArchiveStream = new MemoryStream();
31+
private readonly GZipStream dataGzipStream;
32+
33+
public PackageBuilderImpl() {
34+
dataGzipStream = new GZipStream(NonDisposingStream.Create(dataArchiveStream), CompressionMode.Compress, gzipCompressionLevel);
35+
data = new TarWriter(dataGzipStream, new TarWriterOptions(CompressionType.None, true));
36+
}
37+
38+
public async Task build(Control control, Stream output) {
39+
data.Dispose();
40+
await dataGzipStream.DisposeAsync();
41+
dataArchiveStream.Position = 0;
42+
43+
await using Stream controlArchiveStream = new MemoryStream();
44+
using (IWriter controlArchiveWriter = WriterFactory.Open(controlArchiveStream, ArchiveType.Tar, new GZipWriterOptions { CompressionLevel = gzipCompressionLevel })) {
45+
await using Stream controlFileBuffer = control.serialize().ToByteStream();
46+
controlArchiveWriter.Write("./control", controlFileBuffer);
47+
}
48+
49+
controlArchiveStream.Position = 0;
50+
51+
ArArchiveFile debArchive = new() { Kind = ArArchiveKind.Common };
52+
debArchive.AddFile(new ArBinaryFile {
53+
Name = "debian-binary",
54+
Stream = "2.0\n".ToByteStream()
55+
});
56+
debArchive.AddFile(new ArBinaryFile {
57+
Name = CONTROL_ARCHIVE_FILENAME,
58+
Stream = controlArchiveStream
59+
});
60+
debArchive.AddFile(new ArBinaryFile {
61+
Name = "data.tar.gz",
62+
Stream = dataArchiveStream
63+
});
64+
65+
debArchive.Write(output);
66+
}
67+
68+
public void Dispose() {
69+
data.Dispose();
70+
dataGzipStream.Dispose();
71+
dataArchiveStream.Dispose();
72+
GC.SuppressFinalize(this);
73+
}
74+
75+
public async ValueTask DisposeAsync() {
76+
data.Dispose();
77+
await dataGzipStream.DisposeAsync();
78+
await dataArchiveStream.DisposeAsync();
79+
GC.SuppressFinalize(this);
80+
}
81+
9382
}

‎RaspberryPiDotnetRepository/Orchestrator.cs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,8 @@ await Task.WhenAll(generatedPackages.Where(p => !p.isUpToDateInBlobStorage).Sele
7373
// Upload InRelease index files to Azure Blob Storage
7474
Task<BlobContentInfo?[]> releaseIndexUploads = Task.WhenAll(releaseIndexFiles.Where(file => !file.isUpToDateInBlobStorage).SelectMany(file =>
7575
new[] { file.inreleaseFilePathRelativeToRepo, file.releaseFilePathRelativeToRepo, file.releaseGpgFilePathRelativeToRepo }.Select(relativeFilePath =>
76-
blobStorage.uploadFile(Path.Combine(repoBaseDir, relativeFilePath), relativeFilePath, "text/plain", ct))));
76+
blobStorage.uploadFile(Path.Combine(repoBaseDir, relativeFilePath), relativeFilePath, Path.GetExtension(relativeFilePath) == ".gpg" ? "application/pgp-signature" : "text/plain",
77+
ct))));
7778

7879
await packageIndexUploads;
7980
await releaseIndexUploads;
@@ -82,10 +83,10 @@ await Task.WhenAll(generatedPackages.Where(p => !p.isUpToDateInBlobStorage).Sele
8283
await Task.WhenAll(badgeFiles.Where(file => !file.isUpToDateInBlobStorage)
8384
.Select(file => blobStorage.uploadFile(Path.Combine(repoBaseDir, file.filePathRelativeToRepo), file.filePathRelativeToRepo, "application/json", ct)));
8485
await blobStorage.uploadFile(Path.Combine(repoBaseDir, readmeFilename), readmeFilename, "text/plain", ct);
85-
await blobStorage.uploadFile(Path.Combine(repoBaseDir, gpgPublicKeyFile), gpgPublicKeyFile, "application/octet-stream", ct);
86+
await blobStorage.uploadFile(Path.Combine(repoBaseDir, gpgPublicKeyFile), gpgPublicKeyFile, "application/pgp-keys", ct);
8687

8788
// Clear CDN cache
88-
await cdnClient.purge();
89+
await cdnClient.purge(["/dists/*", "/badges/*", "/manifest.json"]);
8990

9091
// Upload manifest.json file to Azure Blob Storage
9192
await blobStorage.uploadFile(manifestManager.manifestFilePath, manifestManager.manifestFilename, "application/json", ct);

‎RaspberryPiDotnetRepository/RaspberryPiDotnetRepository.csproj

Lines changed: 10 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<ImplicitUsings>enable</ImplicitUsings>
77
<Nullable>enable</Nullable>
88
<NoWarn>8524</NoWarn>
9-
<Version>1.0.0</Version>
9+
<Version>1.0.1</Version>
1010
<Authors>Ben Hutchison</Authors>
1111
<Copyright>© 2024 $(Authors)</Copyright>
1212
<Company>$(Authors)</Company>
@@ -24,13 +24,19 @@
2424
<ItemGroup>
2525
<PackageReference Include="Azure.Identity" Version="1.12.0" />
2626
<PackageReference Include="Azure.ResourceManager.Cdn" Version="1.3.0" />
27-
<PackageReference Include="Azure.Storage.Blobs" Version="12.21.2" />
27+
<PackageReference Include="Azure.Storage.Blobs" Version="12.22.0" />
2828
<PackageReference Include="Bom.Squad" Version="0.3.0" />
29+
<PackageReference Include="DataSizeUnits" Version="3.0.0-beta1" />
2930
<PackageReference Include="LibObjectFile" Version="0.6.0" />
3031
<PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
31-
<PackageReference Include="PgpCore" Version="6.5.0" />
32-
<PackageReference Include="SharpCompress" Version="0.37.2" />
32+
<PackageReference Include="PgpCore" Version="6.5.1" />
33+
<PackageReference Include="SharpCompress" Version="0.38.0" />
34+
<PackageReference Include="System.Text.Json" Version="8.0.4" /> <!-- pinned to non-vulnerable version -->
3335
<PackageReference Include="ThrottleDebounce" Version="2.0.0" />
36+
<PackageReference Include="Unfucked" Version="0.0.0-beta3" />
37+
<PackageReference Include="Unfucked.Compression" Version="0.0.0-beta2" />
38+
<PackageReference Include="Unfucked.DI" Version="0.0.0-beta2" />
39+
<PackageReference Include="Unfucked.PGP" Version="0.0.0-beta2" />
3440
</ItemGroup>
3541

3642
<ItemGroup>
@@ -42,24 +48,6 @@
4248
</None>
4349
</ItemGroup>
4450

45-
<ItemGroup>
46-
<Reference Include="DataSizeUnits">
47-
<HintPath>..\..\DataSizeUnits\DataSizeUnits\bin\Debug\net5.0\DataSizeUnits.dll</HintPath>
48-
</Reference>
49-
<Reference Include="Unfucked">
50-
<HintPath>..\..\Unfucked\Unfucked\bin\Debug\net8.0\Unfucked.dll</HintPath>
51-
</Reference>
52-
<Reference Include="Unfucked.Compression">
53-
<HintPath>..\..\Unfucked\Unfucked.Compression\bin\Debug\netstandard2.0\Unfucked.Compression.dll</HintPath>
54-
</Reference>
55-
<Reference Include="Unfucked.DependencyInjection">
56-
<HintPath>..\..\Unfucked\Unfucked.DependencyInjection\bin\Debug\net6.0\Unfucked.DependencyInjection.dll</HintPath>
57-
</Reference>
58-
<Reference Include="Unfucked.PGP">
59-
<HintPath>..\..\Unfucked\Unfucked.PGP\bin\Debug\netstandard2.0\Unfucked.PGP.dll</HintPath>
60-
</Reference>
61-
</ItemGroup>
62-
6351
<PropertyGroup Condition="'$(GITHUB_ACTIONS)' == 'true' or '$(Configuration)' == 'Release'">
6452
<ContinuousIntegrationBuild>true</ContinuousIntegrationBuild>
6553
</PropertyGroup>

‎RaspberryPiDotnetRepository/Debian/Package/StatisticsService.cs renamed to ‎RaspberryPiDotnetRepository/StatisticsService.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
using DataSizeUnits;
22
using System.Diagnostics;
33

4-
namespace RaspberryPiDotnetRepository.Debian.Package;
4+
namespace RaspberryPiDotnetRepository;
55

66
public interface StatisticsService {
77

‎RaspberryPiDotnetRepository/appsettings.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
"cdnTenantId": null,
1414

1515
// Insert the application/client ID of an Azure OAuth app registration (Azure Portal > App registrations > your app > Overview > Essentials > Application (client) ID)
16+
// This application must be granted a role with both Microsoft.Cdn/profiles/endpoints/read and Microsoft.Cdn/profiles/endpoints/Purge/action permissions on the Resource Group, such as the built-in CDN Endpoint Contributor role, or a custom role that has those two permissions.
1617
"cdnClientId": null,
1718

1819
/*

0 commit comments

Comments
 (0)