Skip to content

Aggregate Protobuf build telemetry into a single per-build report #4575

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pepone wants to merge 1 commit into
base: main
Choose a base branch
from
+64 −62
Open

Aggregate Protobuf build telemetry into a single per-build report #4575

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 34 additions & 9 deletions src/IceRpc.Protobuf.Tools/IceRpc.Protobuf.Tools.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,6 @@
AssemblyFile="$(IceRpcProtobufToolsTaskAssembliesPath)IceRpc.Protobuf.Tools.dll"
Runtime="NET"
/>
<UsingTask
TaskName="IceRpc.Protobuf.Tools.BuildTelemetryTask"
TaskFactory="$(IceRpcProtobufToolsTaskFactory)"
AssemblyFile="$(IceRpcProtobufToolsTaskAssembliesPath)IceRpc.Protobuf.Tools.dll"
Runtime="NET"
/>
<ItemGroup Condition="'$(IceRpcProtobufToolsSourceBuild)' == 'true'">
<!-- Add a reference to the Tools project to ensure it's built before we use its tasks -->
<ProjectReference Include="$(MSBuildThisFileDirectory)IceRpc.Protobuf.Tools.csproj" ReferenceOutputAssembly="false" />
Expand Down Expand Up @@ -69,6 +63,24 @@
<Output ItemName="_ProtoFile" TaskParameter="ComputedSources" />
</UpToDateCheckTask>

<PropertyGroup>
<_ProtocPluginScriptExtension Condition="$([MSBuild]::IsOSPlatform('Windows'))">.bat</_ProtocPluginScriptExtension>
<_ProtocPluginScriptExtension Condition="!$([MSBuild]::IsOSPlatform('Windows'))">.sh</_ProtocPluginScriptExtension>
</PropertyGroup>

<ItemGroup>
<!-- Code-generating plug-ins; protoc runs once per ProtoFile to produce a dependency file per file. -->
<_ProtocCodegenPlugin Include="csharp" />
<_ProtocCodegenPlugin Include="icerpc-csharp">
<Path>$(IceRpcProtocGenPath)protoc-gen-icerpc-csharp$(_ProtocPluginScriptExtension)</Path>
</_ProtocCodegenPlugin>

<!-- Telemetry plug-in; runs once over all ProtoFile items so the upload reports the whole build. -->
<_ProtocTelemetryPlugin Include="icerpc-build-telemetry">
<Path>$(IceRpcProtocBuildTelemetryPath)protoc-gen-icerpc-build-telemetry$(_ProtocPluginScriptExtension)</Path>
</_ProtocTelemetryPlugin>
</ItemGroup>

<!-- Compile the Proto files
The search path determines where protoc compiler locates import files, we add:
- $(IceRpcProtocPath) to import google well-known files from IceRpc.Protobuf.Tools.
Expand All @@ -81,13 +93,26 @@
OutputDir="@(_ProtoFile->'%(OutputDir)')"
SearchPath="@(ProtoSearchPath->'%(FullPath)');$(IceRpcProtocPath);$(MSBuildProjectDirectory)"
ToolsPath="$(IceRpcProtocPath)$(IceRpcProtocPrefix)"
GenPath="$(IceRpcProtocGenPath)"
BuildTelemetryPath="$(IceRpcProtocBuildTelemetryPath)"
RunBuildTelemetry="$(IceRpcBuildTelemetry)"
Plugins="@(_ProtocCodegenPlugin)"
AdditionalOptions="--dependency_out=%(_ProtoFile.OutputDir)/%(_ProtoFile.OutputFileName).d"
Sources="%(_ProtoFile.Identity)"
Condition="'%(_ProtoFile.UpToDate)' != 'true'"
/>

<!--
Run the build-telemetry plug-in once over all ProtoFile items so it can compute a single per-build
compilation hash and file count from the descriptors, regardless of which files were up-to-date.
-->
<ProtocTask
WorkingDirectory="$(MSBuildProjectDirectory)"
OutputDir="$(IntermediateOutputPath)"
SearchPath="@(ProtoSearchPath->'%(FullPath)');$(IceRpcProtocPath);$(MSBuildProjectDirectory)"
ToolsPath="$(IceRpcProtocPath)$(IceRpcProtocPrefix)"
Plugins="@(_ProtocTelemetryPlugin)"
Sources="@(_ProtoFile)"
Condition="'$(IceRpcBuildTelemetry)' == 'true'"
/>

<!--
Include all C# generated source items that have not been already included. We delay this until we are
running the ProtoCompile target so that default includes are already processed.
Expand Down
83 changes: 30 additions & 53 deletions src/IceRpc.Protobuf.Tools/ProtocTask.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
// Copyright (c) ZeroC, Inc.

using IceRpc.CaseConverter.Internal;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using System.Diagnostics;
Expand All @@ -12,15 +11,24 @@ namespace IceRpc.Protobuf.Tools;
// Properties should not return arrays, disabled as this is standard for MSBuild tasks.
#pragma warning disable CA1819

/// <summary>A MSBuild task to generate code from Protobuf files using <c>protoc</c> C# built-in generator and
/// <c>protoc-gen-icerpc-csharp</c> generator.</summary>
/// <summary>An MSBuild task that runs <c>protoc</c> with a configurable set of plug-ins.</summary>
public class ProtocTask : ToolTask
{
/// <summary>Gets or sets the output directory for the generated code; corresponds to the
/// <c>--icerpc-csharp_out=</c> option of the <c>protoc</c> compiler.</summary>
/// <summary>Gets or sets additional options to pass verbatim to <c>protoc</c>.</summary>
public string[] AdditionalOptions { get; set; } = [];

/// <summary>Gets or sets the output directory shared by all configured plug-ins; corresponds to the
/// <c>--&lt;name&gt;_out=</c> option of the <c>protoc</c> compiler.</summary>
[Required]
public string OutputDir { get; set; } = "";

/// <summary>Gets or sets the protoc plug-ins to run. Each item's <c>Identity</c> is the plug-in name (for example
/// <c>csharp</c>, <c>icerpc-csharp</c>, <c>icerpc-build-telemetry</c>). The <c>Path</c> metadata, when not empty,
/// is the full path to the plug-in script and is passed via <c>--plugin protoc-gen-&lt;name&gt;=&lt;path&gt;</c>;
/// an empty <c>Path</c> indicates a protoc built-in such as <c>csharp</c>.</summary>
[Required]
public ITaskItem[] Plugins { get; set; } = [];

/// <summary>Gets or sets the directories in which to search for imports, corresponds to <c>-I</c> protoc compiler
/// option.</summary>
public string[] SearchPath { get; set; } = [];
Expand All @@ -34,18 +42,6 @@ public class ProtocTask : ToolTask
[Required]
public string ToolsPath { get; set; } = "";

/// <summary>Gets or sets the directory containing the protoc-gen-icerpc-csharp scripts.</summary>
[Required]
public string GenPath { get; set; } = "";

/// <summary>Gets or sets the directory containing the protoc-gen-icerpc-build-telemetry scripts.</summary>
[Required]
public string BuildTelemetryPath { get; set; } = "";

/// <summary>Gets or sets a value indicating whether to run the build telemetry plug-in.</summary>
[Required]
public bool RunBuildTelemetry { get; set; }

/// <summary>Gets or sets the working directory for executing the protoc compiler from.</summary>
[Required]
[System.Diagnostics.CodeAnalysis.SuppressMessage(
Expand All @@ -63,44 +59,31 @@ protected override string GenerateCommandLineCommands()
{
var builder = new CommandLineBuilder(false);

// Specify the full path to the protoc-gen-icerpc-csharp script.

string genScriptName =
RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ?
"protoc-gen-icerpc-csharp.bat" : "protoc-gen-icerpc-csharp.sh";
builder.AppendSwitch("--plugin");
builder.AppendFileNameIfNotNull($"protoc-gen-icerpc-csharp={Path.Combine(GenPath, genScriptName)}");
// Register and enable each plug-in.
foreach (ITaskItem plugin in Plugins)
{
string name = plugin.ItemSpec;
string pluginPath = plugin.GetMetadata("Path");

// Add --csharp_out to generate Protobuf C# code
builder.AppendSwitch("--csharp_out");
builder.AppendFileNameIfNotNull(OutputDir);
if (!string.IsNullOrEmpty(pluginPath))
{
builder.AppendSwitch("--plugin");
builder.AppendFileNameIfNotNull($"protoc-gen-{name}={pluginPath}");
}

// Add --icerpc-csharp_out to generate IceRPC + Protobuf integration code
builder.AppendSwitch("--icerpc-csharp_out");
builder.AppendFileNameIfNotNull(OutputDir);
builder.AppendSwitch($"--{name}_out");
builder.AppendFileNameIfNotNull(OutputDir);
}

if (RunBuildTelemetry)
foreach (string option in AdditionalOptions)
{
// Enable build telemetry
string buildTelemetryScriptName =
RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ?
"protoc-gen-icerpc-build-telemetry.bat" : "protoc-gen-icerpc-build-telemetry.sh";

// Specify the full path to the protoc-gen-icerpc-build-telemetry script.
builder.AppendSwitch("--plugin");
builder.AppendFileNameIfNotNull(
$"protoc-gen-icerpc-build-telemetry={Path.Combine(BuildTelemetryPath, buildTelemetryScriptName)}");

// Add --icerpc-build-telemetry_out to enable build telemetry, even though the build telemetry plug-in
// doesn't need to generate any output files.
builder.AppendSwitch("--icerpc-build-telemetry_out");
builder.AppendFileNameIfNotNull(OutputDir);
builder.AppendTextUnquoted(" ");
builder.AppendTextUnquoted(option);
Comment on lines +80 to +81
}

var searchPath = new List<string>(SearchPath);

// Add the sources directories to the import search path
var computedSources = new List<ITaskItem>();
// Add the sources directories to the import search path.
foreach (ITaskItem source in Sources)
{
string fullPath = source.GetMetadata("FullPath");
Expand All @@ -109,14 +92,8 @@ protected override string GenerateCommandLineCommands()
{
searchPath.Add(directory);
}

// Add dependency_out to generate dependency files
builder.AppendSwitch("--dependency_out");
builder.AppendFileNameIfNotNull(
Path.Combine(OutputDir, $"{source.GetMetadata("FileName").ToPascalCase()}.d"));
}

// Add protoc searchPath paths
foreach (string path in searchPath)
{
builder.AppendSwitch("-I");
Expand Down
Loading