udx/AIGame
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit: AIGame Unity client with URP, HybridCLR, YooAsset and tooling.

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
udx
2026-09-21 14:21:07 +08:00
co-authored by Cursor
commit f3f66ab063
931 changed files with 112366 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AIClient/Packages/com.unity.pipeline/Runtime/Telemetry/EvalUsageFingerprinter.cs
+422
View File
@@ -0,0 +1,422 @@
using System.Collections.Generic;
using System.Linq;
using System.Text;
#if UNITY_EDITOR || (UNITY_STANDALONE && DEBUG)
using UnityPipeline.Microsoft.CodeAnalysis;
using UnityPipeline.Microsoft.CodeAnalysis.CSharp;
using UnityPipeline.Microsoft.CodeAnalysis.CSharp.Syntax;
#endif
namespace Unity.Pipeline.Telemetry
{
/// <summary>
/// Structural classification of an eval body. Matches the epic's usage buckets: the majority of
/// eval calls are one-liners (a single method invoke / value read / flag set), which we tag as
/// <see cref="SingleExpression"/>; anything with more than one statement (polling loops, bulk
/// setup) is <see cref="Statements"/>.
/// </summary>
static class EvalClassification
{
public const string SingleExpression = "single-expression";
public const string Statements = "statements";
}
/// <summary>
/// Result of analysing one eval body: its structural <see cref="Classification"/> and the ranked
/// API <see cref="Fingerprints"/> (top-level member-access paths) extracted from the syntax tree.
/// </summary>
struct EvalUsageAnalysis
{
public string Classification;
public IReadOnlyList<string> Fingerprints;
}
/// <summary>
/// Extracts a privacy-preserving "fingerprint" of an eval body from its C# syntax tree, without
/// storing the raw source (AUTHAPI-29). A fingerprint is the top-level member-access path of each
/// access site — e.g. <c>AssetDatabase.Refresh</c>, <c>PlayerSettings.SetScriptingBackend</c>,
/// <c>Object.FindFirstObjectByType&lt;T&gt;</c>, <c>MatchManager.Instance.State</c> — which is
/// enough to rank which APIs agents actually reach for (steering the eval-displacement backlog,
/// AUTHAPI-24).
///
/// PRIVACY CONTRACT — strict whitelist rendering. The only source text that can ever appear in a
/// fingerprint is an identifier, type name, or member name taken from the syntax tree. Literals,
/// interpolated strings, and every expression form not on the whitelist are replaced by the
/// neutral <see cref="Placeholder"/> (<c>"secret".Length</c> records as <c>&lt;expr&gt;.Length</c>);
/// object creations render as <c>new TypeName</c> (constructor arguments never appear); casts
/// unwrap to the inner expression (<c>((Foo)o).Bar</c> records as <c>o.Bar</c>); generic type
/// arguments are normalised to arity placeholders (<c>&lt;T&gt;</c>). A path that would be nothing
/// but the placeholder carries no API information and is dropped entirely.
///
/// Conditional-access chains (<c>a?.B.C()</c>) are fingerprinted with <c>?.</c> normalised to
/// <c>.</c>, so they rank together with their unconditional form.
///
/// Known blind spots — the ranking UNDERCOUNTS these forms (deliberate, to keep the walker
/// simple and re-entrancy-free); consumers of the report should not read absence as proof of
/// non-use:
/// <list type="bullet">
/// <item><description>Arguments nested inside a chain's receiver call are not walked: in
/// <c>Configure(Settings.Load()).Apply()</c> only <c>Configure().Apply</c> is recorded — the
/// callee chain is not re-entered, so <c>Settings.Load</c> is missed.</description></item>
/// <item><description>A bare object creation with no trailing member access (<c>new Foo(...);</c> as a whole
/// statement) produces no fingerprint.</description></item>
/// <item><description>Bodies nested deeper than the walker's depth guard are abandoned mid-walk.</description></item>
/// </list>
///
/// Threading: pure Roslyn + BCL — no Unity API calls — so <see cref="Analyze"/> is safe to run
/// off the main thread. The telemetry writer relies on this and runs it on a background task
/// from a snapshot captured at eval completion.
///
/// This is a SEPARATE parse from the one <c>EvalCodeCompiler</c> does to actually compile and
/// run the eval, by necessity rather than oversight: that parse is of a generated wrapper
/// source (the raw body embedded inside a <c>class PipelineEval_&lt;id&gt; { ... Execute() {
/// ... } }</c>) under <c>SourceCodeKind.Regular</c> with the project's <c>#if</c> defines, not
/// of the raw body alone as a top-level <c>SourceCodeKind.Script</c> compilation unit the way
/// <see cref="Analyze"/> expects it. Sharing that tree would mean threading it out of
/// <c>RoslynCompilationService</c> (on both the success AND failure path — failures still get
/// recorded) and locating the wrapped <c>Execute()</c> body inside it, coupling this class to
/// the wrapper's exact shape. Not worth it today: the extra parse already runs off the eval
/// response path (see Threading above), so it costs background CPU, not eval latency.
/// </summary>
static class EvalUsageFingerprinter
{
/// <summary>
/// Upper bound on fingerprints kept per record, so a pathological body can't bloat the log.
/// Truncation keeps the FIRST N access sites in tree (source) order, not the most frequent —
/// acceptable because bodies with more than 32 access sites are rare outliers.
/// </summary>
public const int MaxFingerprints = 32;
/// <summary>
/// Neutral stand-in for any expression not on the rendering whitelist (literals, interpolated
/// strings, arithmetic, lambdas, …). Guarantees no raw source text leaks into a fingerprint.
/// </summary>
public const string Placeholder = "<expr>";
#if UNITY_EDITOR || (UNITY_STANDALONE && DEBUG)
private static readonly CSharpParseOptions m_ScriptOptions =
new CSharpParseOptions(kind: SourceCodeKind.Script);
/// <summary>
/// Analyse an eval body: classify it and extract its API fingerprints. Never throws — a body
/// that fails to parse yields an empty fingerprint list and a best-effort classification.
/// </summary>
public static EvalUsageAnalysis Analyze(string code)
{
if (string.IsNullOrWhiteSpace(code))
return new EvalUsageAnalysis { Classification = EvalClassification.Statements, Fingerprints = new List<string>() };
CompilationUnitSyntax root;
try
{
root = CSharpSyntaxTree.ParseText(code, m_ScriptOptions).GetCompilationUnitRoot();
}
catch
{
return new EvalUsageAnalysis { Classification = EvalClassification.Statements, Fingerprints = new List<string>() };
}
return new EvalUsageAnalysis
{
Classification = Classify(root),
Fingerprints = ExtractFingerprints(root)
};
}
/// <summary>Convenience overload used by tests: fingerprints only.</summary>
public static IReadOnlyList<string> ExtractFingerprints(string code) => Analyze(code).Fingerprints;
/// <summary>Convenience overload used by tests: classification only.</summary>
public static string Classify(string code) => Analyze(code).Classification;
/// <summary>
/// One-liner iff the body is exactly one statement that is an expression or a value-returning
/// return. A lone loop/if/block (e.g. a polling loop) is a statement body, not a one-liner.
/// </summary>
private static string Classify(CompilationUnitSyntax root)
{
var statements = root.Members
.OfType<GlobalStatementSyntax>()
.Select(m => m.Statement)
.ToList();
if (statements.Count == 1)
{
var only = statements[0];
if (only is ExpressionStatementSyntax ||
(only is ReturnStatementSyntax ret && ret.Expression != null))
{
return EvalClassification.SingleExpression;
}
}
return EvalClassification.Statements;
}
private static IReadOnlyList<string> ExtractFingerprints(CompilationUnitSyntax root)
{
var walker = new FingerprintWalker();
walker.Visit(root);
// Take() keeps the first MaxFingerprints access sites in tree (source) order — see the
// MaxFingerprints doc for why that truncation policy is acceptable.
return walker.Fingerprints.Count > MaxFingerprints
? walker.Fingerprints.Take(MaxFingerprints).ToList()
: walker.Fingerprints;
}
/// <summary>
/// Walks the syntax tree collecting one fingerprint per top-level access site: an invocation's
/// callee (<c>A.B.C(...)</c> → <c>A.B.C</c>) or a member-access chain that is not itself the
/// head of an enclosing call/access (<c>A.Instance.State</c>). Nested access sites inside call
/// arguments are still collected (they are their own sites); the callee chain is not re-counted.
/// A depth counter abandons subtrees nested past <see cref="MaxDepth"/> so a pathological body
/// (machine-generated nesting) can't blow the stack — fingerprinting is best-effort.
/// </summary>
private sealed class FingerprintWalker : CSharpSyntaxWalker
{
/// <summary>Maximum syntax depth walked; deeper subtrees are abandoned.</summary>
private const int MaxDepth = 64;
public readonly List<string> Fingerprints = new List<string>();
private int m_Depth;
public override void Visit(SyntaxNode node)
{
if (m_Depth >= MaxDepth)
return; // Abandon this subtree — best-effort telemetry, never a liability.
m_Depth++;
try
{
base.Visit(node);
}
finally
{
m_Depth--;
}
}
public override void VisitInvocationExpression(InvocationExpressionSyntax node)
{
AddChainHeadFingerprint(node, node.Expression, () => RenderPath(node.Expression));
// Deliberately do NOT recurse into the callee chain (already captured above), but do
// walk the arguments so nested calls/reads passed as arguments are captured too.
VisitArgumentsIfPresent(node.ArgumentList);
}
public override void VisitMemberAccessExpression(MemberAccessExpressionSyntax node)
{
if (IsChainHead(node))
Add(RenderPath(node));
// Continue into the expression side so nested access sites are still discovered;
// inner chain members are guarded by IsChainHead and won't be double-counted.
base.VisitMemberAccessExpression(node);
}
public override void VisitElementAccessExpression(ElementAccessExpressionSyntax node)
{
// A bare element-access chain head (cache[key]) previously produced NO fingerprint
// at all — no member access for the walker to see — while the conditional form
// (cache?[key]) records "cache[]": the two must rank together, so bare element
// access on a member/name receiver is fingerprinted the same way.
AddChainHeadFingerprint(node, node.Expression, () => RenderPath(node));
// Walk the bracket arguments so nested reads/calls inside the indexer are captured.
VisitArgumentsIfPresent(node.ArgumentList);
}
public override void VisitConditionalAccessExpression(ConditionalAccessExpressionSyntax node)
{
// The whole ?. chain is ONE fingerprint (RenderPath normalizes '?.' to '.'):
// descending into its parts would record the receiver and the bound members as
// separate (and misleading) entries.
if (IsChainHead(node))
Add(RenderPath(node));
// Parity with VisitInvocationExpression/VisitElementAccessExpression: only the head
// access's OWN argument list is walked, scoped to WhenNotNull (not the receiver
// before '?.') — the receiver's nested-call arguments are the same documented
// blind spot the unconditional forms accept, not something '?.' should uncover.
// Only the outermost argument lists are visited — the walker recurses within.
// Covers both parenthesized (ArgumentListSyntax) and bracketed
// (BracketedArgumentListSyntax) forms, so an indexer inside a conditional chain
// (a?.b[SomeCall()]) is captured too — a bracketed list was previously invisible
// to this sweep, silently dropping SomeCall.
if (node.WhenNotNull != null)
{
foreach (var argumentList in node.WhenNotNull
.DescendantNodes(n => !(n is ArgumentListSyntax) && !(n is BracketedArgumentListSyntax))
.Where(n => n is ArgumentListSyntax || n is BracketedArgumentListSyntax))
Visit(argumentList);
}
}
// Shared shape between VisitInvocationExpression and VisitElementAccessExpression: add
// the rendered path only for a chain head whose receiver is renderable, computing the
// path lazily so non-chain-head nodes (the overwhelming majority visited) skip RenderPath
// entirely instead of just discarding its result.
private void AddChainHeadFingerprint(SyntaxNode node, ExpressionSyntax receiver, System.Func<string> render)
{
if (IsChainHead(node) && IsMemberOrName(receiver))
Add(render());
}
private void VisitArgumentsIfPresent(BaseArgumentListSyntax argumentList)
{
if (argumentList != null)
Visit(argumentList);
}
private void Add(string path)
{
// A path that is nothing but the neutral placeholder carries no API information —
// drop the fingerprint rather than pollute the ranking with "<expr>" noise.
if (string.IsNullOrEmpty(path) || path == Placeholder)
return;
Fingerprints.Add(path);
}
// A node is the head of its access/call chain when its parent is not another link of the
// same chain (member access, invocation, or element access).
private static bool IsChainHead(SyntaxNode node)
{
var parent = node.Parent;
// See through parentheses: in (a?.b).c the conditional is part of the enclosing
// chain, not a head of its own.
while (parent is ParenthesizedExpressionSyntax paren)
parent = paren.Parent;
return !(parent is MemberAccessExpressionSyntax)
&& !(parent is InvocationExpressionSyntax)
&& !(parent is ElementAccessExpressionSyntax);
}
// Also accepts a call or parenthesized expression as the receiver: RenderPath already
// knows how to render both (GetHandler()() -> "GetHandler()", (A.B)() -> "A.B"), so
// gating them out here was dropping the whole access site instead of rendering it —
// not even the placeholder, since Add() was never reached.
private static bool IsMemberOrName(ExpressionSyntax expr) =>
expr is MemberAccessExpressionSyntax || expr is IdentifierNameSyntax || expr is GenericNameSyntax
|| expr is InvocationExpressionSyntax || expr is ParenthesizedExpressionSyntax;
}
/// <summary>
/// Render an expression to its fingerprint path: dotted identifiers, with generic method type
/// arguments normalised to arity placeholders (<c>&lt;T&gt;</c>, <c>&lt;T1, T2&gt;</c>) so
/// <c>FindFirstObjectByType&lt;Foo&gt;</c> and <c>&lt;Bar&gt;</c> rank as one API.
///
/// STRICT WHITELIST (the privacy contract of the class doc): only the expression forms listed
/// below render source text, and each renders identifiers/type/member names exclusively.
/// Everything else — literals, interpolated strings, arithmetic, lambdas, conditionals, … —
/// falls through to the neutral <see cref="Placeholder"/> so no raw source can ever leak into
/// a fingerprint. Casts unwrap to the inner expression; object creations render as
/// <c>new TypeName</c>. Rendering recursion is bounded by <paramref name="depth"/> as a second
/// layer of defence against pathological nesting (chains hang below their head, so they are
/// not covered by the walker's descent guard).
/// </summary>
private static string RenderPath(ExpressionSyntax expr) => RenderPath(expr, 0);
private const int MaxRenderDepth = 64;
private static string RenderPath(ExpressionSyntax expr, int depth)
{
if (depth >= MaxRenderDepth)
return Placeholder;
depth++;
switch (expr)
{
case IdentifierNameSyntax id:
return id.Identifier.Text;
case GenericNameSyntax gn:
return gn.Identifier.Text + RenderArity(gn.TypeArgumentList.Arguments.Count);
case QualifiedNameSyntax qn: // Namespace-qualified type in type position, e.g. new UnityEngine.GameObject
return RenderPath(qn.Left, depth) + "." + RenderSimpleName(qn.Right);
case AliasQualifiedNameSyntax aq: // global::UnityEngine → keep the name, drop the alias
return RenderSimpleName(aq.Name);
case MemberAccessExpressionSyntax ma:
return RenderPath(ma.Expression, depth) + "." + RenderSimpleName(ma.Name);
case InvocationExpressionSyntax inv:
return RenderPath(inv.Expression, depth) + "()";
case ElementAccessExpressionSyntax ea:
return RenderPath(ea.Expression, depth) + "[]";
case ParenthesizedExpressionSyntax pe:
return RenderPath(pe.Expression, depth);
case CastExpressionSyntax ce: // ((Foo)o).Bar → o.Bar: unwrap to the casted expression
return RenderPath(ce.Expression, depth);
case ObjectCreationExpressionSyntax oc: // new GameObject("x").AddComponent<T> → new GameObject.AddComponent<T>
return "new " + RenderPath(oc.Type, depth);
case PredefinedTypeSyntax pt:
return pt.Keyword.Text;
case ThisExpressionSyntax _:
return "this";
case BaseExpressionSyntax _:
return "base";
case ConditionalAccessExpressionSyntax cond:
// a?.B.C renders as a.B.C: '?.' is null-propagation noise for ranking
// purposes. Rendering only the bound member (the old fall-through via the bare
// MemberBinding) silently dropped the receiver, producing a colliding entry
// ("State.Value" for MatchManager.Instance?.State.Value). When the chain ends
// in a call, fingerprint the CALLEE (a?.B() -> a.B) so it ranks together with
// its unconditional form — the walker uses the same callee convention for
// plain invocation heads.
var conditionalTail = cond.WhenNotNull is InvocationExpressionSyntax topCall
? topCall.Expression
: cond.WhenNotNull;
var tailRendered = RenderPath(conditionalTail, depth);
// An element-binding tail (cache?[key] -> "[]") concatenates WITHOUT a dot so
// the fingerprint matches the unconditional cache[key] form ("cache[]").
return tailRendered.Length > 0 && tailRendered[0] == '['
? RenderPath(cond.Expression, depth) + tailRendered
: RenderPath(cond.Expression, depth) + "." + tailRendered;
case MemberBindingExpressionSyntax mb:
return RenderSimpleName(mb.Name);
case ElementBindingExpressionSyntax _: // the ?[...] binding of a conditional chain
return "[]";
default:
// NOT a raw ToString(): anything off the whitelist (string/char/numeric literals,
// interpolated strings, lambdas, arithmetic, …) is reduced to the neutral
// placeholder so source text can never leak into the telemetry log (AUTHAPI-29).
return Placeholder;
}
}
private static string RenderSimpleName(SimpleNameSyntax name)
{
if (name is GenericNameSyntax gn)
return gn.Identifier.Text + RenderArity(gn.TypeArgumentList.Arguments.Count);
return name.Identifier.Text;
}
private static string RenderArity(int count)
{
if (count <= 0)
return string.Empty;
if (count == 1)
return "<T>";
var sb = new StringBuilder("<");
for (var i = 1; i <= count; i++)
{
if (i > 1)
sb.Append(", ");
sb.Append("T").Append(i);
}
sb.Append(">");
return sb.ToString();
}
#else
/// <summary>Runtime compilation (and therefore eval) is unavailable on this platform.</summary>
public static EvalUsageAnalysis Analyze(string code) =>
new EvalUsageAnalysis { Classification = EvalClassification.Statements, Fingerprints = new List<string>() };
public static IReadOnlyList<string> ExtractFingerprints(string code) => new List<string>();
public static string Classify(string code) => EvalClassification.Statements;
#endif
}
}
AIClient/Packages/com.unity.pipeline/Runtime/Telemetry/EvalUsageFingerprinter.cs.meta
+11
View File
@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: a08e235068524a4aa303c035a80f9008
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:
AIClient/Packages/com.unity.pipeline/Runtime/Telemetry/EvalUsageReport.cs
+266
View File
@@ -0,0 +1,266 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Newtonsoft.Json;
namespace Unity.Pipeline.Telemetry
{
/// <summary>A ranked API fingerprint with its occurrence count across all recorded evals.</summary>
[Serializable]
class FingerprintCount
{
[JsonProperty("member")]
public string Member { get; set; }
[JsonProperty("count")]
public int Count { get; set; }
}
/// <summary>An eval pattern already served by an existing catalog command — a "stop using eval" hit.</summary>
[Serializable]
class CoverageHit
{
[JsonProperty("member")]
public string Member { get; set; }
[JsonProperty("count")]
public int Count { get; set; }
/// <summary>The existing command that already covers this pattern.</summary>
[JsonProperty("command")]
public string Command { get; set; }
}
/// <summary>A frequently-eval'd pattern with no covering command — a candidate for a new typed command.</summary>
[Serializable]
class CoverageGap
{
[JsonProperty("member")]
public string Member { get; set; }
[JsonProperty("count")]
public int Count { get; set; }
/// <summary>A proposed command name from the coverage map, when one exists but is not yet built.</summary>
[JsonProperty("suggestedCommand", NullValueHandling = NullValueHandling.Ignore)]
public string SuggestedCommand { get; set; }
}
/// <summary>
/// Coverage cross-reference between recorded eval fingerprints and the live command catalog:
/// which patterns are already coverable (habit/eval-avoidable) and which high-frequency patterns
/// are still uncovered (the next commands to build — feeds AUTHAPI-17 / AUTHAPI-24).
/// </summary>
[Serializable]
class EvalCoverageSuggestions
{
[JsonProperty("covered")]
public List<CoverageHit> Covered { get; set; } = new List<CoverageHit>();
[JsonProperty("topUncovered")]
public List<CoverageGap> TopUncovered { get; set; } = new List<CoverageGap>();
}
/// <summary>
/// Aggregated eval-usage report: ranked API fingerprints, one-liner vs multi-statement split,
/// error rate, and coverage suggestions. This is the artifact that turns eval-displacement backlog
/// triage into a lookup (AUTHAPI-24 / AUTHAPI-29).
/// </summary>
[Serializable]
class EvalUsageReport
{
[JsonProperty("totalCount")]
public int TotalCount { get; set; }
[JsonProperty("successCount")]
public int SuccessCount { get; set; }
[JsonProperty("errorCount")]
public int ErrorCount { get; set; }
/// <summary>Fraction of recorded evals that failed, 0..1.</summary>
[JsonProperty("errorRate")]
public double ErrorRate { get; set; }
[JsonProperty("singleExpressionCount")]
public int SingleExpressionCount { get; set; }
[JsonProperty("statementsCount")]
public int StatementsCount { get; set; }
/// <summary>Percentage of evals classified as one-liners (single-expression), 0..100.</summary>
[JsonProperty("oneLinerPercentage")]
public double OneLinerPercentage { get; set; }
/// <summary>
/// Total number of DISTINCT fingerprints observed across all records, before the ranked
/// <see cref="Fingerprints"/> list is capped — so truncation is visible
/// (<c>distinctFingerprints &gt; fingerprints.Count</c> means the tail was cut).
/// </summary>
[JsonProperty("distinctFingerprints")]
public int DistinctFingerprints { get; set; }
/// <summary>
/// API fingerprints ranked by descending occurrence count. Capped (see
/// <see cref="EvalUsageAggregator.Build"/>); <see cref="DistinctFingerprints"/> carries the
/// uncapped total.
/// </summary>
[JsonProperty("fingerprints")]
public List<FingerprintCount> Fingerprints { get; set; } = new List<FingerprintCount>();
[JsonProperty("coverageSuggestions")]
public EvalCoverageSuggestions CoverageSuggestions { get; set; } = new EvalCoverageSuggestions();
}
/// <summary>
/// Curated, heuristic map from an API fingerprint to the command name that would cover it. Truth
/// about whether that command actually exists comes from the live catalog at report time, so a
/// mapped-but-absent command surfaces as a concrete gap ("build <c>refresh_assets</c>"). Kept
/// deliberately small and extensible — this is a hint layer, not an exhaustive registry.
/// </summary>
static class EvalCoverageMap
{
public static readonly IReadOnlyDictionary<string, string> Default = new Dictionary<string, string>
{
// The habit case: a command already exists, so these should stop using eval.
{ "PlayerSettings.SetScriptingBackend", "set_player_settings" },
{ "PlayerSettings.GetScriptingBackend", "get_player_settings" },
// Known gaps from the WizardDuelVR evidence: no command yet (feeds AUTHAPI-17).
// NOTE: save_all is NOT a cover for AssetDatabase.SaveAssets — it saves open scenes
// (EditorSceneManager.SaveOpenScenes + AssetDatabase.Refresh) and never persists dirty
// non-scene assets; steering agents off that eval to save_all would silently drop those
// writes. Mapped to a proposed save_assets command so the report surfaces it as a gap.
{ "AssetDatabase.SaveAssets", "save_assets" },
{ "AssetDatabase.Refresh", "refresh_assets" },
};
}
/// <summary>
/// Builds an <see cref="EvalUsageReport"/> from recorded eval telemetry, cross-referencing the API
/// fingerprints against the set of available command names.
/// </summary>
static class EvalUsageAggregator
{
/// <summary>Default cap on the suggested-gaps (<c>topUncovered</c>) list.</summary>
public const int DefaultTopUncoveredLimit = 20;
/// <summary>Default cap on the ranked <see cref="EvalUsageReport.Fingerprints"/> list.</summary>
public const int DefaultFingerprintLimit = 50;
/// <summary>
/// Aggregate <paramref name="records"/> into a report. <paramref name="availableCommands"/> is
/// the set of command names in the live catalog; <paramref name="coverageMap"/> defaults to
/// <see cref="EvalCoverageMap.Default"/>. <paramref name="topUncoveredLimit"/> caps the
/// suggested-gaps list and <paramref name="fingerprintLimit"/> caps the ranked fingerprint
/// list (<see cref="EvalUsageReport.DistinctFingerprints"/> makes any truncation visible;
/// coverage is computed from the FULL ranking, so the fingerprint cap never skews it).
/// Accepted range for both limits: 0 or greater — negative values fall back to the defaults
/// (0 legitimately empties the list).
/// </summary>
public static EvalUsageReport Build(
IReadOnlyList<EvalUsageRecord> records,
ISet<string> availableCommands,
IReadOnlyDictionary<string, string> coverageMap = null,
int topUncoveredLimit = DefaultTopUncoveredLimit,
int fingerprintLimit = DefaultFingerprintLimit)
{
records = records ?? new List<EvalUsageRecord>();
availableCommands = availableCommands ?? new HashSet<string>();
coverageMap = coverageMap ?? EvalCoverageMap.Default;
if (topUncoveredLimit < 0)
topUncoveredLimit = DefaultTopUncoveredLimit;
if (fingerprintLimit < 0)
fingerprintLimit = DefaultFingerprintLimit;
var report = new EvalUsageReport { TotalCount = records.Count };
var counts = new Dictionary<string, int>();
foreach (var record in records)
{
if (record.Success)
report.SuccessCount++;
else
report.ErrorCount++;
if (record.Classification == EvalClassification.SingleExpression)
report.SingleExpressionCount++;
else
report.StatementsCount++;
if (record.Fingerprints == null)
continue;
foreach (var fingerprint in record.Fingerprints)
{
if (string.IsNullOrEmpty(fingerprint))
continue;
counts.TryGetValue(fingerprint, out var current);
counts[fingerprint] = current + 1;
}
}
if (report.TotalCount > 0)
{
report.ErrorRate = (double)report.ErrorCount / report.TotalCount;
report.OneLinerPercentage = 100.0 * report.SingleExpressionCount / report.TotalCount;
}
// Deterministic ranking: count desc, then member name asc as a tiebreak.
var ranked = counts
.Select(kv => new FingerprintCount { Member = kv.Key, Count = kv.Value })
.OrderByDescending(f => f.Count)
.ThenBy(f => f.Member, StringComparer.Ordinal)
.ToList();
// Cap the reported ranking (an accumulated log can hold thousands of distinct
// fingerprints); DistinctFingerprints preserves the uncapped total so truncation shows.
report.DistinctFingerprints = ranked.Count;
report.Fingerprints = ranked.Count > fingerprintLimit ? ranked.Take(fingerprintLimit).ToList() : ranked;
// Coverage cross-references the FULL ranking, so a tight fingerprint cap cannot hide a
// covered pattern or demote an uncovered one; topUncovered has its own cap.
report.CoverageSuggestions = BuildCoverage(ranked, availableCommands, coverageMap, topUncoveredLimit);
return report;
}
private static EvalCoverageSuggestions BuildCoverage(
List<FingerprintCount> ranked,
ISet<string> availableCommands,
IReadOnlyDictionary<string, string> coverageMap,
int topUncoveredLimit)
{
var suggestions = new EvalCoverageSuggestions();
foreach (var entry in ranked)
{
coverageMap.TryGetValue(entry.Member, out var mappedCommand);
var covered = mappedCommand != null && availableCommands.Contains(mappedCommand);
if (covered)
{
suggestions.Covered.Add(new CoverageHit
{
Member = entry.Member,
Count = entry.Count,
Command = mappedCommand
});
}
else
{
suggestions.TopUncovered.Add(new CoverageGap
{
Member = entry.Member,
Count = entry.Count,
// A mapped-but-absent command is a concrete proposal; no mapping leaves it null.
SuggestedCommand = mappedCommand
});
}
}
// Negative limits were normalised to the default in Build, so a plain cap suffices.
if (suggestions.TopUncovered.Count > topUncoveredLimit)
suggestions.TopUncovered = suggestions.TopUncovered.Take(topUncoveredLimit).ToList();
return suggestions;
}
}
}
AIClient/Packages/com.unity.pipeline/Runtime/Telemetry/EvalUsageReport.cs.meta
+11
View File
@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: ffdbe00cca0b4defb784b5c92af3bdf2
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:
AIClient/Packages/com.unity.pipeline/Runtime/Telemetry/EvalUsageTelemetry.cs
+481
View File
@@ -0,0 +1,481 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using Newtonsoft.Json;
using UnityEngine;
#if UNITY_EDITOR
using System.Threading.Tasks;
#endif
namespace Unity.Pipeline.Telemetry
{
/// <summary>
/// One appended line of eval-usage telemetry. Serialized as a single compact JSON object per line
/// (JSONL). The raw <see cref="Source"/> is omitted unless the local <see cref="EvalUsageTelemetry.StoreSource"/>
/// opt-in is set, so by default only shape/fingerprint data is persisted (AUTHAPI-29).
/// </summary>
[Serializable]
class EvalUsageRecord
{
[JsonProperty("time")]
public string Time { get; set; }
/// <summary>The command that ran the source: "eval" or "eval_file".</summary>
[JsonProperty("command")]
public string Command { get; set; }
[JsonProperty("success")]
public bool Success { get; set; }
/// <summary>Short error category on failure (e.g. "Compilation Failed"); omitted on success.</summary>
[JsonProperty("error", NullValueHandling = NullValueHandling.Ignore)]
public string Error { get; set; }
/// <summary>Character length of the eval body.</summary>
[JsonProperty("payloadLength")]
public int PayloadLength { get; set; }
/// <summary>Newline-delimited line count of the eval body.</summary>
[JsonProperty("lineCount")]
public int LineCount { get; set; }
[JsonProperty("executionTimeMs")]
public long ExecutionTimeMs { get; set; }
/// <summary>"single-expression" | "statements" — see <see cref="EvalClassification"/>.</summary>
[JsonProperty("classification")]
public string Classification { get; set; }
/// <summary>Top-level API member-access paths extracted from the syntax tree.</summary>
[JsonProperty("fingerprints")]
public List<string> Fingerprints { get; set; } = new List<string>();
/// <summary>Raw eval source. Present ONLY when the explicit opt-in is set; omitted otherwise.</summary>
[JsonProperty("source", NullValueHandling = NullValueHandling.Ignore)]
public string Source { get; set; }
}
/// <summary>
/// Local-first, privacy-first instrumentation of <c>eval</c>/<c>eval_file</c> (AUTHAPI-29). Per
/// invocation it appends one <see cref="EvalUsageRecord"/> to
/// <c>&lt;project&gt;/Library/Pipeline/eval-usage.jsonl</c>. There is NO off-machine transmission:
/// disabling <see cref="Enabled"/> stops even local recording, which satisfies "analytics opt-out
/// fully disables any off-machine transmission" trivially — nothing ever leaves the machine.
///
/// RECORDING IS EDITOR-ONLY. Eval telemetry exists to steer the editor command-surface migration,
/// so <see cref="Record"/>/<see cref="RecordInBackground"/> compile to no-ops outside the editor
/// (<c>#if UNITY_EDITOR</c>). This also keeps player builds from writing a "Library/Pipeline"
/// folder next to <c>Application.dataPath</c> (the APK path on Android, inside the .app bundle on
/// macOS) — a location players may not even be able to write. Release players cannot eval at all
/// (the compiler stub blocks it); development players can, but are deliberately not instrumented.
/// The class itself stays compiled in players so shared code links, and the read side returns
/// empty results there.
///
/// The production hook is fire-and-forget: <see cref="RecordInBackground"/> snapshots the eval
/// outcome plus the current configuration on the calling (main) thread and runs the Roslyn parse,
/// fingerprint extraction, and file append on a thread-pool task, so recording adds no latency to
/// the eval response.
///
/// Modelled on <c>PipelineTransactionLog</c>: a lock serialises the append (evals can overlap on
/// the /api/progress path) and the reader takes the same lock, all I/O is best-effort and swallows
/// errors so telemetry never breaks a command, and a directory-parameterised seam makes it
/// hermetically testable. Unlike the debug transaction log, the JSONL is meant to ACCUMULATE
/// across sessions so the ranked report has data, so it is bounded by a byte-size cap with
/// rotation rather than reset per session; <see cref="ReadRecords"/> merges the rotated ".old"
/// file back in, bounding readable history to roughly 2× <see cref="MaxFileBytes"/>.
/// </summary>
static class EvalUsageTelemetry
{
/// <summary>
/// Byte-size cap for the active JSONL before it rotates to the ".old" file (default 5 MB).
/// A settable field (not a const) so tests can force the size-cap rotation path cheaply.
/// </summary>
public static long MaxFileBytes = 5L * 1024 * 1024;
/// <summary>
/// Upper bound, in characters, on the raw source persisted per record when the
/// <see cref="StoreSource"/> opt-in is set (64 K). Longer bodies are cut at this bound and
/// <see cref="SourceTruncationMarker"/> is appended so truncation is explicit;
/// <see cref="EvalUsageRecord.PayloadLength"/> still reports the full original length.
/// </summary>
public const int MaxStoredSourceChars = 64 * 1024;
/// <summary>Marker appended to a stored <see cref="EvalUsageRecord.Source"/> that was truncated.</summary>
public const string SourceTruncationMarker = "…[truncated]";
private const string FileName = "eval-usage.jsonl";
private const string OldFileName = "eval-usage.old.jsonl";
/// <summary>Master switch. When false, <see cref="Record"/> is a no-op (no local write, no transmission).</summary>
public static bool Enabled = true;
/// <summary>Opt-in: persist the raw eval source in each record. Off by default (privacy-first).</summary>
public static bool StoreSource = false;
/// <summary>
/// Overrides the storage directory. Null uses <c>&lt;project&gt;/Library/Pipeline</c>. Tests set
/// this to a temp directory so they never touch the real project telemetry file.
/// </summary>
public static string OverrideDirectory;
private static readonly object s_AppendGate = new object();
/// <summary>
/// Number of background records queued by <see cref="RecordInBackground"/> and not yet
/// flushed. Explicitly initialised: in player builds the recording path compiles away and the
/// field is only ever read (by <see cref="WaitForPendingRecords"/>), which would otherwise
/// raise CS0649.
/// </summary>
private static int s_PendingRecords = 0;
#if UNITY_EDITOR
/// <summary>Serialises background writes in submission order (see <see cref="RecordInBackground"/>).
/// Editor-only like the recording path itself: Task is not even imported in player builds.</summary>
private static readonly object s_WriteChainGate = new object();
private static Task s_WriteChain = Task.CompletedTask;
#endif
/// <summary>Main-thread-resolved cache of the default (dataPath-derived) storage directory.</summary>
private static string s_CachedDefaultDirectory;
/// <summary>
/// Record one eval invocation synchronously on the calling thread. Editor-only: compiles to a
/// no-op in player builds (see class doc). Best-effort: any failure (parse, I/O) is swallowed
/// with a warning so instrumentation can never break the eval it is measuring. The production
/// eval hook uses <see cref="RecordInBackground"/> so the eval response is never delayed; this
/// synchronous seam remains for tests and direct callers that need the write completed on
/// return.
/// </summary>
public static void Record(string command, string code, bool success, string error, long executionTimeMs)
{
#if UNITY_EDITOR
if (!Enabled)
return;
TryWriteRecord(ResolveDirectory(), DateTime.UtcNow, command, code, success, error, executionTimeMs, StoreSource);
#endif
}
/// <summary>
/// Record one eval invocation without blocking the calling thread. Editor-only: compiles to a
/// no-op in player builds (see class doc). Everything the write needs — the eval primitives
/// plus a snapshot of the current <see cref="Enabled"/>/<see cref="StoreSource"/> configuration
/// and the resolved directory (which touches Unity API and must be read on the main thread) —
/// is captured here on the calling thread; the Roslyn parse, fingerprint extraction, and
/// lock-serialised append then run on a thread-pool task, so telemetry adds zero latency to
/// the eval response. Failures on the background task are swallowed with a warning
/// (<c>Debug.LogWarning</c> is thread-safe). A domain reload can at worst abandon one in-flight
/// record; its partial trailing line is skipped by <see cref="ReadRecords"/>.
/// </summary>
public static void RecordInBackground(string command, string code, bool success, string error, long executionTimeMs)
{
#if UNITY_EDITOR
if (!Enabled)
return;
// Snapshot on the calling thread: config statics can be flipped live by the inspector,
// and ResolveDirectory may touch Application.dataPath (main-thread-only Unity API).
var directory = ResolveDirectory();
var storeSource = StoreSource;
var timeUtc = DateTime.UtcNow;
Interlocked.Increment(ref s_PendingRecords);
// Chain onto the previous write instead of racing independent Task.Runs: back-to-back
// evals (batch, tight agent loops) must land in submission order — ReadRecords
// documents the log as chronological — and the chain also means at most one telemetry
// write competes with the pool at a time.
lock (s_WriteChainGate)
{
s_WriteChain = s_WriteChain.ContinueWith(_ =>
{
try
{
TryWriteRecord(directory, timeUtc, command, code, success, error, executionTimeMs, storeSource);
}
finally
{
Interlocked.Decrement(ref s_PendingRecords);
}
}, System.Threading.CancellationToken.None, TaskContinuationOptions.None, TaskScheduler.Default);
}
#endif
}
/// <summary>
/// Block until every background record queued by <see cref="RecordInBackground"/> has been
/// flushed, or <paramref name="timeoutMs"/> elapses (returns false on timeout). Recording is
/// fire-and-forget, so tests (and teardown paths that delete the storage directory) drain the
/// queue with this before asserting on / removing the JSONL.
/// </summary>
public static bool WaitForPendingRecords(int timeoutMs = 5000)
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
while (Volatile.Read(ref s_PendingRecords) > 0)
{
if (stopwatch.ElapsedMilliseconds >= timeoutMs)
return false;
Thread.Sleep(1);
}
return true;
}
#if UNITY_EDITOR
/// <summary>
/// <see cref="WriteRecord"/>, with the shared best-effort guard: telemetry must never affect
/// eval behavior (or crash a background task), so any failure (parse, I/O) is swallowed with
/// a warning instead of propagating. Shared by <see cref="Record"/>'s synchronous call and
/// <see cref="RecordInBackground"/>'s queued continuation.
/// </summary>
private static void TryWriteRecord(
string directory,
DateTime timeUtc,
string command,
string code,
bool success,
string error,
long executionTimeMs,
bool storeSource)
{
try
{
WriteRecord(directory, timeUtc, command, code, success, error, executionTimeMs, storeSource);
}
catch (Exception ex)
{
Debug.LogWarning($"Pipeline eval telemetry write failed: {ex.Message}");
}
}
/// <summary>
/// Build and append one record from an already-captured snapshot. Runs on whichever thread the
/// caller chose; deliberately touches no Unity API (the fingerprinter is pure Roslyn and the
/// directory was resolved by the caller), so it is safe on a thread-pool task.
/// </summary>
private static void WriteRecord(
string directory,
DateTime timeUtc,
string command,
string code,
bool success,
string error,
long executionTimeMs,
bool storeSource)
{
var analysis = EvalUsageFingerprinter.Analyze(code);
var record = new EvalUsageRecord
{
Time = timeUtc.ToString("o"),
Command = command,
Success = success,
Error = success ? null : error,
PayloadLength = code?.Length ?? 0,
LineCount = CountLines(code),
ExecutionTimeMs = executionTimeMs,
Classification = analysis.Classification,
Fingerprints = new List<string>(analysis.Fingerprints),
Source = storeSource ? BoundStoredSource(code) : null
};
Append(directory, record);
}
/// <summary>
/// Bound the opt-in raw source at <see cref="MaxStoredSourceChars"/>, appending
/// <see cref="SourceTruncationMarker"/> when cut, so one giant eval_file body can't balloon
/// a single record (the JSONL size cap only rotates between appends).
/// </summary>
private static string BoundStoredSource(string code)
{
if (string.IsNullOrEmpty(code) || code.Length <= MaxStoredSourceChars)
return code;
return code.Substring(0, MaxStoredSourceChars) + SourceTruncationMarker;
}
#endif
/// <summary>
/// Append one record to the JSONL in an explicit directory. Testable seam behind
/// <see cref="Record"/>. Rotates first if the active file has reached the size cap, so each
/// file stays bounded to roughly <see cref="MaxFileBytes"/> plus one record.
/// </summary>
public static void Append(string directory, EvalUsageRecord record)
{
lock (s_AppendGate)
{
var root = Path.GetFullPath(directory);
Directory.CreateDirectory(root);
var path = DataFilePath(root, FileName);
if (File.Exists(path) && new FileInfo(path).Length >= MaxFileBytes)
Rotate(root);
File.AppendAllText(path, JsonConvert.SerializeObject(record, Formatting.None) + "\n");
}
}
/// <summary>
/// Canonical absolute path of one of this class's fixed data files inside the telemetry
/// directory, with containment re-checked after normalization. The directory is the only
/// dynamic path segment (a local editor setting or the test override — never protocol
/// input) and the file names are compile-time constants, so this is an explicit invariant
/// at the IO boundary: any input that would resolve outside its canonical directory is
/// rejected instead of opened.
/// </summary>
private static string DataFilePath(string directory, string fileName)
{
if (!TryResolveDataFilePath(directory, fileName, out var full))
throw new InvalidOperationException(
$"Telemetry data file path '{full}' escapes its directory '{directory}'.");
return full;
}
/// <summary>
/// The single canonicalize-and-contain implementation behind both the throwing
/// <see cref="DataFilePath"/> (writer sites) and the silently-skipping reader — one place
/// to fix normalization, per review.
/// </summary>
private static bool TryResolveDataFilePath(string directory, string fileName, out string fullPath)
{
var root = Path.GetFullPath(directory);
fullPath = Path.GetFullPath(Path.Combine(root, fileName));
var rootWithSeparator = root.EndsWith(Path.DirectorySeparatorChar.ToString(), StringComparison.Ordinal)
? root
: root + Path.DirectorySeparatorChar;
return fullPath.StartsWith(rootWithSeparator, StringComparison.Ordinal);
}
/// <summary>
/// Move the active JSONL to the ".old" backup, replacing any prior backup, so the active file
/// starts fresh. No-op when there is no active file. Exposed as the testable rotation seam.
/// Takes the append gate so a rotation can never interleave with a concurrent append or read
/// (the lock is re-entrant, so the rotate-inside-<see cref="Append"/> path is unaffected).
/// </summary>
public static void Rotate(string directory)
{
lock (s_AppendGate)
{
var root = Path.GetFullPath(directory);
Directory.CreateDirectory(root);
var path = DataFilePath(root, FileName);
var oldPath = DataFilePath(root, OldFileName);
RotatingFileBackup.RotateToBackup(path, oldPath);
}
}
/// <summary>
/// Read back every record from <paramref name="directory"/>: the rotated
/// <c>eval-usage.old.jsonl</c> first (when present), then the active file, so the report sees
/// the full retained history — bounded to roughly 2× <see cref="MaxFileBytes"/> — in
/// chronological order. Malformed lines are skipped so a partial trailing write can never
/// break the report.
///
/// Concurrency: takes the same gate as <see cref="Append"/>/<see cref="Rotate"/> (this class
/// is the only writer, and appends now happen on background tasks), and additionally opens
/// each file tolerantly — <c>FileShare.ReadWrite</c>, with I/O errors caught — so a reader
/// racing an out-of-process writer, or a file swapped away between the existence check and the
/// open (the rotation TOCTOU), degrades to "return what parsed" instead of throwing.
/// </summary>
public static IReadOnlyList<EvalUsageRecord> ReadRecords(string directory)
{
var records = new List<EvalUsageRecord>();
lock (s_AppendGate)
{
ReadFileInto(directory, OldFileName, records);
ReadFileInto(directory, FileName, records);
}
return records;
}
/// <summary>
/// Best-effort JSONL parse of one fixed-name data file into <paramref name="records"/>; see
/// <see cref="ReadRecords"/>. The canonicalize-and-contain check is repeated inline here (not
/// only in <see cref="DataFilePath"/>) so the validated path and the open are in the same
/// method — the same invariant, kept visible at the IO site.
/// </summary>
private static void ReadFileInto(string directory, string fileName, List<EvalUsageRecord> records)
{
// Same canonicalize-and-contain invariant as the writer sites, via the one shared
// implementation; a reader skips silently where a writer throws.
if (!TryResolveDataFilePath(directory, fileName, out var path))
return; // outside the telemetry directory — unreachable for the constant file names
try
{
using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete))
using (var reader = new StreamReader(stream))
{
string line;
while ((line = reader.ReadLine()) != null)
{
if (string.IsNullOrWhiteSpace(line))
continue;
try
{
var record = JsonConvert.DeserializeObject<EvalUsageRecord>(line);
if (record != null)
records.Add(record);
}
catch (JsonException)
{
// Skip a malformed / partially written line; keep the rest of the log usable.
}
}
}
}
catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException)
{
// Covers FileNotFound/DirectoryNotFound (nothing recorded yet, or the file was rotated
// away mid-read), sharing violations from external processes, and ACL/AV denials
// (UnauthorizedAccessException does NOT derive from IOException): telemetry read-back
// is best-effort, so return whatever parsed instead of throwing.
}
}
/// <summary>
/// Resolve the storage directory: the override if set, else &lt;project&gt;/Library/Pipeline.
/// The default derives from <c>Application.dataPath</c> (a main-thread Unity API), so the
/// first non-overridden call must happen on the main thread — the editor startup path primes
/// it — after which the cached value is safe from any thread (e.g. the off-main-thread
/// <c>report_evals</c> command).
/// </summary>
public static string ResolveDirectory()
{
if (!string.IsNullOrEmpty(OverrideDirectory))
return OverrideDirectory;
if (s_CachedDefaultDirectory == null)
{
#if UNITY_EDITOR
var projectRoot = Path.GetDirectoryName(Application.dataPath);
s_CachedDefaultDirectory = Path.Combine(projectRoot, "Library", "Pipeline");
#else
// Player builds: recording is compiled out and nothing is ever written, but
// report_evals (MainThreadRequired=false) can reach this off the main thread
// before any main-thread code primed the cache — and Application.dataPath is
// main-thread-only. A stable process-temp location keeps the read path safe and
// empty, so the command returns the documented zeroed report instead of throwing.
s_CachedDefaultDirectory = Path.Combine(Path.GetTempPath(), "UnityPipelineEvalUsage");
#endif
}
return s_CachedDefaultDirectory;
}
private static int CountLines(string code)
{
if (string.IsNullOrEmpty(code))
return 0;
var lines = 1;
for (var i = 0; i < code.Length; i++)
{
var c = code[i];
if (c == '\n')
lines++;
else if (c == '\r' && (i + 1 >= code.Length || code[i + 1] != '\n'))
lines++; // lone \r (classic Mac) is a line break; \r\n already counts via \n
}
return lines;
}
}
}
AIClient/Packages/com.unity.pipeline/Runtime/Telemetry/EvalUsageTelemetry.cs.meta
+11
View File
@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: ae2e135fbc1d424592da87027c687d94
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant: