using System;
using System.Collections.Generic;
using System.Linq;
using Newtonsoft.Json;
namespace Unity.Pipeline.Telemetry
{
/// A ranked API fingerprint with its occurrence count across all recorded evals.
[Serializable]
class FingerprintCount
{
[JsonProperty("member")]
public string Member { get; set; }
[JsonProperty("count")]
public int Count { get; set; }
}
/// An eval pattern already served by an existing catalog command — a "stop using eval" hit.
[Serializable]
class CoverageHit
{
[JsonProperty("member")]
public string Member { get; set; }
[JsonProperty("count")]
public int Count { get; set; }
/// The existing command that already covers this pattern.
[JsonProperty("command")]
public string Command { get; set; }
}
/// A frequently-eval'd pattern with no covering command — a candidate for a new typed command.
[Serializable]
class CoverageGap
{
[JsonProperty("member")]
public string Member { get; set; }
[JsonProperty("count")]
public int Count { get; set; }
/// A proposed command name from the coverage map, when one exists but is not yet built.
[JsonProperty("suggestedCommand", NullValueHandling = NullValueHandling.Ignore)]
public string SuggestedCommand { get; set; }
}
///
/// Coverage cross-reference between recorded eval fingerprints and the live command catalog:
/// which patterns are already coverable (habit/eval-avoidable) and which high-frequency patterns
/// are still uncovered (the next commands to build — feeds AUTHAPI-17 / AUTHAPI-24).
///
[Serializable]
class EvalCoverageSuggestions
{
[JsonProperty("covered")]
public List Covered { get; set; } = new List();
[JsonProperty("topUncovered")]
public List TopUncovered { get; set; } = new List();
}
///
/// Aggregated eval-usage report: ranked API fingerprints, one-liner vs multi-statement split,
/// error rate, and coverage suggestions. This is the artifact that turns eval-displacement backlog
/// triage into a lookup (AUTHAPI-24 / AUTHAPI-29).
///
[Serializable]
class EvalUsageReport
{
[JsonProperty("totalCount")]
public int TotalCount { get; set; }
[JsonProperty("successCount")]
public int SuccessCount { get; set; }
[JsonProperty("errorCount")]
public int ErrorCount { get; set; }
/// Fraction of recorded evals that failed, 0..1.
[JsonProperty("errorRate")]
public double ErrorRate { get; set; }
[JsonProperty("singleExpressionCount")]
public int SingleExpressionCount { get; set; }
[JsonProperty("statementsCount")]
public int StatementsCount { get; set; }
/// Percentage of evals classified as one-liners (single-expression), 0..100.
[JsonProperty("oneLinerPercentage")]
public double OneLinerPercentage { get; set; }
///
/// Total number of DISTINCT fingerprints observed across all records, before the ranked
/// list is capped — so truncation is visible
/// (distinctFingerprints > fingerprints.Count means the tail was cut).
///
[JsonProperty("distinctFingerprints")]
public int DistinctFingerprints { get; set; }
///
/// API fingerprints ranked by descending occurrence count. Capped (see
/// ); carries the
/// uncapped total.
///
[JsonProperty("fingerprints")]
public List Fingerprints { get; set; } = new List();
[JsonProperty("coverageSuggestions")]
public EvalCoverageSuggestions CoverageSuggestions { get; set; } = new EvalCoverageSuggestions();
}
///
/// Curated, heuristic map from an API fingerprint to the command name that would cover it. Truth
/// about whether that command actually exists comes from the live catalog at report time, so a
/// mapped-but-absent command surfaces as a concrete gap ("build refresh_assets"). Kept
/// deliberately small and extensible — this is a hint layer, not an exhaustive registry.
///
static class EvalCoverageMap
{
public static readonly IReadOnlyDictionary Default = new Dictionary
{
// The habit case: a command already exists, so these should stop using eval.
{ "PlayerSettings.SetScriptingBackend", "set_player_settings" },
{ "PlayerSettings.GetScriptingBackend", "get_player_settings" },
// Known gaps from the WizardDuelVR evidence: no command yet (feeds AUTHAPI-17).
// NOTE: save_all is NOT a cover for AssetDatabase.SaveAssets — it saves open scenes
// (EditorSceneManager.SaveOpenScenes + AssetDatabase.Refresh) and never persists dirty
// non-scene assets; steering agents off that eval to save_all would silently drop those
// writes. Mapped to a proposed save_assets command so the report surfaces it as a gap.
{ "AssetDatabase.SaveAssets", "save_assets" },
{ "AssetDatabase.Refresh", "refresh_assets" },
};
}
///
/// Builds an from recorded eval telemetry, cross-referencing the API
/// fingerprints against the set of available command names.
///
static class EvalUsageAggregator
{
/// Default cap on the suggested-gaps (topUncovered) list.
public const int DefaultTopUncoveredLimit = 20;
/// Default cap on the ranked list.
public const int DefaultFingerprintLimit = 50;
///
/// Aggregate into a report. is
/// the set of command names in the live catalog; defaults to
/// . caps the
/// suggested-gaps list and caps the ranked fingerprint
/// list ( makes any truncation visible;
/// coverage is computed from the FULL ranking, so the fingerprint cap never skews it).
/// Accepted range for both limits: 0 or greater — negative values fall back to the defaults
/// (0 legitimately empties the list).
///
public static EvalUsageReport Build(
IReadOnlyList records,
ISet availableCommands,
IReadOnlyDictionary coverageMap = null,
int topUncoveredLimit = DefaultTopUncoveredLimit,
int fingerprintLimit = DefaultFingerprintLimit)
{
records = records ?? new List();
availableCommands = availableCommands ?? new HashSet();
coverageMap = coverageMap ?? EvalCoverageMap.Default;
if (topUncoveredLimit < 0)
topUncoveredLimit = DefaultTopUncoveredLimit;
if (fingerprintLimit < 0)
fingerprintLimit = DefaultFingerprintLimit;
var report = new EvalUsageReport { TotalCount = records.Count };
var counts = new Dictionary();
foreach (var record in records)
{
if (record.Success)
report.SuccessCount++;
else
report.ErrorCount++;
if (record.Classification == EvalClassification.SingleExpression)
report.SingleExpressionCount++;
else
report.StatementsCount++;
if (record.Fingerprints == null)
continue;
foreach (var fingerprint in record.Fingerprints)
{
if (string.IsNullOrEmpty(fingerprint))
continue;
counts.TryGetValue(fingerprint, out var current);
counts[fingerprint] = current + 1;
}
}
if (report.TotalCount > 0)
{
report.ErrorRate = (double)report.ErrorCount / report.TotalCount;
report.OneLinerPercentage = 100.0 * report.SingleExpressionCount / report.TotalCount;
}
// Deterministic ranking: count desc, then member name asc as a tiebreak.
var ranked = counts
.Select(kv => new FingerprintCount { Member = kv.Key, Count = kv.Value })
.OrderByDescending(f => f.Count)
.ThenBy(f => f.Member, StringComparer.Ordinal)
.ToList();
// Cap the reported ranking (an accumulated log can hold thousands of distinct
// fingerprints); DistinctFingerprints preserves the uncapped total so truncation shows.
report.DistinctFingerprints = ranked.Count;
report.Fingerprints = ranked.Count > fingerprintLimit ? ranked.Take(fingerprintLimit).ToList() : ranked;
// Coverage cross-references the FULL ranking, so a tight fingerprint cap cannot hide a
// covered pattern or demote an uncovered one; topUncovered has its own cap.
report.CoverageSuggestions = BuildCoverage(ranked, availableCommands, coverageMap, topUncoveredLimit);
return report;
}
private static EvalCoverageSuggestions BuildCoverage(
List ranked,
ISet availableCommands,
IReadOnlyDictionary coverageMap,
int topUncoveredLimit)
{
var suggestions = new EvalCoverageSuggestions();
foreach (var entry in ranked)
{
coverageMap.TryGetValue(entry.Member, out var mappedCommand);
var covered = mappedCommand != null && availableCommands.Contains(mappedCommand);
if (covered)
{
suggestions.Covered.Add(new CoverageHit
{
Member = entry.Member,
Count = entry.Count,
Command = mappedCommand
});
}
else
{
suggestions.TopUncovered.Add(new CoverageGap
{
Member = entry.Member,
Count = entry.Count,
// A mapped-but-absent command is a concrete proposal; no mapping leaves it null.
SuggestedCommand = mappedCommand
});
}
}
// Negative limits were normalised to the default in Build, so a plain cap suffices.
if (suggestions.TopUncovered.Count > topUncoveredLimit)
suggestions.TopUncovered = suggestions.TopUncovered.Take(topUncoveredLimit).ToList();
return suggestions;
}
}
}