Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 174 additions & 4 deletions packages/app/cypress/component/scatter-graph.cy.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,7 @@ describe('ScatterGraph', () => {
cy.get('#test-scatter-m3-eagle svg .line-label text').should('not.contain.text', 'MTP');
});

it('removes a cross-engine AgentX STP overlay before rendering', () => {
it('prefers a cross-engine AgentX STP overlay over the conflicting official series', () => {
const chartDefinition = createMockChartDefinition({
chartType: 'interactivity',
y_tpPerGpu_roofline: 'upper_left',
Expand Down Expand Up @@ -410,9 +410,97 @@ describe('ScatterGraph', () => {
},
);

cy.get('#test-scatter-agentx-engine-guard svg .roofline-path').should('exist');
cy.get('#test-scatter-agentx-engine-guard svg .overlay-roofline-path').should('not.exist');
cy.get('@setActiveOverlayHwTypes').should('have.been.called');
// The unofficial run was loaded to be seen: its engine family wins the
// cross-engine exclusion, so the overlay renders and the conflicting
// official SGLang series is deselected (restorable by dismissing the run).
// Official rooflines stay in the DOM when deselected — they hide via opacity.
cy.get('#test-scatter-agentx-engine-guard svg .overlay-roofline-path').should('exist');
cy.get('#test-scatter-agentx-engine-guard svg .roofline-path').should(
'have.css',
'opacity',
'0',
);
// No reconciliation write-back: the provider's overlay selection already
// matches the resolved selection.
cy.get('@setActiveOverlayHwTypes').should('not.have.been.called');
});

it('renders both official and overlay AgentX STP series from the same engine family', () => {
const chartDefinition = createMockChartDefinition({
chartType: 'interactivity',
y_tpPerGpu_roofline: 'upper_left',
});
const officialData = [8, 16, 32].map((x, index) =>
createMockInferenceData({
hwKey: 'b200_vllm',
x,
y: 320 - index * 40,
precision: Precision.FP4,
}),
);
const runUrl = 'https://github.com/x/y/actions/runs/agentx-vllm-same-family';
const overlayData = {
data: [8, 16, 32].map((x, index) =>
createMockInferenceData({
hwKey: 'h100_vllm',
x,
y: 260 - index * 40,
precision: Precision.FP4,
run_url: runUrl,
}),
),
hardwareConfig: hwConfig,
label: 'agentx-vllm-same-family',
runUrl,
};
const exclusion = buildExclusion([
{ suffix: null, stripPrefixes: ['dynamo-', 'mori-', 'llmd-', 'mooncake-'] },
]);
const namespacedExclusion = {
familyOf: (key: string) =>
exclusion.familyOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key),
groupOf: (key: string) =>
exclusion.groupOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key),
};

mountWithProviders(
<div style={{ width: 800, height: 600 }}>
<ScatterGraph
chartId="test-scatter-agentx-same-family"
modelLabel="DeepSeek V4 Pro"
data={officialData}
xLabel="Concurrency"
yLabel="Throughput / GPU (tok/s)"
chartDefinition={chartDefinition}
overlayData={overlayData}
/>
</div>,
{
inference: {
hardwareConfig: hwConfig,
activeHwTypes: new Set(['b200_vllm']),
hwTypesWithData: new Set(['b200_vllm']),
selectedModel: Model.DeepSeek_V4_Pro,
selectedSequence: Sequence.AgenticTraces,
selectedPrecisions: [Precision.FP4],
showLineLabels: true,
resolveComparisonSelection: (proposed, prev = new Set()) =>
resolveExclusionGroups(proposed, prev, namespacedExclusion, 'keep-sticky'),
},
unofficial: {
activeOverlayHwTypes: new Set(['h100_vllm']),
allOverlayHwTypes: new Set(['h100_vllm']),
},
},
);

// Same engine family: no exclusion applies, both series render.
cy.get('#test-scatter-agentx-same-family svg .overlay-roofline-path').should('exist');
cy.get('#test-scatter-agentx-same-family svg .roofline-path').should(
'have.css',
'opacity',
'1',
);
});
});

Expand Down Expand Up @@ -466,6 +554,88 @@ describe('ChartDisplay engine comparison guard', () => {
cy.get('[data-testid="inference-results-table"] tbody tr').should('have.length', 1);
});

it('keeps a cross-engine unofficial overlay in table mode and drops the conflicting official', () => {
const chartDefinition = createMockChartDefinition({ chartType: 'interactivity' });
const sglangRow = createMockInferenceData({
hwKey: 'b200_sglang',
hw: 'Official SGLang',
model: Model.DeepSeek_V4_Pro,
precision: Precision.FP4,
});
const runUrl = 'https://github.com/x/y/actions/runs/456';
const overlayRow = createMockInferenceData({
hwKey: 'h100_vllm',
hw: 'Unofficial vLLM',
model: Model.DeepSeek_V4_Pro,
precision: Precision.FP4,
run_url: runUrl,
});
const exclusion = buildExclusion([
{ suffix: null, stripPrefixes: ['dynamo-', 'mori-', 'llmd-', 'mooncake-'] },
]);
const namespacedExclusion = {
familyOf: (key: string) =>
exclusion.familyOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key),
groupOf: (key: string) =>
exclusion.groupOf(key.startsWith('overlay:') ? key.slice('overlay:'.length) : key),
};
const resolveSelection = (proposed: Set<string>, prev = new Set<string>()) =>
resolveExclusionGroups(proposed, prev, namespacedExclusion, 'keep-sticky');
const runInfo = {
id: 456,
name: 'agentx-vllm-overlay',
branch: 'agentx-vllm-overlay',
sha: 'def456',
createdAt: '2026-07-10T00:00:00Z',
url: runUrl,
conclusion: 'success',
status: 'completed',
isNonMainBranch: true,
};

mountWithProviders(<ChartDisplay />, {
inference: {
graphs: [
{
model: Model.DeepSeek_V4_Pro,
sequence: Sequence.AgenticTraces,
chartDefinition,
data: [sglangRow],
},
],
selectedModel: Model.DeepSeek_V4_Pro,
selectedSequence: Sequence.AgenticTraces,
selectedXAxisMode: 'interactivity',
activeHwTypes: new Set(['b200_sglang']),
hwTypesWithData: new Set(['b200_sglang']),
resolveComparisonSelection: resolveSelection,
},
globalFilters: {
selectedModel: Model.DeepSeek_V4_Pro,
selectedSequence: Sequence.AgenticTraces,
effectiveSequence: Sequence.AgenticTraces,
},
unofficial: {
isUnofficialRun: true,
unofficialRunInfo: runInfo,
unofficialRunInfos: [runInfo],
runIndexByUrl: { [runUrl]: 0, '456': 0 },
getOverlayData: () => ({ data: [overlayRow], hardwareConfig: hwConfig }),
activeOverlayHwTypes: new Set(['h100_vllm']),
allOverlayHwTypes: new Set(['h100_vllm']),
},
});

cy.get('[data-testid="inference-table-view-btn"]').click();
// The unofficial run's engine family wins the cross-engine exclusion: the
// overlay row stays and the conflicting official SGLang row is dropped.
cy.get('[data-testid="inference-results-table"] tbody tr').should('have.length', 1);
cy.get('[data-testid="inference-results-table"] tbody').contains('vLLM').should('exist');
cy.get('[data-testid="inference-results-table"] tbody').contains('SGLang').should('not.exist');
// The reconciliation effect must not strip the run's hw types from the provider.
cy.get('@setActiveOverlayHwTypes').should('not.have.been.called');
});

it('keeps an explicitly empty official legend out of table mode', () => {
const chartDefinition = createMockChartDefinition({ chartType: 'interactivity' });
const row = createMockInferenceData({
Expand Down
29 changes: 21 additions & 8 deletions packages/app/src/components/inference/ui/ChartDisplay.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -479,8 +479,17 @@ export default function ChartDisplay() {
const overlayKeys = overlayRowsScopeChanged ? overlayScope : activeScopedOverlayKeys;
const proposed = new Set(officialKeys);
overlayKeys.forEach((key) => proposed.add(`overlay:${key}`));
const previous = new Set(activeOfficialKeys);
activeScopedOverlayKeys.forEach((key) => previous.add(`overlay:${key}`));
// Overlay-preferring sticky set (mirrors ScatterGraph's exclusionStickySet):
// an unofficial run loaded via `?unofficialrun=` exists to be seen, so its
// engine family wins cross-engine exclusion over the official selection.
// Otherwise this effect would strip the run's hw types from the provider
// before the chart ever renders them, with no legend affordance to recover.
const previous = new Set<string>();
if (overlayKeys.size > 0) {
overlayKeys.forEach((key) => previous.add(`overlay:${key}`));
} else {
activeOfficialKeys.forEach((key) => previous.add(key));
}
const resolved = resolveComparisonSelection(proposed, previous).result;
const overlay = new Set<string>();
for (const key of resolved) {
Expand Down Expand Up @@ -541,17 +550,22 @@ export default function ChartDisplay() {
const activeOfficialKeys = new Set(
[...selectedOfficialHwTypes].filter((key) => availableOfficialKeys.has(key)),
);
const activeScopedOverlayKeys = new Set(
[...activeOverlayHwTypes].filter((key) => availableOverlayKeys.has(key)),
);
const officialKeys = activeOfficialKeys;
const overlayKeys = new Set(
[...resolvedScopedOverlayHwTypes].filter((key) => availableOverlayKeys.has(key)),
);
const proposed = new Set(officialKeys);
overlayKeys.forEach((key) => proposed.add(`overlay:${key}`));
const previous = new Set(activeOfficialKeys);
activeScopedOverlayKeys.forEach((key) => previous.add(`overlay:${key}`));
// Same overlay-preferring sticky set as resolvedScopedOverlayHwTypes:
// while overlay rows are visible, the run's engine family wins. Built
// from the already-resolved overlayKeys (not the provider selection,
// which can lag by a render after a scope change).
const previous = new Set<string>();
if (overlayKeys.size > 0) {
overlayKeys.forEach((key) => previous.add(`overlay:${key}`));
} else {
activeOfficialKeys.forEach((key) => previous.add(key));
}
Comment thread
cquil11 marked this conversation as resolved.
const resolved = resolveComparisonSelection(proposed, previous).result;

return {
Expand All @@ -565,7 +579,6 @@ export default function ChartDisplay() {
selectedPrecisions,
quickFilters,
selectedOfficialHwTypes,
activeOverlayHwTypes,
resolvedScopedOverlayHwTypes,
resolveComparisonSelection,
],
Expand Down
28 changes: 20 additions & 8 deletions packages/app/src/components/inference/ui/ScatterGraph.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -453,13 +453,22 @@ const ScatterGraph = React.memo(
rawOverlayHwTypes.forEach((key) => combined.add(`overlay:${key}`));
return combined;
}, [rawOfficialHwTypes, rawOverlayHwTypes]);
// Sticky preference for cross-engine exclusion resolution. An unofficial
// run loaded via `?unofficialrun=` exists to be seen, so when its configs
// conflict with the official engine family the run wins and the conflicting
// official series are deselected — the reverse would leave the overlay
// permanently hidden with no legend affordance to surface it (the run's
// legend entry is not a toggle). Officials stay restorable by dismissing
// the run. Without overlay points the official selection is sticky as before.
const exclusionStickySet = useMemo(() => {
if (rawOverlayHwTypes.size > 0) {
return new Set([...rawOverlayHwTypes].map((key) => `overlay:${key}`));
}
return rawOfficialHwTypes.size > 0 ? rawOfficialHwTypes : rawUnifiedSelection;
}, [rawOverlayHwTypes, rawOfficialHwTypes, rawUnifiedSelection]);
const resolvedUnifiedSelection = useMemo(
() =>
resolveComparisonSelection(
rawUnifiedSelection,
rawOfficialHwTypes.size > 0 ? rawOfficialHwTypes : rawUnifiedSelection,
).result,
[rawUnifiedSelection, rawOfficialHwTypes, resolveComparisonSelection],
() => resolveComparisonSelection(rawUnifiedSelection, exclusionStickySet).result,
[rawUnifiedSelection, exclusionStickySet, resolveComparisonSelection],
);
const resolvedHwTypes = useMemo(() => {
const official = new Set<string>();
Expand Down Expand Up @@ -552,14 +561,17 @@ const ScatterGraph = React.memo(
setLocalOfficialOverride(null);
return;
}
const resolved = resolveComparisonSelection(allUnifiedHwTypes, effectiveOfficialHwTypes);
// Same sticky preference as the render-time resolution: keep the overlay
// run's engine family so a reset doesn't flip the chart back to the
// conflicting official family and re-hide the loaded run.
const resolved = resolveComparisonSelection(allUnifiedHwTypes, exclusionStickySet);
commitUnifiedSelection(resolved.result);
}, [
selectAllHwTypes,
overlayData,
setLocalOfficialOverride,
allUnifiedHwTypes,
effectiveOfficialHwTypes,
exclusionStickySet,
resolveComparisonSelection,
commitUnifiedSelection,
]);
Expand Down
24 changes: 19 additions & 5 deletions packages/app/src/lib/exclusion.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,17 +153,31 @@ describe('AgentX STP engine exclusion', () => {
});
});

it('keeps the official engine when an automatic overlay load conflicts', () => {
it('keeps whichever engine family the sticky set names when a load conflicts', () => {
const proposed = new Set(['b300_sglang', 'overlay:b300_vllm']);
const resolved = resolveExclusionGroups(

// Official-sticky prev: the official engine wins, overlay dropped.
const officialSticky = resolveExclusionGroups(
proposed,
new Set(['b300_sglang']),
namespacedAgenticEx,
'keep-sticky',
);
expect([...resolved.result]).toEqual(['b300_sglang']);
expect(resolved.keptGroup).toBe('sglang');
expect(resolved.droppedGroups).toEqual(['vllm']);
expect([...officialSticky.result]).toEqual(['b300_sglang']);
expect(officialSticky.keptGroup).toBe('sglang');
expect(officialSticky.droppedGroups).toEqual(['vllm']);

// Overlay-sticky prev (what ScatterGraph passes while an unofficial run is
// loaded): the run's engine wins and the official series is dropped.
const overlaySticky = resolveExclusionGroups(
proposed,
new Set(['overlay:b300_vllm']),
namespacedAgenticEx,
'keep-sticky',
);
expect([...overlaySticky.result]).toEqual(['overlay:b300_vllm']);
expect(overlaySticky.keptGroup).toBe('vllm');
expect(overlaySticky.droppedGroups).toEqual(['sglang']);
});
});

Expand Down
Loading