diff --git a/src/client/envExt/api.internal.ts b/src/client/envExt/api.internal.ts index 5edfb712072e..69050ff5980c 100644 --- a/src/client/envExt/api.internal.ts +++ b/src/client/envExt/api.internal.ts @@ -16,6 +16,7 @@ import { executeCommand } from '../common/vscodeApis/commandApis'; import { getConfiguration, getWorkspaceFolders } from '../common/vscodeApis/workspaceApis'; import { traceError, traceLog } from '../logging'; import { Interpreters } from '../common/utils/localize'; +import type { EnvsIntegrationDecisionReason, EnvsIntegrationDecisionTelemetry } from '../telemetry/types'; export const ENVS_EXTENSION_ID = 'ms-python.vscode-python-envs'; @@ -54,18 +55,55 @@ export function shouldEnvExtHandleActivation(): boolean { return true; } +function getEnvExtensionDecisionInputs(): Pick< + EnvsIntegrationDecisionTelemetry, + 'envsAvailableToHostNow' | 'envsActiveNow' | 'envsResolvedSettingNow' +> { + const config = getConfiguration('python'); + const extension = getExtension(ENVS_EXTENSION_ID); + return { + envsAvailableToHostNow: !!extension, + envsActiveNow: extension?.isActive ?? false, + envsResolvedSettingNow: config?.get('useEnvironmentsExtension', false) ?? false, + }; +} + let _useExt: boolean | undefined; +let _envsDecisionReason: EnvsIntegrationDecisionReason | undefined; export function useEnvExtension(): boolean { if (_useExt !== undefined) { return _useExt; } - const config = getConfiguration('python'); - const inExpSetting = config?.get('useEnvironmentsExtension', false) ?? false; + const { envsAvailableToHostNow, envsResolvedSettingNow } = getEnvExtensionDecisionInputs(); // If extension is installed and in experiment, then use it. - _useExt = !!getExtension(ENVS_EXTENSION_ID) && inExpSetting; + _useExt = envsAvailableToHostNow && envsResolvedSettingNow; + _envsDecisionReason = !envsAvailableToHostNow + ? 'extensionUnavailable' + : envsResolvedSettingNow + ? 'enabled' + : 'resolvedSettingFalse'; return _useExt; } +/** + * Reports the cached integration decision inputs and their current values without initializing or changing the cache. + */ +export function getEnvExtensionDecisionTelemetry(): EnvsIntegrationDecisionTelemetry { + return { + envsDecisionReason: _envsDecisionReason, + ...getEnvExtensionDecisionInputs(), + envsCachedDecision: _useExt, + }; +} + +/** + * Resets the cached integration decision for unit tests. + */ +export function _resetEnvExtensionDecisionCache(): void { + _useExt = undefined; + _envsDecisionReason = undefined; +} + const onDidChangeEnvironmentEnvExtEmitter: EventEmitter = new EventEmitter< DidChangeEnvironmentEventArgs >(); diff --git a/src/client/envExt/telemetry.ts b/src/client/envExt/telemetry.ts index 5575d6a26558..d59e411c9e91 100644 --- a/src/client/envExt/telemetry.ts +++ b/src/client/envExt/telemetry.ts @@ -3,6 +3,7 @@ import { IDisposableRegistry } from '../common/types'; import { getConfiguration, getWorkspaceFolders, onDidChangeConfiguration } from '../common/vscodeApis/workspaceApis'; +import { getEnvExtensionDecisionTelemetry } from './api.internal'; import { sendTelemetryEvent } from '../telemetry'; import { EventName } from '../telemetry/constants'; import { EnvsExplicitFalseScope } from '../telemetry/types'; @@ -45,6 +46,7 @@ export function registerEnvironmentsExtensionTelemetry(disposables: IDisposableR if (event.affectsConfiguration('python.useEnvironmentsExtension')) { sendTelemetryEvent(EventName.ENVIRONMENTS_EXTENSION_SETTING_CHANGED, undefined, { envsExplicitFalseScope: getEnvsExplicitFalseScope(), + ...getEnvExtensionDecisionTelemetry(), }); } }), diff --git a/src/client/startupTelemetry.ts b/src/client/startupTelemetry.ts index 40a5d6b87e7d..fb36da55a1a5 100644 --- a/src/client/startupTelemetry.ts +++ b/src/client/startupTelemetry.ts @@ -16,9 +16,19 @@ import { sendTelemetryEvent } from './telemetry'; import { EventName } from './telemetry/constants'; import { EditorLoadTelemetry } from './telemetry/types'; import { IStartupDurations } from './types'; -import { useEnvExtension } from './envExt/api.internal'; +import { getEnvExtensionDecisionTelemetry, useEnvExtension } from './envExt/api.internal'; import { getEnvsExplicitFalseScope } from './envExt/telemetry'; +function getEnvExtensionStartupTelemetry() { + const telemetry = getEnvExtensionDecisionTelemetry(); + return { + envsDecisionReason: telemetry.envsDecisionReason, + envsAvailableToHostNow: telemetry.envsAvailableToHostNow, + envsActiveNow: telemetry.envsActiveNow, + envsResolvedSettingNow: telemetry.envsResolvedSettingNow, + }; +} + export async function sendStartupTelemetry( activatedPromise: Promise, durations: IStartupDurations, @@ -99,6 +109,7 @@ async function getActivationTelemetryProps( terminal: terminalShellType, isFirstSession, envsExplicitFalseScope: getEnvsExplicitFalseScope(), + ...getEnvExtensionStartupTelemetry(), }; } const interpreterService = serviceContainer.get(IInterpreterService); @@ -157,5 +168,6 @@ async function getActivationTelemetryProps( isFirstSession, usingEnvironmentsExtension, envsExplicitFalseScope: getEnvsExplicitFalseScope(), + ...getEnvExtensionStartupTelemetry(), }; } diff --git a/src/client/telemetry/index.ts b/src/client/telemetry/index.ts index 7d95dc9b9d31..2ee2cf91ceef 100644 --- a/src/client/telemetry/index.ts +++ b/src/client/telemetry/index.ts @@ -13,7 +13,7 @@ import { EnvironmentType, PythonEnvironment } from '../pythonEnvironments/info'; import { TensorBoardPromptSelection } from '../tensorBoard/constants'; import { EventName } from './constants'; import type { UnitTestRunFailureCategory } from './constants'; -import type { EnvsExplicitFalseScope, TestTool } from './types'; +import type { EnvsExplicitFalseScope, EnvsIntegrationDecisionReason, TestTool } from './types'; /** * Checks whether telemetry is supported. @@ -360,7 +360,11 @@ export interface IEventNamePropertyMapping { "usingglobalinterpreter" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "luabud" }, "isfirstsession" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "luabud" }, "usingenvironmentsextension" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eduardovil" }, - "envsexplicitfalsescope" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" } + "envsexplicitfalsescope" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }, + "envsdecisionreason" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }, + "envsavailabletohostnow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }, + "envsactivenow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }, + "envsresolvedsettingnow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" } } */ [EventName.EDITOR_LOAD]: { @@ -416,6 +420,22 @@ export interface IEventNamePropertyMapping { * 'none' excludes defaults; 'multiple' means more than one of user, workspace, or folder. */ envsExplicitFalseScope?: EnvsExplicitFalseScope; + /** + * The inputs that produced the cached Python Environments integration decision. + */ + envsDecisionReason?: EnvsIntegrationDecisionReason; + /** + * Whether Python Environments is currently available to this extension host. + */ + envsAvailableToHostNow?: boolean; + /** + * Whether Python Environments is currently active in this extension host. + */ + envsActiveNow?: boolean; + /** + * The current resolved value of python.useEnvironmentsExtension. + */ + envsResolvedSettingNow?: boolean; }; /** * Reports explicit-false scopes after a python.useEnvironmentsExtension configuration change. @@ -423,11 +443,21 @@ export interface IEventNamePropertyMapping { */ /* __GDPR__ "environments_extension.setting_changed" : { - "envsexplicitfalsescope" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" } + "envsexplicitfalsescope" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }, + "envsdecisionreason" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }, + "envsavailabletohostnow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }, + "envsactivenow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }, + "envsresolvedsettingnow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }, + "envscacheddecision" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" } } */ [EventName.ENVIRONMENTS_EXTENSION_SETTING_CHANGED]: { envsExplicitFalseScope: EnvsExplicitFalseScope; + envsDecisionReason?: EnvsIntegrationDecisionReason; + envsAvailableToHostNow: boolean; + envsActiveNow: boolean; + envsResolvedSettingNow: boolean; + envsCachedDecision?: boolean; }; /** * Telemetry event sent when substituting Environment variables to calculate value of variables diff --git a/src/client/telemetry/types.ts b/src/client/telemetry/types.ts index 11f8a6017b91..b3987187551f 100644 --- a/src/client/telemetry/types.ts +++ b/src/client/telemetry/types.ts @@ -10,6 +10,16 @@ export type EditorLoadTelemetry = IEventNamePropertyMapping[EventName.EDITOR_LOA export type EnvsExplicitFalseScope = 'none' | 'user' | 'workspace' | 'folder' | 'multiple'; +export type EnvsIntegrationDecisionReason = 'enabled' | 'extensionUnavailable' | 'resolvedSettingFalse'; + +export type EnvsIntegrationDecisionTelemetry = { + envsDecisionReason?: EnvsIntegrationDecisionReason; + envsAvailableToHostNow: boolean; + envsActiveNow: boolean; + envsResolvedSettingNow: boolean; + envsCachedDecision?: boolean; +}; + export type PythonInterpreterTelemetry = IEventNamePropertyMapping[EventName.PYTHON_INTERPRETER]; export type TestTool = 'pytest' | 'unittest'; export type TestRunTelemetry = IEventNamePropertyMapping[EventName.UNITTEST_RUN]; diff --git a/src/test/envExt/api.internal.unit.test.ts b/src/test/envExt/api.internal.unit.test.ts new file mode 100644 index 000000000000..f713c6fc6871 --- /dev/null +++ b/src/test/envExt/api.internal.unit.test.ts @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { assert } from 'chai'; +import * as sinon from 'sinon'; +import * as typemoq from 'typemoq'; +import { Extension, WorkspaceConfiguration } from 'vscode'; +import * as extensionsApi from '../../client/common/vscodeApis/extensionsApi'; +import * as workspaceApis from '../../client/common/vscodeApis/workspaceApis'; +import { + _resetEnvExtensionDecisionCache, + ENVS_EXTENSION_ID, + getEnvExtensionDecisionTelemetry, + useEnvExtension, +} from '../../client/envExt/api.internal'; + +function configuration(resolvedSetting: boolean): WorkspaceConfiguration { + const config = typemoq.Mock.ofType(); + config.setup((c) => c.get('useEnvironmentsExtension', false)).returns(() => resolvedSetting); + return config.object; +} + +suite('Python Environments integration decision telemetry', () => { + let getConfiguration: sinon.SinonStub; + let getExtension: sinon.SinonStub; + + setup(() => { + _resetEnvExtensionDecisionCache(); + getConfiguration = sinon.stub(workspaceApis, 'getConfiguration').returns(configuration(false)); + getExtension = sinon.stub(extensionsApi, 'getExtension').returns(undefined); + }); + + teardown(() => { + _resetEnvExtensionDecisionCache(); + sinon.restore(); + }); + + const cases = [ + { + name: 'extension unavailable', + available: false, + active: false, + resolvedSetting: true, + decision: false, + reason: 'extensionUnavailable', + }, + { + name: 'resolved setting false', + available: true, + active: true, + resolvedSetting: false, + decision: false, + reason: 'resolvedSettingFalse', + }, + { + name: 'integration enabled', + available: true, + active: true, + resolvedSetting: true, + decision: true, + reason: 'enabled', + }, + ] as const; + + cases.forEach(({ name, available, active, resolvedSetting, decision, reason }) => { + test(`captures ${name}`, () => { + getConfiguration.returns(configuration(resolvedSetting)); + getExtension.returns(available ? ({ isActive: active } as Extension) : undefined); + + assert.strictEqual(useEnvExtension(), decision); + assert.deepEqual(getEnvExtensionDecisionTelemetry(), { + envsDecisionReason: reason, + envsAvailableToHostNow: available, + envsActiveNow: active, + envsResolvedSettingNow: resolvedSetting, + envsCachedDecision: decision, + }); + sinon.assert.alwaysCalledWithExactly(getExtension, ENVS_EXTENSION_ID); + }); + }); + + test('reports live inputs without changing a stale cached decision', () => { + assert.isFalse(useEnvExtension()); + getConfiguration.returns(configuration(true)); + getExtension.returns({ isActive: true } as Extension); + + assert.deepEqual(getEnvExtensionDecisionTelemetry(), { + envsDecisionReason: 'extensionUnavailable', + envsAvailableToHostNow: true, + envsActiveNow: true, + envsResolvedSettingNow: true, + envsCachedDecision: false, + }); + assert.isFalse(useEnvExtension()); + }); + + test('does not initialize the cached decision when telemetry is requested', () => { + getConfiguration.returns(configuration(true)); + getExtension.returns({ isActive: false } as Extension); + + assert.deepEqual(getEnvExtensionDecisionTelemetry(), { + envsDecisionReason: undefined, + envsAvailableToHostNow: true, + envsActiveNow: false, + envsResolvedSettingNow: true, + envsCachedDecision: undefined, + }); + }); +}); diff --git a/src/test/envExt/telemetry.unit.test.ts b/src/test/envExt/telemetry.unit.test.ts index 6e15ad6d9834..46d6fab011aa 100644 --- a/src/test/envExt/telemetry.unit.test.ts +++ b/src/test/envExt/telemetry.unit.test.ts @@ -8,6 +8,7 @@ import * as typemoq from 'typemoq'; import { ConfigurationChangeEvent, EventEmitter, Uri, WorkspaceConfiguration, WorkspaceFolder } from 'vscode'; import { IDisposableRegistry } from '../../client/common/types'; import * as workspaceApis from '../../client/common/vscodeApis/workspaceApis'; +import * as envExt from '../../client/envExt/api.internal'; import { getEnvsExplicitFalseScope, registerEnvironmentsExtensionTelemetry } from '../../client/envExt/telemetry'; import * as telemetry from '../../client/telemetry'; import { EventName } from '../../client/telemetry/constants'; @@ -44,6 +45,13 @@ suite('Environments extension telemetry', () => { let sendTelemetryEvent: sinon.SinonStub; let changes: EventEmitter; let disposables: IDisposableRegistry; + const decisionTelemetry = { + envsDecisionReason: 'resolvedSettingFalse' as const, + envsAvailableToHostNow: true, + envsActiveNow: true, + envsResolvedSettingNow: false, + envsCachedDecision: false, + }; setup(() => { getConfiguration = sinon.stub(workspaceApis, 'getConfiguration').returns(configuration({})); @@ -51,6 +59,7 @@ suite('Environments extension telemetry', () => { sendTelemetryEvent = sinon.stub(telemetry, 'sendTelemetryEvent'); changes = new EventEmitter(); sinon.stub(workspaceApis, 'onDidChangeConfiguration').callsFake((listener) => changes.event(listener)); + sinon.stub(envExt, 'getEnvExtensionDecisionTelemetry').returns(decisionTelemetry); disposables = []; }); @@ -170,7 +179,7 @@ suite('Environments extension telemetry', () => { ['user', 'workspace', 'none'].map((envsExplicitFalseScope) => [ EventName.ENVIRONMENTS_EXTENSION_SETTING_CHANGED, undefined, - { envsExplicitFalseScope }, + { envsExplicitFalseScope, ...decisionTelemetry }, ]), ); }); @@ -185,7 +194,7 @@ suite('Environments extension telemetry', () => { sendTelemetryEvent, EventName.ENVIRONMENTS_EXTENSION_SETTING_CHANGED, undefined, - { envsExplicitFalseScope: 'folder' }, + { envsExplicitFalseScope: 'folder', ...decisionTelemetry }, ); }); diff --git a/src/test/startupTelemetry.unit.test.ts b/src/test/startupTelemetry.unit.test.ts index 1c6004007454..e63851ad93bc 100644 --- a/src/test/startupTelemetry.unit.test.ts +++ b/src/test/startupTelemetry.unit.test.ts @@ -73,6 +73,13 @@ suite('Startup Telemetry - hasUserDefinedPythonPath()', async () => { .returns(() => interpreterPathService.object); sinon.stub(constants, 'isTestExecution').returns(false); sinon.stub(envExt, 'useEnvExtension').returns(false); + sinon.stub(envExt, 'getEnvExtensionDecisionTelemetry').returns({ + envsDecisionReason: 'resolvedSettingFalse', + envsAvailableToHostNow: true, + envsActiveNow: true, + envsResolvedSettingNow: false, + envsCachedDecision: false, + }); sinon.stub(envExtTelemetry, 'getEnvsExplicitFalseScope').returns('workspace'); sendTelemetryEvent = sinon.stub(telemetry, 'sendTelemetryEvent'); durations = { @@ -101,11 +108,19 @@ suite('Startup Telemetry - hasUserDefinedPythonPath()', async () => { sendTelemetryEvent, EventName.EDITOR_LOAD, durations, - sinon.match({ envsExplicitFalseScope: 'workspace', isFirstSession: false }), + sinon.match({ + envsExplicitFalseScope: 'workspace', + envsDecisionReason: 'resolvedSettingFalse', + envsAvailableToHostNow: true, + envsActiveNow: true, + envsResolvedSettingNow: false, + isFirstSession: false, + }), ); if (isTrusted) { expect(sendTelemetryEvent.firstCall.args[2].usingEnvironmentsExtension).to.equal(false); } + expect(sendTelemetryEvent.firstCall.args[2]).not.to.have.property('envsCachedDecision'); }); }); });