Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 108 additions & 3 deletions packages/cli-kit/src/public/node/analytics.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
import {reportAnalyticsEvent, recordTiming, recordError, recordRetry, recordEvent} from './analytics.js'
import {
reportAnalyticsEvent,
sendAnalyticsEventFromFile,
recordTiming,
recordError,
recordRetry,
recordEvent,
} from './analytics.js'
import * as os from './os.js'
import {
analyticsDisabled,
Expand All @@ -15,13 +22,15 @@
import {mockAndCaptureOutput} from './testing/output.js'
import {addPublicMetadata, addSensitiveMetadata} from './metadata.js'
import {sendErrorToBugsnag} from './error-handler.js'
import {hashString} from './crypto.js'
import {hashString, randomUUID} from './crypto.js'
import {exec} from './system.js'
import * as store from '../../private/node/analytics/storage.js'
import {startAnalytics} from '../../private/node/analytics.js'
import {CLI_KIT_VERSION} from '../common/version.js'
import {setLastSeenAuthMethod, setLastSeenUserIdAfterAuth} from '../../private/node/session.js'

import {test, expect, describe, vi, beforeEach, afterEach, MockedFunction} from 'vitest'
import {stat} from 'node:fs/promises'
import {randomUUID as nodeRandomUUID} from 'node:crypto'

vi.mock('./context/local.js')
vi.mock('./os.js')
Expand All @@ -32,6 +41,7 @@
vi.mock('./monorail.js')
vi.mock('./cli.js')
vi.mock('./error-handler.js')
vi.mock('./system.js')

function restoreEnvVariable(key: string, value: string | undefined): void {
if (value === undefined) {
Expand All @@ -44,6 +54,7 @@
describe('event tracking', () => {
const currentDate = new Date(Date.UTC(2022, 1, 1, 10, 0, 0))
let publishEventMock: MockedFunction<typeof publishMonorailEvent>
let execMock: MockedFunction<typeof exec>

beforeEach(() => {
vi.setSystemTime(currentDate)
Expand All @@ -53,10 +64,12 @@
vi.mocked(ciPlatform).mockReturnValue({isCI: true, name: 'vitest', metadata: {}})
vi.mocked(macAddress).mockResolvedValue('macAddress')
vi.mocked(hashString).mockReturnValue('hashed-macaddress')
vi.mocked(randomUUID).mockImplementation(nodeRandomUUID)
vi.mocked(isUnitTest).mockReturnValue(true)
vi.mocked(cloudEnvironment).mockReturnValue({platform: 'localhost', editor: false})
vi.mocked(os.platformAndArch).mockReturnValue({platform: 'darwin', arch: 'arm64'})
publishEventMock = vi.mocked(publishMonorailEvent).mockReturnValue(Promise.resolve({type: 'ok'}))
execMock = vi.mocked(exec).mockResolvedValue(undefined)
})

afterEach(() => {
Expand All @@ -72,6 +85,92 @@
})
}

async function sendReportedAnalyticsPayload(): Promise<void> {
expect(execMock).toHaveBeenCalledOnce()
const execArgs = execMock.mock.calls[0]![1]
expect(execArgs.slice(1, 3)).toEqual(['send-analytics', '--payload-file'])

const payloadFile = execArgs[3]
if (!payloadFile) {
throw new Error('Expected send-analytics to receive a payload file')
}

await sendAnalyticsEventFromFile(payloadFile)
}

test('waits for the analytics process on Windows', async () => {
await inProjectWithFile('package.json', async (args) => {
// Given
const commandContent = {command: 'info', topic: 'app'}
await startAnalytics({commandContent, args, currentTime: currentDate.getTime() - 100})
vi.mocked(os.platformAndArch).mockReturnValue({platform: 'windows', arch: 'arm64'})

let resolveAnalyticsProcess: () => void = () => {}
const analyticsProcess = new Promise<void>((resolve) => {
resolveAnalyticsProcess = resolve
})
execMock.mockReturnValueOnce(analyticsProcess)

const config = {
runHook: vi.fn().mockResolvedValue({successes: [], failures: []}),
plugins: [],
} as any

// When
let reportCompleted = false
const report = reportAnalyticsEvent({config, exitMode: 'expected_error'}).then(() => {
reportCompleted = true
})
await vi.waitFor(() => expect(execMock).toHaveBeenCalledOnce())

// Then
expect(reportCompleted).toBe(false)
expect(execMock).toHaveBeenCalledWith(
expect.anything(),
expect.anything(),
expect.objectContaining({background: false}),
)
resolveAnalyticsProcess()
await report
expect(reportCompleted).toBe(true)
await sendReportedAnalyticsPayload()
})
})

test('does not wait for the analytics process on non-Windows platforms', async () => {
await inProjectWithFile('package.json', async (args) => {
// Given
const commandContent = {command: 'info', topic: 'app'}
await startAnalytics({commandContent, args, currentTime: currentDate.getTime() - 100})

let resolveAnalyticsProcess: () => void = () => {}
const analyticsProcess = new Promise<void>((resolve) => {
resolveAnalyticsProcess = resolve
})
execMock.mockReturnValueOnce(analyticsProcess)

const config = {
runHook: vi.fn().mockResolvedValue({successes: [], failures: []}),
plugins: [],
} as any

// When
await reportAnalyticsEvent({config, exitMode: 'expected_error'})

// Then
expect(execMock).toHaveBeenCalledWith(
expect.anything(),
expect.anything(),
expect.objectContaining({background: true}),
)
const payloadFile = execMock.mock.calls[0]![1][3]!
expect(payloadFile).toMatch(/shopify-cli-analytics-[\da-f-]+\.json$/u)
expect((await stat(payloadFile)).mode & 0o777).toBe(0o600)

Check failure on line 168 in packages/cli-kit/src/public/node/analytics.test.ts

View workflow job for this annotation

GitHub Actions / Unit tests with Node 26.1.0 in windows-latest (shard 2/2)

[@shopify/cli-kit] src/public/node/analytics.test.ts > event tracking > does not wait for the analytics process on non-Windows platforms

AssertionError: expected 438 to be 384 // Object.is equality - Expected + Received - 384 + 438 ❯ src/public/node/analytics.test.ts:168:54 ❯ src/public/node/analytics.test.ts:84:7 ❯ Module.inTemporaryDirectory src/public/node/fs.ts:81:12 ❯ inProjectWithFile src/public/node/analytics.test.ts:80:5 ❯ src/public/node/analytics.test.ts:141:5
resolveAnalyticsProcess()
await sendAnalyticsEventFromFile(payloadFile)
})
})

test('sends the expected data to Monorail with cached app info', async () => {
await inProjectWithFile('package.json', async (args) => {
// Given
Expand All @@ -95,6 +194,7 @@
plugins: pluginsMap,
} as any
await reportAnalyticsEvent({config, exitMode: 'ok'})
await sendReportedAnalyticsPayload()
// Then
const version = CLI_KIT_VERSION
const expectedPayloadPublic = {
Expand Down Expand Up @@ -155,6 +255,7 @@
plugins: [],
} as any
await reportAnalyticsEvent({config, exitMode: 'ok'})
await sendReportedAnalyticsPayload()

// Then
expect(publishEventMock).toHaveBeenCalledOnce()
Expand All @@ -179,6 +280,7 @@
plugins: [],
} as any
await reportAnalyticsEvent({config, errorMessage: 'Permission denied', exitMode: 'unexpected_error'})
await sendReportedAnalyticsPayload()

// Then
const version = CLI_KIT_VERSION
Expand Down Expand Up @@ -219,6 +321,7 @@
plugins: [],
} as any
await reportAnalyticsEvent({config, exitMode: 'ok'})
await sendReportedAnalyticsPayload()

// Then
const expectedPayloadSensitive = {
Expand All @@ -243,6 +346,7 @@
plugins: [],
} as any
await reportAnalyticsEvent({config, exitMode: 'ok'})
await sendReportedAnalyticsPayload()

expect(publishEventMock).toHaveBeenCalledOnce()
expect(publishEventMock.mock.calls[0]![2]).toMatchObject({
Expand Down Expand Up @@ -272,6 +376,7 @@
plugins: [],
} as any
await reportAnalyticsEvent({config, exitMode: 'ok'})
await sendReportedAnalyticsPayload()

// Then
const sensitivePayload = publishEventMock.mock.calls[0]![2]
Expand Down
126 changes: 96 additions & 30 deletions packages/cli-kit/src/public/node/analytics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,58 @@ interface ReportAnalyticsEventOptions {
exitMode: CommandExitMode
}

export async function sendAnalyticsEventFromFile(payloadFile: string): Promise<void> {
const {readFile, removeFile} = await import('./fs.js')
try {
const payloadStr = await readFile(payloadFile)
const payload = JSON.parse(payloadStr)

const doMonorail = async () => {
if (payload.skipMonorailAnalytics) return
const response = await publishMonorailEvent(MONORAIL_COMMAND_TOPIC, payload.public, payload.sensitive)
if (response.type === 'error') {
outputDebug(response.message)
}
}

const doOpenTelemetry = async () => {
if (payload.skipMetricAnalytics) return

const active = payload.public.cmd_all_timing_active_ms ?? 0
const network = payload.public.cmd_all_timing_network_ms ?? 0
const prompt = payload.public.cmd_all_timing_prompts_ms ?? 0

return recordMetrics(
{
skipMetricAnalytics: payload.skipMetricAnalytics,
cliVersion: payload.public.cli_version,
owningPlugin: payload.public.cmd_all_plugin ?? '@shopify/cli',
command: payload.public.command,
exitMode: payload.public.cmd_all_exit,
},
{
active,
network,
prompt,
},
)
}

await Promise.all([doMonorail(), doOpenTelemetry()])
} catch (error) {
if (error instanceof Error) {
outputDebug(`Failed to send analytics in background: ${error.message}`)
} else {
throw error
}
} finally {
await removeFile(payloadFile).catch((error: unknown) => {
const message = error instanceof Error ? error.message : String(error)
outputDebug(`Failed to remove background analytics payload: ${message}`)
})
}
}

/**
* Report an analytics event, sending it off to Monorail -- Shopify's internal analytics service.
*
Expand All @@ -45,8 +97,7 @@ interface ReportAnalyticsEventOptions {
export async function reportAnalyticsEvent(options: ReportAnalyticsEventOptions): Promise<void> {
try {
const payload = await buildPayload(options)
if (payload === undefined) {
// Nothing to log
if (payload === undefined || payload.public.command === 'send-analytics') {
return
}

Expand All @@ -65,40 +116,55 @@ export async function reportAnalyticsEvent(options: ReportAnalyticsEventOptions)

const skipMonorailAnalytics = !alwaysLogAnalytics() && analyticsDisabled()
const skipMetricAnalytics = !alwaysLogMetrics() && analyticsDisabled()
if (skipMonorailAnalytics || skipMetricAnalytics) {
if (skipMonorailAnalytics && skipMetricAnalytics) {
outputDebug(outputContent`Skipping command analytics, payload: ${outputToken.json(payload)}`)
return
}

const doMonorail = async () => {
if (skipMonorailAnalytics) {
return
}
const response = await publishMonorailEvent(MONORAIL_COMMAND_TOPIC, payload.public, payload.sensitive)
if (response.type === 'error') {
outputDebug(response.message)
}
const {platformAndArch} = await import('./os.js')
const sendInBackground = platformAndArch().platform !== 'windows'
const deliveryDescription = sendInBackground ? ' in background' : ''
outputDebug(outputContent`Sending command analytics${deliveryDescription}, payload: ${outputToken.json(payload)}`)

const {joinPath} = await import('./path.js')
const {tmpdir} = await import('node:os')
const {removeFile, writeFile} = await import('./fs.js')
const {randomUUID} = await import('./crypto.js')

const payloadPath = joinPath(tmpdir(), `shopify-cli-analytics-${randomUUID()}.json`)

const fullPayload = {
...payload,
skipMonorailAnalytics,
skipMetricAnalytics,
}
const doOpenTelemetry = async () => {
const active = payload.public.cmd_all_timing_active_ms ?? 0
const network = payload.public.cmd_all_timing_network_ms ?? 0
const prompt = payload.public.cmd_all_timing_prompts_ms ?? 0

return recordMetrics(
{
skipMetricAnalytics,
cliVersion: payload.public.cli_version,
owningPlugin: payload.public.cmd_all_plugin ?? '@shopify/cli',
command: payload.public.command,
exitMode: options.exitMode,
},
{
active,
network,
prompt,
},
)
await writeFile(payloadPath, JSON.stringify(fullPayload), {encoding: 'utf8', mode: 0o600, flag: 'wx'})

const {exec} = await import('./system.js')
const argv = process.argv
if (!argv[0] || !argv[1]) {
await removeFile(payloadPath)
return
}
const nodeBinary = argv[0]
const shopifyBinary = argv[1]
const args = [shopifyBinary, 'send-analytics', '--payload-file', payloadPath]

const analyticsProcess = exec(nodeBinary, args, {
background: sendInBackground,
env: {...process.env, SHOPIFY_CLI_NO_ANALYTICS: '1'},
externalErrorHandler: async (error: unknown) => {
await removeFile(payloadPath)
outputDebug(`Failed to send analytics in background: ${(error as Error).message}`)
},
})
if (sendInBackground) {
// eslint-disable-next-line no-void
void analyticsProcess
} else {
await analyticsProcess
}
await Promise.all([doMonorail(), doOpenTelemetry()])

// eslint-disable-next-line no-catch-all/no-catch-all
} catch (error) {
Expand Down
2 changes: 2 additions & 0 deletions packages/cli-kit/src/public/node/fs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,8 @@ export function appendFileSync(path: string, data: string): void {

export interface WriteOptions {
encoding: BufferEncoding
mode?: number
flag?: string
}

/**
Expand Down
4 changes: 2 additions & 2 deletions packages/cli-kit/src/public/node/hooks/postrun.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,11 @@ export const hook: Hook.Postrun = async ({config, Command}) => {
const command = Command.id.replace(/:/g, ' ')
outputDebug(`Completed command ${command}`)

if (!command.includes('notifications') && !command.includes('upgrade')) await autoUpgradeIfNeeded()
if (!command.includes('notifications') && !command.includes('upgrade') && !command.includes('send-analytics'))
await autoUpgradeIfNeeded()

const {reportAnalyticsEvent} = await import('../analytics.js')
await reportAnalyticsEvent({config, exitMode: 'ok'})

postRunHookCompleted = true
}

Expand Down
1 change: 1 addition & 0 deletions packages/cli-kit/src/public/node/notifications-system.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ const COMMANDS_TO_SKIP = [
'theme:init',
'hydrogen:init',
'cache:clear',
'send-analytics',
]

function url(): string {
Expand Down
Loading
Loading