#!/usr/bin/env node // @ts-nocheck import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; const DEFAULT_DAYS = 30; const DEFAULT_LIMIT = 50; const DEFAULT_SOURCES = ['publish-log', 'repo-memories', 'transcripts']; const TRANSCRIPT_KEYWORDS = [ 'agent', 'audit', 'bootstrap', 'checklist', 'hook', 'instruction', 'prompt', 'publish', 'review', 'script', 'setup', 'skill', 'template', 'workflow', ]; function usage() { console.error(`Usage: resources/scripts/audit-copilot-usage.sh [options] Options: --days Audit the last days. Default: ${DEFAULT_DAYS} --since Audit from an explicit ISO 8601 timestamp. --workspace Only include workspaces whose path contains . --exclude-workspace

Exclude a workspace path prefix. Repeatable. Defaults to the repo root. --include-sources Comma-separated list: publish-log,repo-memories,transcripts. --limit Max number of candidates to emit. Default: ${DEFAULT_LIMIT} --machine-id Override the machine id used in the output path. --output-dir Write audit files into an explicit output directory. --repo-root Internal override for the repository root. --help Show this help text. `); } function parseArgs(argv) { const options = { days: DEFAULT_DAYS, excludeWorkspaces: [], includeSources: [...DEFAULT_SOURCES], limit: DEFAULT_LIMIT, repoRoot: process.cwd(), workspaceFilter: '', }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === '--help') { usage(); process.exit(0); } if (arg === '--days') { options.days = Number(argv[index + 1]); index += 1; continue; } if (arg === '--since') { options.since = argv[index + 1]; index += 1; continue; } if (arg === '--workspace') { options.workspaceFilter = argv[index + 1] ?? ''; index += 1; continue; } if (arg === '--exclude-workspace') { options.excludeWorkspaces.push(argv[index + 1] ?? ''); index += 1; continue; } if (arg === '--include-sources') { options.includeSources = (argv[index + 1] ?? '') .split(',') .map((entry) => entry.trim()) .filter(Boolean); index += 1; continue; } if (arg === '--limit') { options.limit = Number(argv[index + 1]); index += 1; continue; } if (arg === '--machine-id') { options.machineId = argv[index + 1] ?? ''; index += 1; continue; } if (arg === '--output-dir') { options.outputDir = argv[index + 1] ?? ''; index += 1; continue; } if (arg === '--repo-root') { options.repoRoot = argv[index + 1] ?? options.repoRoot; index += 1; continue; } throw new Error(`Unknown argument: ${arg}`); } if (!Number.isFinite(options.days) || options.days <= 0) { throw new Error('--days must be a positive integer.'); } if (!Number.isFinite(options.limit) || options.limit <= 0) { throw new Error('--limit must be a positive integer.'); } const includeSources = new Set(); for (const source of options.includeSources) { if (source === 'all') { DEFAULT_SOURCES.forEach((entry) => includeSources.add(entry)); continue; } if (!DEFAULT_SOURCES.includes(source)) { throw new Error(`Unsupported source: ${source}`); } includeSources.add(source); } options.includeSources = includeSources.size === 0 ? [...DEFAULT_SOURCES] : [...includeSources]; options.repoRoot = path.resolve(options.repoRoot); const normalizedExcludes = [options.repoRoot, ...options.excludeWorkspaces] .map((entry) => path.resolve(entry)) .filter((entry, index, entries) => entry && entries.indexOf(entry) === index); options.excludeWorkspaces = normalizedExcludes; return options; } function ensureDir(dirPath) { fs.mkdirSync(dirPath, { recursive: true }); } function sanitizeMachineId(input) { const source = input || os.hostname() || 'local-machine'; const sanitized = source .toLowerCase() .replace(/[^a-z0-9._-]+/g, '-') .replace(/^-+|-+$/g, ''); return sanitized || 'local-machine'; } function buildTimestamp(now) { return now.toISOString().replace(/:/g, '-'); } function getSinceTimestamp(options, now) { if (options.since) { const parsed = Date.parse(options.since); if (Number.isNaN(parsed)) { throw new Error(`Invalid --since timestamp: ${options.since}`); } return parsed; } return now.getTime() - options.days * 24 * 60 * 60 * 1000; } function readJsonFile(filePath) { try { return JSON.parse(fs.readFileSync(filePath, 'utf8')); } catch { return null; } } function safeStat(filePath) { try { return fs.statSync(filePath); } catch { return null; } } function fileUriToPath(uri) { try { return fileURLToPath(uri); } catch { return null; } } function listDirectories(rootPath) { if (!fs.existsSync(rootPath)) { return []; } return fs .readdirSync(rootPath, { withFileTypes: true }) .filter((entry) => entry.isDirectory()) .map((entry) => entry.name) .sort(); } function isExcludedWorkspace(folderPath, excludedRoots) { return excludedRoots.some((excludedRoot) => { if (!excludedRoot) { return false; } return folderPath === excludedRoot || folderPath.startsWith(`${excludedRoot}${path.sep}`); }); } function cleanTsvCell(value) { return String(value ?? '').replace(/[\t\n\r]+/g, ' ').trim(); } function slugify(value) { return String(value ?? '') .toLowerCase() .replace(/[^a-z0-9]+/g, '-') .replace(/^-+|-+$/g, '') .slice(0, 64) || 'candidate'; } function compactText(value, maxLength = 88) { const normalized = String(value ?? '').replace(/\s+/g, ' ').trim(); if (normalized.length <= maxLength) { return normalized; } return `${normalized.slice(0, maxLength - 3)}...`; } function formatIso(value) { return new Date(value).toISOString(); } function formatMetric(value) { return value === null || value === undefined ? 'n/a' : String(value); } function escapeMarkdownTable(value) { return cleanTsvCell(value).replace(/\|/g, '\\|'); } function wrapCodeFence(value) { return ['```text', value.trimEnd(), '```'].join('\n'); } function formatOutcomeCounts(historyEntries) { const counts = new Map(); for (const entry of historyEntries ?? []) { counts.set(entry.outcome, (counts.get(entry.outcome) ?? 0) + 1); } return [...counts.entries()].map(([outcome, count]) => `${outcome}=${count}`); } function describeCandidateBenefit(candidate) { switch (candidate.suggestion) { case 'promote-skill': return 'A shared skill preserves a reusable workflow so you do not have to restate the same process in fresh prompts or per-repo notes.'; case 'promote-instruction': return 'A shared instruction turns repeated guidance into a durable rule set that applies across future sessions.'; case 'promote-agent': return 'A shared agent packages this behavior as a repeatable working mode instead of relying on ad hoc prompt wording.'; case 'promote-hook': return 'A shared hook can enforce or observe the workflow automatically instead of relying on manual steps.'; case 'promote-script': return 'A shared script moves repeated operational work out of chat and into deterministic automation.'; case 'promote-prompt': return 'A shared prompt creates a stable entrypoint for a workflow you already reach for, reducing prompt rewriting.'; case 'template-only': return 'A shared template captures the structure of the pattern without forcing it into a runtime resource prematurely.'; case 'docs-only': return 'Documentation keeps the insight discoverable even when it is not stable enough to become a shared runtime artifact.'; default: return 'This candidate may represent a reusable pattern worth keeping in a more discoverable shared form.'; } } function describeCandidateContext(candidate, evidence) { if (candidate.sourceType === 'publish-log') { const outcomeCounts = formatOutcomeCounts(candidate.historyEntries); const lines = [ 'This candidate was selected from actual publish history, which is stronger evidence than a one-off prompt or note.', `The same artifact fingerprint was processed ${candidate.recurrence} time(s) during the audit window${outcomeCounts.length === 0 ? '.' : ` (${outcomeCounts.join(', ')}).`}`, `That means you kept returning to the same shared-${candidate.category} target rather than inventing a new one each time.`, ]; if (candidate.sourceRefs[0]) { lines.push(`The publish target recorded in history was ${candidate.sourceRefs[0]}.`); } if (!evidence.evidencePath) { lines.push('The current target path is not available locally now, so this bundle is explaining a historical reuse signal rather than showing a live artifact snapshot.'); } return lines; } if (candidate.sourceType === 'repo-memory') { return [ 'This candidate came from persisted repo memory, which means it was important enough to save as a reusable note during prior work.', `It appeared in ${candidate.workspaceCount} workspace(s) during the audit window and remained present as stored working context.`, ]; } return [ 'This candidate came from repeated user request patterns in persisted transcripts.', `It recurred ${candidate.recurrence} time(s) across ${candidate.workspaceCount} workspace(s), which suggests a workflow you may want to standardize.`, ]; } function buildScoreExplanation(candidate) { const portabilityLines = [`Base portability starts at ${portabilityBaseForSuggestion(candidate.suggestion)} for ${candidate.suggestion}.`]; if (candidate.sourceType === 'publish-log') { portabilityLines.push('+18 because publish history is strong proof that the pattern already became a shared artifact at least once.'); } if (candidate.workspaceCount > 1) { portabilityLines.push(`+8 because it showed up across ${candidate.workspaceCount} workspaces.`); } if (candidate.recurrence > 1) { portabilityLines.push(`+5 because it recurred ${candidate.recurrence} times in the audit window.`); } if (candidate.repoSpecific) { portabilityLines.push('-20 because the content looks repo-specific or machine-specific.'); } const maturityBase = candidate.sourceType === 'publish-log' ? 90 : candidate.sourceType === 'repo-memory' ? 70 : 55; const maturityLines = [`Base maturity starts at ${maturityBase} for ${candidate.sourceType}.`]; if (candidate.recurrence > 1) { maturityLines.push('+5 recurrence bonus because the same pattern came back more than once.'); } const recurrenceScore = Math.min(100, candidate.recurrence * 20); const workspaceScore = Math.min(100, candidate.workspaceCount * 25); const formulaText = `round(${candidate.portabilityScore}*0.45 + ${candidate.maturityScore}*0.20 + ${recurrenceScore}*0.20 + ${workspaceScore}*0.15) = ${candidate.valueRank}`; return { portabilityLines, maturityLines, recurrenceScore, workspaceScore, formulaText, }; } function describeTokenCostSignal(candidate) { if (candidate.sourceType !== 'transcript') { return [ 'This candidate does not come from transcript prompt text, so no prompt-length cost proxy was computed.', ]; } return [ 'This is a prompt-length proxy derived from persisted user message characters, not exact billed token usage.', `Average prompt chars: ${formatMetric(candidate.avgPromptChars)}`, `Repeated prompt chars in the audit window: ${formatMetric(candidate.repeatedPromptChars)}`, `Signal: ${candidate.tokenCostSignal}`, ]; } function buildCandidateCaveats(candidate, evidence) { const caveats = []; if (!evidence.evidencePath) { caveats.push('No local artifact preview was available, so the review should rely more on audit context and scoring than on content inspection.'); } if (candidate.workspaceCount === 0 && candidate.sourceType === 'publish-log') { caveats.push('Workspace count is zero because publish-log candidates are repo-level events, not workspace-indexed transcript or memory events.'); } if (candidate.repoSpecific) { caveats.push('The scoring model detected repo-specific or machine-specific signals, so promote carefully.'); } if (candidate.sourceType === 'transcript') { caveats.push('Prompt-size fields are character-count proxies from persisted transcript text, not exact token billing data.'); } return caveats; } function buildHistoryLines(candidate) { if (candidate.sourceType !== 'publish-log' || !candidate.historyEntries?.length) { return []; } return candidate.historyEntries.map((entry) => { const sourcePart = entry.source ? ` :: source=${entry.source}` : ''; return `- ${entry.timestamp} :: ${entry.outcome}${sourcePart}`; }); } function readFirstExistingEvidencePath(candidate) { for (const sourceRef of candidate.sourceRefs) { if (!sourceRef) { continue; } if (candidate.sourceType === 'publish-log') { const stat = safeStat(sourceRef); if (stat?.isDirectory()) { const skillFile = path.join(sourceRef, 'SKILL.md'); if (fs.existsSync(skillFile)) { return skillFile; } } } if (fs.existsSync(sourceRef)) { return sourceRef; } } return null; } function buildTranscriptEvidence(filePath) { const userMessages = []; const lines = fs.readFileSync(filePath, 'utf8').split(/\r?\n/).filter(Boolean); for (const line of lines) { try { const event = JSON.parse(line); if (event.type === 'user.message' && typeof event.data?.content === 'string') { userMessages.push(compactText(event.data.content, 220)); } } catch { continue; } if (userMessages.length >= 3) { break; } } if (userMessages.length === 0) { return 'No transcript evidence preview was available.'; } return userMessages.map((message, index) => `${index + 1}. ${message}`).join('\n'); } function buildFileEvidence(filePath) { const lines = fs .readFileSync(filePath, 'utf8') .split(/\r?\n/) .map((line) => line.trimEnd()) .filter((line) => line.trim()) .slice(0, 20); if (lines.length === 0) { return 'No evidence preview was available.'; } return lines.join('\n'); } function buildEvidenceBundle(candidate) { const evidencePath = readFirstExistingEvidencePath(candidate); if (!evidencePath) { return { evidencePath: null, preview: 'No local evidence preview was available for this candidate.', }; } const preview = candidate.sourceType === 'transcript' ? buildTranscriptEvidence(evidencePath) : buildFileEvidence(evidencePath); return { evidencePath, preview }; } function writeCandidateDetailFiles(detailsDir, candidates) { ensureDir(detailsDir); for (const candidate of candidates) { const detailFileName = `${candidate.id}.md`; const detailFilePath = path.join(detailsDir, detailFileName); const detailFileRelativePath = path.join('pattern-details', detailFileName); const evidence = buildEvidenceBundle(candidate); const scoreExplanation = buildScoreExplanation(candidate); const caveats = buildCandidateCaveats(candidate, evidence); const historyLines = buildHistoryLines(candidate); const lines = [ `# ${candidate.title}`, '', `- Candidate ID: ${candidate.id}`, `- Suggested action: ${candidate.suggestion}`, `- Category: ${candidate.category}`, `- Source type: ${candidate.sourceType}`, `- Value rank: ${candidate.valueRank}`, `- Portability score: ${candidate.portabilityScore}`, `- Maturity score: ${candidate.maturityScore}`, `- Recurrence: ${candidate.recurrence}`, `- Workspace count: ${candidate.workspaceCount}`, `- First seen: ${candidate.firstSeen}`, `- Last seen: ${candidate.lastSeen}`, `- Token cost signal: ${candidate.tokenCostSignal}`, `- Avg prompt chars: ${formatMetric(candidate.avgPromptChars)}`, `- Repeated prompt chars: ${formatMetric(candidate.repeatedPromptChars)}`, '', ]; if (evidence.evidencePath) { lines.push( '## Evidence Path', '', `- ${evidence.evidencePath}`, '', ); } lines.push( '## Recommendation', '', `Suggested outcome: ${candidate.suggestion}`, '', '## Potential Benefit', '', describeCandidateBenefit(candidate), '', '## Audit Context', '', ...describeCandidateContext(candidate, evidence), '', '## Token Cost Signal', '', ...describeTokenCostSignal(candidate), '', '## Why It Ranked This Highly', '', `- Portability score: ${candidate.portabilityScore}`, ...scoreExplanation.portabilityLines.map((line) => ` - ${line}`), `- Maturity score: ${candidate.maturityScore}`, ...scoreExplanation.maturityLines.map((line) => ` - ${line}`), `- Recurrence score contribution uses ${scoreExplanation.recurrenceScore} from ${candidate.recurrence} occurrence(s).`, `- Workspace spread contribution uses ${scoreExplanation.workspaceScore} from ${candidate.workspaceCount} workspace(s).`, `- Value rank formula: ${scoreExplanation.formulaText}`, '', '## Notes', '', candidate.notes || 'No additional notes were captured.', '', '## Review Caveats', '', ...(caveats.length === 0 ? ['- No material caveats were detected for this candidate.'] : caveats.map((line) => `- ${line}`)), '', '## Source References', '', ...candidate.sourceRefs.map((sourceRef) => `- ${sourceRef}`), '', '## History', '', ...(historyLines.length === 0 ? ['- No additional event history was captured for this candidate type.'] : historyLines), '', '## Evidence Preview', '', wrapCodeFence(evidence.preview), ); fs.writeFileSync(detailFilePath, `${lines.join('\n')}\n`, 'utf8'); candidate.detailFile = detailFileRelativePath; } } function buildWorkspaceIndex(rootPath, workspaceFilter, excludedRoots) { const workspaces = []; for (const storageId of listDirectories(rootPath)) { const storagePath = path.join(rootPath, storageId); const workspaceJson = readJsonFile(path.join(storagePath, 'workspace.json')); if (!workspaceJson?.folder) { continue; } const folderPath = fileUriToPath(workspaceJson.folder); if (!folderPath) { continue; } if (isExcludedWorkspace(folderPath, excludedRoots)) { continue; } if ( workspaceFilter && !folderPath.includes(workspaceFilter) && !storageId.includes(workspaceFilter) && !path.basename(folderPath).includes(workspaceFilter) ) { continue; } workspaces.push({ folderPath, name: path.basename(folderPath), storageId, storagePath, }); } return workspaces.sort((left, right) => left.name.localeCompare(right.name)); } function normalizePromptText(content) { return content .toLowerCase() .replace(/[`*_>#]/g, ' ') .replace(/[^a-z0-9/ ._-]+/g, ' ') .replace(/\s+/g, ' ') .trim(); } function hasTranscriptKeyword(content) { const normalized = normalizePromptText(content); return TRANSCRIPT_KEYWORDS.some((keyword) => normalized.includes(keyword)); } function extractMarkdownTitle(content, fallback) { const lines = content.split(/\r?\n/); for (const line of lines) { const trimmed = line.trim(); if (trimmed.startsWith('# ')) { return trimmed.slice(2).trim(); } } for (const line of lines) { const trimmed = line.trim(); if (trimmed && !trimmed.startsWith('-') && !trimmed.startsWith('*')) { return trimmed; } } return fallback; } function inferSuggestion(text) { const lower = text.toLowerCase(); if (/(prompt)/.test(lower)) { return 'promote-prompt'; } if (/(instruction|rule|standard|policy|checklist)/.test(lower)) { return 'promote-instruction'; } if (/(agent)/.test(lower)) { return 'promote-agent'; } if (/(hook)/.test(lower)) { return 'promote-hook'; } if (/(script|shell|cli)/.test(lower)) { return 'promote-script'; } if (/(template|overlay)/.test(lower)) { return 'template-only'; } if (/(skill|workflow|audit|bootstrap|publish|setup)/.test(lower)) { return 'promote-skill'; } if (/(guide|reference|docs|architecture|notes?)/.test(lower)) { return 'docs-only'; } return 'docs-only'; } function inferCategory(suggestion) { if (suggestion.startsWith('promote-')) { return suggestion.slice('promote-'.length); } if (suggestion === 'template-only') { return 'template'; } if (suggestion === 'docs-only') { return 'documentation'; } return 'candidate'; } function likelyRepoSpecific(text, workspaceNames) { const lower = text.toLowerCase(); if (/\/users\/|library\/application support|developer\//.test(lower)) { return true; } return workspaceNames.some((name) => { const normalized = name.toLowerCase(); return normalized.length > 4 && lower.includes(normalized); }); } function portabilityBaseForSuggestion(suggestion) { switch (suggestion) { case 'promote-skill': return 78; case 'promote-instruction': return 74; case 'promote-agent': return 70; case 'promote-hook': return 66; case 'promote-script': return 68; case 'promote-prompt': return 58; case 'template-only': return 60; case 'docs-only': return 52; default: return 45; } } function computePortabilityScore({ sourceType, suggestion, recurrence, workspaceCount, repoSpecific }) { let score = portabilityBaseForSuggestion(suggestion); if (sourceType === 'publish-log') { score += 18; } if (workspaceCount > 1) { score += 8; } if (recurrence > 1) { score += 5; } if (repoSpecific) { score -= 20; } return Math.max(0, Math.min(100, score)); } function computeMaturityScore(sourceType, recurrence) { let score = 55; if (sourceType === 'publish-log') { score = 90; } else if (sourceType === 'repo-memory') { score = 70; } if (recurrence > 1) { score += 5; } return Math.min(100, score); } function computeValueRank({ portabilityScore, maturityScore, recurrence, workspaceCount }) { const recurrenceScore = Math.min(100, recurrence * 20); const workspaceScore = Math.min(100, workspaceCount * 25); return Math.min( 100, Math.round( portabilityScore * 0.45 + maturityScore * 0.2 + recurrenceScore * 0.2 + workspaceScore * 0.15, ), ); } function defaultTokenCostMetrics() { return { avgPromptChars: null, repeatedPromptChars: null, tokenCostSignal: 'n/a', }; } function computeTranscriptTokenCostMetrics({ recurrence, promptCharsTotal, promptCharsMax }) { if (!recurrence || !promptCharsTotal) { return defaultTokenCostMetrics(); } const avgPromptChars = Math.round(promptCharsTotal / recurrence); let tokenCostSignal = 'low'; if (promptCharsTotal >= 2400 || avgPromptChars >= 360 || promptCharsMax >= 500) { tokenCostSignal = 'very-high'; } else if (promptCharsTotal >= 1200 || avgPromptChars >= 220 || promptCharsMax >= 320) { tokenCostSignal = 'high'; } else if (promptCharsTotal >= 500 || avgPromptChars >= 120 || promptCharsMax >= 180) { tokenCostSignal = 'medium'; } return { avgPromptChars, repeatedPromptChars: promptCharsTotal, tokenCostSignal, }; } function tokenCostSignalRank(signal) { switch (signal) { case 'very-high': return 4; case 'high': return 3; case 'medium': return 2; case 'low': return 1; default: return 0; } } function titleFromTarget(kind, targetPath) { const base = path.basename(targetPath); if (kind === 'skill') { return base; } return base .replace(/\.prompt\.md$/i, '') .replace(/\.instructions\.md$/i, '') .replace(/\.agent\.md$/i, '') .replace(/\.json$/i, ''); } function collectPublishLogCandidates(filePath, sinceTimestamp, excludedRoots) { const stats = { candidateCount: 0, scannedEntries: 0, }; if (!fs.existsSync(filePath)) { return { candidates: [], stats }; } const groups = new Map(); const lines = fs.readFileSync(filePath, 'utf8').split(/\r?\n/).filter(Boolean); for (const line of lines) { const [timestamp, kind, source, target, origin, fingerprint, outcome] = line.split('\t'); const eventTime = Date.parse(timestamp); if (Number.isNaN(eventTime) || eventTime < sinceTimestamp) { continue; } if (target && isExcludedWorkspace(path.resolve(target), excludedRoots)) { continue; } stats.scannedEntries += 1; const key = `${kind}\t${target}\t${fingerprint}`; if (!groups.has(key)) { groups.set(key, { fingerprint, firstSeen: eventTime, historyEntries: [], kind, lastSeen: eventTime, origin, outcomes: new Map(), recurrence: 0, sourceRefs: new Set(), target, }); } const group = groups.get(key); group.recurrence += 1; group.firstSeen = Math.min(group.firstSeen, eventTime); group.lastSeen = Math.max(group.lastSeen, eventTime); group.sourceRefs.add(target); group.historyEntries.push({ origin, outcome, source, target, timestamp: formatIso(eventTime), }); group.outcomes.set(outcome, (group.outcomes.get(outcome) ?? 0) + 1); } const candidates = [...groups.values()].map((group) => { const suggestion = `promote-${group.kind}`; const workspaceNames = []; const portabilityScore = computePortabilityScore({ sourceType: 'publish-log', suggestion, recurrence: group.recurrence, workspaceCount: 0, repoSpecific: false, }); const maturityScore = computeMaturityScore('publish-log', group.recurrence); const valueRank = computeValueRank({ portabilityScore, maturityScore, recurrence: group.recurrence, workspaceCount: 0, }); const outcomes = [...group.outcomes.entries()] .map(([outcome, count]) => `${outcome}=${count}`) .join(', '); return { id: `publish-${slugify(`${group.kind}-${titleFromTarget(group.kind, group.target)}`)}`, sourceType: 'publish-log', category: group.kind, title: titleFromTarget(group.kind, group.target), suggestion, recurrence: group.recurrence, workspaceCount: 0, ...defaultTokenCostMetrics(), portabilityScore, maturityScore, valueRank, firstSeen: formatIso(group.firstSeen), lastSeen: formatIso(group.lastSeen), sourceRefs: [...group.sourceRefs], notes: `origin=${group.origin}; outcomes=${outcomes}`, historyEntries: group.historyEntries.sort((left, right) => left.timestamp.localeCompare(right.timestamp)), repoSpecific: false, workspaceNames, }; }); stats.candidateCount = candidates.length; return { candidates, stats }; } function collectRepoMemoryCandidates(workspaces, sinceTimestamp) { const stats = { candidateCount: 0, scannedFiles: 0, }; const groups = new Map(); for (const workspace of workspaces) { const repoMemoryDir = path.join( workspace.storagePath, 'GitHub.copilot-chat', 'memory-tool', 'memories', 'repo', ); if (!fs.existsSync(repoMemoryDir)) { continue; } for (const entry of fs.readdirSync(repoMemoryDir, { withFileTypes: true })) { if (!entry.isFile() || !entry.name.endsWith('.md')) { continue; } const filePath = path.join(repoMemoryDir, entry.name); const fileStat = safeStat(filePath); if (!fileStat || fileStat.mtimeMs < sinceTimestamp) { continue; } stats.scannedFiles += 1; const content = fs.readFileSync(filePath, 'utf8'); const fallbackTitle = entry.name.replace(/\.md$/i, '').replace(/-/g, ' '); const title = extractMarkdownTitle(content, fallbackTitle); const key = slugify(entry.name.replace(/\.md$/i, '')); if (!groups.has(key)) { groups.set(key, { contentSamples: [], firstSeen: fileStat.mtimeMs, lastSeen: fileStat.mtimeMs, paths: new Set(), recurrence: 0, title, workspaceNames: new Set(), }); } const group = groups.get(key); group.recurrence += 1; group.firstSeen = Math.min(group.firstSeen, fileStat.mtimeMs); group.lastSeen = Math.max(group.lastSeen, fileStat.mtimeMs); group.paths.add(filePath); group.workspaceNames.add(workspace.name); if (group.contentSamples.length < 2) { group.contentSamples.push(content); } } } const candidates = [...groups.values()].map((group) => { const sampleText = group.contentSamples.join(' '); const suggestion = inferSuggestion(`${group.title} ${sampleText}`); const workspaceNames = [...group.workspaceNames].sort(); const repoSpecific = likelyRepoSpecific(`${group.title} ${sampleText}`, workspaceNames); const portabilityScore = computePortabilityScore({ sourceType: 'repo-memory', suggestion, recurrence: group.recurrence, workspaceCount: workspaceNames.length, repoSpecific, }); const maturityScore = computeMaturityScore('repo-memory', group.recurrence); const valueRank = computeValueRank({ portabilityScore, maturityScore, recurrence: group.recurrence, workspaceCount: workspaceNames.length, }); return { id: `memory-${slugify(group.title)}`, sourceType: 'repo-memory', category: inferCategory(suggestion), title: compactText(group.title), suggestion, recurrence: group.recurrence, workspaceCount: workspaceNames.length, ...defaultTokenCostMetrics(), portabilityScore, maturityScore, valueRank, firstSeen: formatIso(group.firstSeen), lastSeen: formatIso(group.lastSeen), sourceRefs: [...group.paths].sort(), notes: `repo memories from ${workspaceNames.join(', ') || 'unknown workspaces'}`, historyEntries: [], repoSpecific, workspaceNames, }; }); stats.candidateCount = candidates.length; return { candidates, stats }; } function collectTranscriptCandidates(workspaces, sinceTimestamp) { const stats = { candidateCount: 0, scannedPrompts: 0, scannedSessions: 0, }; const groups = new Map(); for (const workspace of workspaces) { const transcriptDir = path.join(workspace.storagePath, 'GitHub.copilot-chat', 'transcripts'); if (!fs.existsSync(transcriptDir)) { continue; } for (const entry of fs.readdirSync(transcriptDir, { withFileTypes: true })) { if (!entry.isFile() || !entry.name.endsWith('.jsonl')) { continue; } const transcriptPath = path.join(transcriptDir, entry.name); const sessionIdsSeen = new Set(); let sessionStartTimestamp = null; const lines = fs.readFileSync(transcriptPath, 'utf8').split(/\r?\n/).filter(Boolean); const events = []; for (const line of lines) { try { events.push(JSON.parse(line)); } catch { continue; } } for (const event of events) { if (event.type === 'session.start') { sessionStartTimestamp = Date.parse(event.data?.startTime ?? event.timestamp); break; } } if (sessionStartTimestamp === null || Number.isNaN(sessionStartTimestamp) || sessionStartTimestamp < sinceTimestamp) { continue; } stats.scannedSessions += 1; for (const event of events) { if (event.type !== 'user.message' || typeof event.data?.content !== 'string') { continue; } const content = event.data.content.trim(); if (content.length < 24 || content.length > 500 || !hasTranscriptKeyword(content)) { continue; } const normalized = normalizePromptText(content); if (!normalized || sessionIdsSeen.has(normalized)) { continue; } sessionIdsSeen.add(normalized); stats.scannedPrompts += 1; if (!groups.has(normalized)) { groups.set(normalized, { firstSeen: sessionStartTimestamp, lastSeen: sessionStartTimestamp, promptCharsMax: 0, promptCharsTotal: 0, recurrence: 0, sample: compactText(content, 100), sourceRefs: new Set(), workspaceNames: new Set(), }); } const group = groups.get(normalized); const promptChars = content.length; group.recurrence += 1; group.firstSeen = Math.min(group.firstSeen, sessionStartTimestamp); group.lastSeen = Math.max(group.lastSeen, sessionStartTimestamp); group.promptCharsMax = Math.max(group.promptCharsMax, promptChars); group.promptCharsTotal += promptChars; group.sourceRefs.add(transcriptPath); group.workspaceNames.add(workspace.name); } } } const candidates = [...groups.values()] .filter((group) => group.recurrence >= 2 || group.workspaceNames.size >= 2) .map((group) => { const workspaceNames = [...group.workspaceNames].sort(); const suggestion = inferSuggestion(group.sample); const repoSpecific = likelyRepoSpecific(group.sample, workspaceNames); const tokenCostMetrics = computeTranscriptTokenCostMetrics({ recurrence: group.recurrence, promptCharsTotal: group.promptCharsTotal, promptCharsMax: group.promptCharsMax, }); const portabilityScore = computePortabilityScore({ sourceType: 'transcript', suggestion, recurrence: group.recurrence, workspaceCount: workspaceNames.length, repoSpecific, }); const maturityScore = computeMaturityScore('transcript', group.recurrence); const valueRank = computeValueRank({ portabilityScore, maturityScore, recurrence: group.recurrence, workspaceCount: workspaceNames.length, }); return { id: `transcript-${slugify(group.sample)}`, sourceType: 'transcript', category: inferCategory(suggestion), title: group.sample, suggestion, recurrence: group.recurrence, workspaceCount: workspaceNames.length, ...tokenCostMetrics, portabilityScore, maturityScore, valueRank, firstSeen: formatIso(group.firstSeen), lastSeen: formatIso(group.lastSeen), sourceRefs: [...group.sourceRefs].sort(), notes: 'repeated user request pattern from persisted transcripts', historyEntries: [], repoSpecific, workspaceNames, }; }); stats.candidateCount = candidates.length; return { candidates, stats }; } function writeWorkspaceIndex(filePath, workspaces) { const lines = ['storage_id\tworkspace_name\tworkspace_path']; for (const workspace of workspaces) { lines.push( [workspace.storageId, workspace.name, workspace.folderPath] .map(cleanTsvCell) .join('\t'), ); } fs.writeFileSync(filePath, `${lines.join('\n')}\n`, 'utf8'); } function writeCandidatesReport(filePath, candidates) { const header = [ 'id', 'source_type', 'suggested_action', 'category', 'title', 'detail_file', 'recurrence', 'workspace_count', 'token_cost_signal', 'avg_prompt_chars', 'repeated_prompt_chars', 'portability_score', 'maturity_score', 'value_rank', 'first_seen', 'last_seen', 'source_refs', 'notes', ]; const lines = [header.join('\t')]; for (const candidate of candidates) { lines.push( [ candidate.id, candidate.sourceType, candidate.suggestion, candidate.category, candidate.title, candidate.detailFile ?? '', candidate.recurrence, candidate.workspaceCount, candidate.tokenCostSignal, candidate.avgPromptChars, candidate.repeatedPromptChars, candidate.portabilityScore, candidate.maturityScore, candidate.valueRank, candidate.firstSeen, candidate.lastSeen, candidate.sourceRefs.join(' | '), candidate.notes, ] .map(cleanTsvCell) .join('\t'), ); } fs.writeFileSync(filePath, `${lines.join('\n')}\n`, 'utf8'); } function writeSelectionManifest(filePath, candidates) { const header = [ 'decision', 'review_note', 'id', 'suggested_action', 'category', 'title', 'detail_file', 'value_rank', 'portability_score', 'recurrence', 'workspace_count', 'token_cost_signal', 'avg_prompt_chars', 'repeated_prompt_chars', 'first_seen', 'last_seen', 'source_type', 'notes', ]; const lines = [header.join('\t')]; for (const candidate of candidates) { lines.push( [ '', '', candidate.id, candidate.suggestion, candidate.category, candidate.title, candidate.detailFile ?? '', candidate.valueRank, candidate.portabilityScore, candidate.recurrence, candidate.workspaceCount, candidate.tokenCostSignal, candidate.avgPromptChars, candidate.repeatedPromptChars, candidate.firstSeen, candidate.lastSeen, candidate.sourceType, candidate.notes, ] .map(cleanTsvCell) .join('\t'), ); } fs.writeFileSync(filePath, `${lines.join('\n')}\n`, 'utf8'); } function writeSummary(filePath, context) { const topCandidates = context.candidates.slice(0, 10); const transcriptCostCandidates = context.candidates .filter((candidate) => tokenCostSignalRank(candidate.tokenCostSignal) > 0) .sort( (left, right) => tokenCostSignalRank(right.tokenCostSignal) - tokenCostSignalRank(left.tokenCostSignal) || (right.repeatedPromptChars ?? 0) - (left.repeatedPromptChars ?? 0) || right.recurrence - left.recurrence, ) .slice(0, 5); const lines = [ '# Copilot Reuse Audit', '', `- Generated: ${context.generatedAt}`, `- Repo root: ${context.repoRoot}`, `- Machine id: ${context.machineId}`, `- Time window start: ${context.since}`, `- Time window end: ${context.until}`, `- Workspace filter: ${context.workspaceFilter || '(all indexed workspaces)'}`, `- Excluded workspace roots: ${context.excludeWorkspaces.length === 0 ? '(none)' : context.excludeWorkspaces.join(', ')}`, `- Included sources: ${context.includeSources.join(', ')}`, `- Output directory: ${context.outputDir}`, '', '## Inventory', '', `- Indexed workspaces: ${context.workspaceCount}`, `- Publish log entries in window: ${context.sourceStats.publishLog.scannedEntries}`, `- Repo memory files in window: ${context.sourceStats.repoMemories.scannedFiles}`, `- Transcript sessions in window: ${context.sourceStats.transcripts.scannedSessions}`, `- Transcript prompts considered: ${context.sourceStats.transcripts.scannedPrompts}`, `- Candidates emitted: ${context.candidates.length}`, `- Transcript candidates with prompt-cost signal: ${context.candidates.filter((candidate) => tokenCostSignalRank(candidate.tokenCostSignal) > 0).length}`, '', '## Top Candidates', '', '| ID | Title | Suggested action | Source | Value rank |', '| --- | --- | --- | --- | ---: |', ]; if (topCandidates.length === 0) { lines.push('| (none) | No candidates were detected in the selected window. | docs-only | audit | 0 |'); } else { for (const candidate of topCandidates) { lines.push( `| ${escapeMarkdownTable(candidate.id)} | ${escapeMarkdownTable(candidate.title)} | ${escapeMarkdownTable(candidate.suggestion)} | ${escapeMarkdownTable(candidate.sourceType)} | ${candidate.valueRank} |`, ); } } lines.push( '', '## Prompt Cost Signals', '', '| ID | Title | Signal | Avg chars | Repeated chars |', '| --- | --- | --- | ---: | ---: |', ); if (transcriptCostCandidates.length === 0) { lines.push('| (none) | No transcript prompt-cost signals were detected in the selected window. | n/a | 0 | 0 |'); } else { for (const candidate of transcriptCostCandidates) { lines.push( `| ${escapeMarkdownTable(candidate.id)} | ${escapeMarkdownTable(candidate.title)} | ${escapeMarkdownTable(candidate.tokenCostSignal)} | ${formatMetric(candidate.avgPromptChars)} | ${formatMetric(candidate.repeatedPromptChars)} |`, ); } } lines.push( '', '## Files', '', '- pattern-details/', '- workspace-index.tsv', '- candidates-report.tsv', '- selection-manifest.tsv', '', '## Review Guidance', '', '1. Start with selection-manifest.tsv and fill in the decision column only after reviewing the linked detail file for each candidate.', '2. Prefer promote-skill, promote-instruction, promote-agent, promote-hook, promote-script, or promote-prompt only when the pattern is portable across repositories.', '3. Use the transcript prompt-cost fields as triage signals for likely expensive repeated prompts, not as exact billing data.', '4. Use template-only or docs-only for guidance that is helpful but not suitable as a shared runtime resource.', '5. Use discard or needs-sanitization when the candidate contains secrets, machine-specific paths, or repo-specific assumptions.', '', ); fs.writeFileSync(filePath, `${lines.join('\n')}\n`, 'utf8'); } function main() { const options = parseArgs(process.argv.slice(2)); const now = new Date(); const sinceTimestamp = getSinceTimestamp(options, now); const machineId = sanitizeMachineId(options.machineId); const timestamp = buildTimestamp(now); const outputDir = options.outputDir ? path.resolve(options.outputDir) : path.join(options.repoRoot, '.local', 'audits', machineId, timestamp); ensureDir(outputDir); const workspaceStorageRoot = process.env.COPILOT_AUDIT_WORKSPACE_STORAGE_ROOT || path.join(os.homedir(), 'Library', 'Application Support', 'Code', 'User', 'workspaceStorage'); const publishLogPath = path.join( process.env.COPILOT_RESOURCES_STATE_DIR || path.join(os.homedir(), '.copilot-resources-state'), 'publish-log.tsv', ); const workspaces = buildWorkspaceIndex( workspaceStorageRoot, options.workspaceFilter, options.excludeWorkspaces, ); writeWorkspaceIndex(path.join(outputDir, 'workspace-index.tsv'), workspaces); let candidates = []; const sourceStats = { publishLog: { candidateCount: 0, scannedEntries: 0 }, repoMemories: { candidateCount: 0, scannedFiles: 0 }, transcripts: { candidateCount: 0, scannedPrompts: 0, scannedSessions: 0 }, }; if (options.includeSources.includes('publish-log')) { const result = collectPublishLogCandidates( publishLogPath, sinceTimestamp, options.excludeWorkspaces, ); candidates = candidates.concat(result.candidates); sourceStats.publishLog = result.stats; } if (options.includeSources.includes('repo-memories')) { const result = collectRepoMemoryCandidates(workspaces, sinceTimestamp); candidates = candidates.concat(result.candidates); sourceStats.repoMemories = result.stats; } if (options.includeSources.includes('transcripts')) { const result = collectTranscriptCandidates(workspaces, sinceTimestamp); candidates = candidates.concat(result.candidates); sourceStats.transcripts = result.stats; } candidates = candidates .sort((left, right) => right.valueRank - left.valueRank || right.recurrence - left.recurrence || left.title.localeCompare(right.title)) .slice(0, options.limit); writeCandidateDetailFiles(path.join(outputDir, 'pattern-details'), candidates); writeCandidatesReport(path.join(outputDir, 'candidates-report.tsv'), candidates); writeSelectionManifest(path.join(outputDir, 'selection-manifest.tsv'), candidates); writeSummary(path.join(outputDir, 'audit-summary.md'), { candidates, generatedAt: now.toISOString(), includeSources: options.includeSources, machineId, excludeWorkspaces: options.excludeWorkspaces, outputDir, repoRoot: options.repoRoot, since: new Date(sinceTimestamp).toISOString(), sourceStats, until: now.toISOString(), workspaceCount: workspaces.length, workspaceFilter: options.workspaceFilter, }); console.log('Copilot audit complete.'); console.log(`Output directory: ${outputDir}`); console.log(`Summary: ${path.join(outputDir, 'audit-summary.md')}`); console.log(`Candidates: ${path.join(outputDir, 'candidates-report.tsv')}`); console.log(`Selection manifest: ${path.join(outputDir, 'selection-manifest.tsv')}`); } main();