Glassworm style attacks added to repo policy check

.repolicy.json

@@ -3,6 +3,7 @@
     "ts/packages/typechat/",
     "ts/pnpm-lock.yaml",
     "docs/pnpm-lock.yaml",
-    "ts/pnpm-workspace.yaml"
+    "ts/pnpm-workspace.yaml",
+    "ts/examples/websiteAliases/cache/"
   ]
 }

ts/package.json

@@ -61,7 +61,7 @@
     "prettier": "^3.5.3",
     "shx": "^0.4.0"
   },
-  "packageManager": "pnpm@10.30.3+sha512.c961d1e0a2d8e354ecaa5166b822516668b7f44cb5bd95122d590dd81922f606f5473b6d23ec4a5be05e7fcd18e8488d47d978bbe981872f1145d06e9a740017",
+  "packageManager": "pnpm@10.32.1+sha512.a706938f0e89ac1456b6563eab4edf1d1faf3368d1191fc5c59790e96dc918e4456ab2e67d613de1043d2e8c81f87303e6b40d4ffeca9df15ef1ad567348f2be",
   "engines": {
     "node": ">=20",
     "pnpm": ">=10"

ts/packages/defaultAgentProvider/data/config.test.json

@@ -0,0 +1,33 @@
+{
+  "description": "Test configuration - only includes agents needed for shell integration tests",
+  "agents": {
+    "calendar": {
+      "name": "calendar",
+      "execMode": "dispatcher"
+    },
+    "list": {
+      "name": "list-agent",
+      "execMode": "dispatcher"
+    },
+    "chat": {
+      "name": "chat-agent",
+      "execMode": "dispatcher"
+    },
+    "greeting": {
+      "name": "greeting-agent",
+      "execMode": "dispatcher"
+    }
+  },
+  "explainers": {
+    "v5": {
+      "constructions": {
+        "data": ["./data/explainer/v5/data/player/basic.json"],
+        "file": "./data/explainer/v5/constructions.json"
+      }
+    }
+  },
+  "tests": [
+    "./test/data/explanations/**/**/*.json",
+    "./test/repo/explanations/**/**/*.json"
+  ]
+}

ts/packages/dispatcher/dispatcher/src/utils/fsUtils.ts

@@ -49,6 +49,9 @@ export async function lockInstanceDir(instanceDir: string) {
             isExiting = true;
         });
         return await lockfile.lock(instanceDir, {
+            // Retry for up to ~15 seconds to handle the case where a previous
+            // process was forcibly killed and its lock file is not yet stale.
+            retries: { retries: 15, minTimeout: 1000, maxTimeout: 1000 },
             onCompromised: (err) => {
                 if (isExiting) {
                     // We are exiting, just ignore the error

ts/packages/shell/package.json

@@ -41,7 +41,7 @@
     "prettier": "prettier --check . --ignore-path ../../.prettierignore",
     "prettier:fix": "prettier --write . --ignore-path ../../.prettierignore",
     "shell:smoke": "npx playwright test simple.spec.ts",
-    "shell:test": "npx playwright test",
+    "shell:test": "pnpm run jest-esm && npx playwright test",
     "start": "npm run prepare-vite && electron-vite preview -- --env ../../.env",
     "start:connect": "npm run prepare-vite && electron-vite preview -- --env ../../.env --connect",
     "start:nosandbox": "npm run prepare-vite && electron-vite preview --noSandbox -- --env ../../.env",

ts/packages/shell/playwright.config.ts

@@ -16,6 +16,8 @@ import { defineConfig, devices } from "@playwright/test";
  */
 export default defineConfig({
     testDir: "./test",
+    /* Exclude jest unit tests in partialCompletion – they are run separately via jest-esm */
+    testIgnore: ["**/partialCompletion/**"],
     /* Run tests sequentially otherwise the client will complain about locked session file */
     fullyParallel: false,
     /* Fail the build on CI if you accidentally left test.only in the source code. */
@@ -37,7 +39,7 @@ export default defineConfig({
     },
 
     maxFailures: 0,
-    timeout: 300_000, // Set global timeout to 120 seconds
+    timeout: 600_000, // Set global timeout to 10 minutes (for LLM-heavy tests)
 
     /* Configure projects for major browsers */
     projects: [

ts/packages/shell/src/main/instance.ts

@@ -32,7 +32,7 @@ import {
 import { getStatusSummary } from "agent-dispatcher/helpers/status";
 import { setPendingUpdateCallback } from "./commands/update.js";
 import { createClientIORpcClient } from "@typeagent/dispatcher-rpc/clientio/client";
-import { isProd } from "./index.js";
+import { isProd, isTest } from "./index.js";
 import { getFsStorageProvider } from "dispatcher-node-providers";
 import { ensureAndConnectDispatcher } from "@typeagent/agent-server-client";
 
@@ -151,13 +151,14 @@ async function initializeDispatcher(
                     "instanceDir is required when not in connect mode",
                 );
             }
+            const configName = isTest ? "test" : undefined;
             const indexingServiceRegistry =
-                await getIndexingServiceRegistry(instanceDir);
+                await getIndexingServiceRegistry(instanceDir, configName);
 
             newDispatcher = await createDispatcher("shell", {
                 appAgentProviders: [
                     createShellAgentProvider(shellWindow),
-                    ...getDefaultAppAgentProviders(instanceDir),
+                    ...getDefaultAppAgentProviders(instanceDir, configName),
                 ],
                 agentInitOptions: {
                     browser: browserControl.control,
@@ -250,7 +251,12 @@ async function initializeDispatcher(
 
         return dispatcher;
     } catch (e: any) {
-        dialog.showErrorBox("Exception initializing dispatcher", e.stack);
+        if (isTest) {
+            // In test mode, avoid blocking dialogs so the process can exit cleanly
+            console.error("Exception initializing dispatcher:", e.stack);
+        } else {
+            dialog.showErrorBox("Exception initializing dispatcher", e.stack);
+        }
         return undefined;
     }
 }

ts/packages/shell/test/testHelper.ts

@@ -114,7 +114,7 @@ async function startShell(testGreetings: boolean = false): Promise<Page> {
             await expect(inputLocator).toHaveAttribute(
                 "contenteditable",
                 "true",
-                { timeout: 30000 },
+                { timeout: 120000 },
             );
 
             return mainWindow;
@@ -270,14 +270,15 @@ export async function sendUserRequestAndWaitForResponse(
 export async function sendUserRequestAndWaitForCompletion(
     prompt: string,
     page: Page,
+    timeout: number = 90000,
 ): Promise<string> {
     const locators = await getAgentMessageLocators(page);
 
     // send the user request
     await sendUserRequest(prompt, page);
 
     // wait for agent response
-    return waitForAgentMessage(page, 30000, locators.length + 1, true);
+    return waitForAgentMessage(page, timeout, locators.length + 1, true);
 }
 
 /**

ts/tools/scripts/policyChecks/invisibleUnicode.mjs

@@ -0,0 +1,128 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+/**
+ * Invisible Unicode character scanner.
+ *
+ * Detects Unicode characters that are invisible to human reviewers but can
+ * be used to hide malicious code in source files. This covers two known
+ * attack classes:
+ *
+ *  - Trojan Source (CVE-2021-42574): Bidirectional control characters that
+ *    cause editors/diff tools to display code in a different order than it
+ *    is actually parsed and executed by the compiler/interpreter.
+ *
+ *  - GlassWorm / steganographic payloads: Zero-width and invisible Unicode
+ *    characters used to encode hidden instructions inside ordinary-looking
+ *    source files. The hidden payload is extracted at runtime by a small
+ *    decoder that is itself concealed using the same technique.
+ *
+ * References:
+ *   https://trojansource.codes/ (CVE-2021-42574)
+ *   https://www.scientificamerican.com/article/glassworm-malware-hides-in-invisible-open-source-code/
+ */
+
+// Bidirectional control characters — can reorder what editors display vs
+// what compilers see, allowing an attacker to make malicious code look like
+// an innocent comment or string literal.
+const BIDI_CHARS = [
+    { code: 0x200e, name: "LEFT-TO-RIGHT MARK (LRM)" },
+    { code: 0x200f, name: "RIGHT-TO-LEFT MARK (RLM)" },
+    { code: 0x202a, name: "LEFT-TO-RIGHT EMBEDDING (LRE)" },
+    { code: 0x202b, name: "RIGHT-TO-LEFT EMBEDDING (RLE)" },
+    { code: 0x202c, name: "POP DIRECTIONAL FORMATTING (PDF)" },
+    { code: 0x202d, name: "LEFT-TO-RIGHT OVERRIDE (LRO)" },
+    { code: 0x202e, name: "RIGHT-TO-LEFT OVERRIDE (RLO)" },
+    { code: 0x2066, name: "LEFT-TO-RIGHT ISOLATE (LRI)" },
+    { code: 0x2067, name: "RIGHT-TO-LEFT ISOLATE (RLI)" },
+    { code: 0x2068, name: "FIRST STRONG ISOLATE (FSI)" },
+    { code: 0x2069, name: "POP DIRECTIONAL ISOLATE (PDI)" },
+];
+
+// Zero-width and invisible characters — can encode steganographic payloads
+// that are completely invisible in editors, code review tools, and diffs.
+const ZERO_WIDTH_CHARS = [
+    { code: 0x200b, name: "ZERO WIDTH SPACE (ZWSP)" },
+    { code: 0x200c, name: "ZERO WIDTH NON-JOINER (ZWNJ)" },
+    { code: 0x200d, name: "ZERO WIDTH JOINER (ZWJ)" },
+    { code: 0x2060, name: "WORD JOINER" },
+    { code: 0x2061, name: "FUNCTION APPLICATION" },
+    { code: 0x2062, name: "INVISIBLE TIMES" },
+    { code: 0x2063, name: "INVISIBLE SEPARATOR" },
+    { code: 0x2064, name: "INVISIBLE PLUS" },
+    // U+FEFF is the UTF-8 BOM when it appears at byte offset 0 of a file, but
+    // is otherwise a zero-width no-break space that has no visible form.
+    { code: 0xfeff, name: "ZERO WIDTH NO-BREAK SPACE / BOM (ZWNBSP)" },
+];
+
+const ALL_SUSPICIOUS = [...BIDI_CHARS, ...ZERO_WIDTH_CHARS];
+
+// Build a single regex that matches any of the suspicious characters.
+const SUSPICIOUS_PATTERN = ALL_SUSPICIOUS.map((c) =>
+    String.fromCodePoint(c.code),
+).join("");
+const SUSPICIOUS_REGEX = new RegExp(`[${SUSPICIOUS_PATTERN}]`, "g");
+
+// Map from character → descriptor for fast lookup when reporting.
+const CHAR_INFO = new Map(
+    ALL_SUSPICIOUS.map((c) => [String.fromCodePoint(c.code), c]),
+);
+
+/**
+ * Scan a Repofile for invisible Unicode characters.
+ *
+ * Returns true if clean, or an array of human-readable error strings if any
+ * suspicious characters are found.
+ */
+function checkInvisibleUnicode(file) {
+    const content = file.content;
+    const lines = content.split("\n");
+    const errors = [];
+
+    for (let lineIdx = 0; lineIdx < lines.length; lineIdx++) {
+        const line = lines[lineIdx];
+        // Reset lastIndex for each line by creating a fresh exec loop.
+        const re = new RegExp(SUSPICIOUS_REGEX.source, "g");
+        let match;
+        while ((match = re.exec(line)) !== null) {
+            const info = CHAR_INFO.get(match[0]);
+            // U+FEFF is the UTF-8 BOM marker: tolerate it only at the very
+            // first character of the file (line 0, column 0).
+            if (info.code === 0xfeff && lineIdx === 0 && match.index === 0) {
+                continue;
+            }
+            // U+200D ZERO WIDTH JOINER is legitimately used inside emoji
+            // sequences (e.g., 👨‍💻 = 👨 + ZWJ + 💻). Non-BMP emoji are
+            // encoded as surrogate pairs in JavaScript strings, so if the
+            // character immediately before the ZWJ is a low surrogate
+            // (U+DC00–U+DFFF) we can be confident this is an emoji sequence
+            // rather than an attempt to hide content.
+            if (info.code === 0x200d && match.index > 0) {
+                const prevCode = line.charCodeAt(match.index - 1);
+                if (prevCode >= 0xdc00 && prevCode <= 0xdfff) {
+                    continue;
+                }
+            }
+            const hex = info.code.toString(16).toUpperCase().padStart(4, "0");
+            errors.push(
+                `Line ${lineIdx + 1}, col ${match.index + 1}: ` +
+                    `Invisible Unicode U+${hex} ${info.name}`,
+            );
+        }
+    }
+
+    return errors.length === 0 ? true : errors;
+}
+
+// Apply the check to all common source-code file types.
+const SOURCE_FILE_PATTERN =
+    /.*\.[cm]?[jt]sx?$|.*\.py$|.*\.cs$|.*\.ya?ml$|.*\.json$|.*\.html?$|.*\.(sh|cmd|bat|ps1)$/i;
+
+export const rules = [
+    {
+        name: "invisible-unicode",
+        match: SOURCE_FILE_PATTERN,
+        check: (file) => checkInvisibleUnicode(file),
+        applyToDependencies: true,
+    },
+];