Merge main into dev/georgeng/clientio_improvements_1

Resolved conflicts in favour of explicit cancelInteraction approach over
the auto-cancel-on-disconnect (cancelByConnection) approach from #2178.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: GeorgeNgMsft

TODO.md

@@ -64,34 +64,18 @@ This file collates all TODO comments found across the repository, organized by t
 
 | File | Line | TODO | Effort | Feasibility | Scope | Recommendation | Needs Human? |
 |------|------|------|--------|-------------|-------|----------------|--------------|
-| `ts/examples/cacheRESTEndpoint/src/index.ts` | 51 | actionName should match interface name | Low | High | Local | Fix | No |
-| `ts/examples/cacheRESTEndpoint/src/index.ts` | 56 | actually parse the schema instead of using regex | Medium | High | Local | Fix | No |
-| `ts/examples/chat/src/codeChat/commandTransformer.ts` | 59 | Validate that result.data is a valid NamedArgs? | Low | High | Local | Fix | No |
 | `ts/examples/chat/src/codeChat/commandTransformer.ts` | 170 | the same for args (currently not used by code chat) | Low | Medium | Local | No Fix | Yes |
-| `ts/examples/commandHistogram/src/main.ts` | 65 | move DB name to .env/config? | Low | High | Local | Fix | No |
-| `ts/examples/commandHistogram/src/main.ts` | 67 | move collection name to .env/config? | Low | High | Local | Fix | No |
 | `ts/examples/docuProc/src/pdfImporter.ts` | 210 | Try pre-computing embeddings in parallel to fill the embeddings cache (is that cache safe?) | Medium | Medium | Local | Fix | Yes |
-| `ts/examples/docuProc/src/pdfImporter.ts` | 344 | Therefore remove. | Low | High | Local | Fix | No |
 | `ts/examples/docuProc/src/pdfQNAInteractiveApp.ts` | 1024 | Allow for multiple concurrent sessions. | High | Medium | Component | No Fix | Yes |
 | `ts/examples/docuProc/src/pdfQNAInteractiveApp.ts` | 1031 | Cut off by total size, not count. | Low | High | Local | Fix | No |
 | `ts/examples/docuProc/src/pdfQNAInteractiveApp.ts` | 1151 | limit how much we write per blob too (if there are multiple). | Low | High | Local | Fix | No |
 | `ts/examples/docuProc/src/pdfQNAInteractiveApp.ts` | 1178 | Colorize code blocks. | Low | High | Local | Fix | No |
 | `ts/examples/memoryProviders/src/sqlite/keyValueTable.ts` | 195 | support | Medium | Medium | Component | Fix | Yes |
-| `ts/examples/memoryProviders/src/sqlite/textTable.ts` | 333 | use IN clause | Low | High | Local | Fix | No |
-| `ts/examples/memoryProviders/src/sqlite/textTable.ts` | 381 | use a JOIN | Low | High | Local | Fix | No |
 | `ts/examples/memoryProviders/src/sqlite/textTable.ts` | 511 | optimize by lowering into DB if possible | Medium | Medium | Local | Fix | Yes |
 | `ts/examples/memoryProviders/src/sqlite/textTable.ts` | 613 | Optimize | Medium | Medium | Local | Fix | Yes |
-| `ts/examples/memoryProviders/src/sqlite/vectorTable.ts` | 70 | ID generation | Low | High | Local | Fix | No |
 | `ts/examples/schemaStudio/src/schemaCommands.ts` | 94 | Generating settings command schemas... | Medium | Medium | Component | Fix | Yes |
-| `ts/examples/schemaStudio/src/urlCommands.ts` | 88 | handle redirects + default parameters, etc. | Medium | Medium | Local | Fix | No |
-| `ts/examples/spelunker/src/main.ts` | 44 | switch to whatever interactive-app uses to parse the command line? | Low | High | Local | Fix | No |
-| `ts/examples/spelunker/src/main.ts` | 77 | Use a proper command-line parser? | Low | High | Local | Fix | No |
-| `ts/examples/spelunker/src/pythonChunker.ts` | 81 | validate that JSON matches our schema. | Low | High | Local | Fix | No |
 | `ts/examples/spelunker/src/pythonImporter.ts` | 4 | Most of this is not Python specific; generalize to other languages. | High | Medium | Component | Fix | Yes |
-| `ts/examples/spelunker/src/pythonImporter.ts` | 71 | Make generic over languages | High | Medium | Component | Fix | No |
-| `ts/examples/spelunker/src/pythonImporter.ts` | 128 | Make this a function argument | Low | High | Local | Fix | No |
 | `ts/examples/spelunker/src/pythonImporter.ts` | 181 | Try pre-computing embeddings in parallel to fill the embeddings cache (is that cache safe?) | Medium | Medium | Local | Fix | Yes |
-| `ts/examples/spelunker/src/pythonImporter.ts` | 302 | Therefore remove. | Low | High | Local | Fix | No |
 | `ts/examples/spelunker/src/queryInterface.ts` | 905 | Allow for multiple concurrent sessions. | High | Medium | Component | Fix | No |
 | `ts/examples/spelunker/src/queryInterface.ts` | 912 | Cut off by total size, not count. | Low | High | Local | Fix | No |
 | `ts/examples/spelunker/src/queryInterface.ts` | 1035 | limit how much we write per blob too (if there are multiple). | Low | High | Local | Fix | No |
@@ -123,15 +107,8 @@ This file collates all TODO comments found across the repository, organized by t
 | `ts/packages/agentSdk/src/agentInterface.ts` | 57 | enable non-stringify pas content. | Medium | Medium | Cross-cutting | Fix | Yes |
 | `ts/packages/agentSdk/src/agentInterface.ts` | 234 | only utf8 & base64 is supported for now. | Medium | Medium | Component | Fix | No |
 | `ts/packages/agentSdkWrapper/src/webtask/tracing/types.ts` | 138 | Phase 2: Extract key elements from HTML | High | Low | Component | No Fix | Yes |
-| `ts/packages/agents/browser/src/extension/serviceWorker/index.ts` | 347 | sniffing the RPC call arguments. Fix typing. | Low | High | Local | Fix | No |
-| `ts/packages/agents/browser/src/extension/views/entityGraphView.ts` | 1341 | limit this to "contains" relationships | Low | High | Local | Fix | No |
 | `ts/packages/agents/browser/src/extension/views/extensionServiceBase.ts` | 579 | remove "type" from this dictionary. That will remove the need to wrap these values in a "parameters" object | Medium | Medium | Component | Fix | No |
-| `ts/packages/agents/browser/src/extension/views/macrosLibrary.ts` | 650 | Implement macro editing | Medium | High | Component | Fix | No |
 | `ts/packages/agents/browser/src/extension/views/topicGraphView.ts` | 296 | Implement topic viewport neighborhood functionality | High | Medium | Component | Fix | Yes |
-| `ts/packages/agents/browser/src/views/client/pdf/core/pdfViewer.ts` | 245 | Implement text search functionality | Medium | High | Component | Fix | No |
-| `ts/packages/agents/browser/src/views/client/pdf/core/pdfViewer.ts` | 258 | Implement find next | Low | High | Local | Fix | No |
-| `ts/packages/agents/browser/src/views/client/pdf/core/pdfViewer.ts` | 266 | Implement find previous | Low | High | Local | Fix | No |
-| `ts/packages/agents/browser/src/views/client/pdf/core/pdfViewer.ts` | 282 | Implement download functionality | Medium | High | Component | Fix | No |
 | `ts/packages/agents/calendar/src/calendarActionHandlerV3.ts` | 1120 | Implement sophisticated date parsing | Medium | High | Local | Fix | No |
 | `ts/packages/agents/calendar/src/calendarActionHandlerV3.ts` | 1126 | Implement sophisticated time parsing | Medium | High | Local | Fix | No |
 | `ts/packages/agents/desktop/src/connector.ts` | 127 | add shared agent storage or known storage location (requires permissions, trusted agents, etc.) | High | Low | Cross-cutting | No Fix | Yes |
@@ -161,7 +138,6 @@ This file collates all TODO comments found across the repository, organized by t
 | `ts/packages/agents/spelunker/src/embeddings.ts` | 47 | tune | Medium | Medium | Local | No Fix | Yes |
 | `ts/packages/agents/spelunker/src/embeddings.ts` | 186 | Fix this | Medium | Medium | Local | Fix | Yes |
 | `ts/packages/agents/spelunker/src/eval.ts` | 27 | Read this from a file that can be edited before each run | Low | High | Local | Fix | No |
-| `ts/packages/agents/spelunker/src/pythonChunker.ts` | 51 | validate that JSON matches our schema. | Low | High | Local | Fix | No |
 | `ts/packages/agents/spelunker/src/searchCode.ts` | 237 | tune | Medium | Medium | Local | No Fix | Yes |
 | `ts/packages/agents/spelunker/src/searchCode.ts` | 238 | tune | Medium | Medium | Local | Fix | Yes |
 | `ts/packages/agents/spelunker/src/searchCode.ts` | 268 | tune | Medium | Medium | Local | Fix | Yes |
@@ -174,8 +150,6 @@ This file collates all TODO comments found across the repository, organized by t
 | `ts/packages/agents/spelunker/src/searchCode.ts` | 458 | Use appendDisplay (requires passing actionContext) | Low | Medium | Local | Fix | No |
 | `ts/packages/agents/spelunker/src/spelunkerActionHandler.ts` | 89 | What other standard functions could be handy here? | Medium | Low | Component | No Fix | Yes |
 | `ts/packages/agents/spelunker/src/summarizing.ts` | 42 | Prompt engineering | High | Medium | Local | Fix | Yes |
-| `ts/packages/agents/spelunker/src/summarizing.ts` | 160 | Remove export once we're using summaries again. | Low | High | Local | Fix | No |
-| `ts/packages/agents/spelunker/src/summarizing.ts` | 178 | Make the values two elements, comment start and comment end | Low | High | Local | Fix | No |
 | `ts/packages/agents/spelunker/src/typescriptChunker.ts` | 198 | Move to caller? | Low | High | Local | Fix | No |
 | `ts/packages/agents/video/src/videoActionHandler.ts` | 54 | dynamic duration | Medium | Medium | Local | Fix | Yes |
 | `ts/packages/agents/weather/src/weatherActionHandler.ts` | 227 | Add more sophisticated validation: | Medium | High | Local | Fix | No |

ts/examples/cacheRESTEndpoint/src/index.ts

@@ -11,7 +11,13 @@ import {
 import { existsSync } from "fs";
 import path from "path";
 import { NodeType, SchemaParser } from "@typeagent/action-schema";
-import { NameValue } from "typeagent";
+
+type SchemaEntry = {
+    name: string;
+    camelCaseName: string;
+    value: string;
+    id?: string;
+};
 
 // Create an instance of an Express application
 const app = express();
@@ -26,7 +32,7 @@ const cacheFile = "data/v5_sample.json";
 let agentCache: AgentCache;
 
 // load the schema
-let schemas: NameValue[] = [];
+let schemas: SchemaEntry[] = [];
 const schemaFile: string = `../../packages/agents/settings/src/settingsActionSchema.ts`;
 
 // Set up a route for the root URL ('/')
@@ -48,17 +54,11 @@ app.get("/", async (req: Request, res: Response) => {
             matches.forEach((construction) => {
                 // add the id of the schema (if it exists) to the action
                 construction.match.actions.forEach((a) => {
-                    // TODO: actionName should match interface name
-                    const schema = schemas.find((s) =>
-                        s.name.startsWith(a.action.actionName),
+                    const schema = schemas.find(
+                        (s) => s.camelCaseName === a.action.actionName,
                     );
                     if (schema) {
-                        // TODO: actually parse the schema instead of using regex
-                        const idMatch = schema.value.match(/id:\s*([^;]+);/);
-                        const idValue = idMatch
-                            ? idMatch[1].trim().replaceAll('"', "")
-                            : undefined;
-                        (a.action as any).id = idValue;
+                        (a.action as any).id = schema.id;
                     }
                 });
 
@@ -100,11 +100,11 @@ app.listen(port, async () => {
     }
 });
 
-function getActionSchema(filePath: string): NameValue[] {
+function getActionSchema(filePath: string): SchemaEntry[] {
     const schema = new SchemaParser();
     schema.loadSchema(filePath);
     const types = schema.actionTypeNames();
-    const schemas: NameValue[] = [];
+    const result: SchemaEntry[] = [];
     for (const type of types) {
         const node = schema.openActionNode(type);
         let schemaText = node?.symbol.value!;
@@ -113,7 +113,18 @@ function getActionSchema(filePath: string): NameValue[] {
                 schemaText += "\n\n" + subType.symbol.value;
             }
         }
-        schemas.push({ name: node?.symbol.name!, value: schemaText });
+        const idChild = node!.children.find((c) => c.symbol.name === "id");
+        const rawId = idChild?.symbol.value ?? "";
+        const idValue = rawId ? JSON.parse(rawId.trim()) : undefined;
+        const interfaceName = node?.symbol.name!;
+        const camelCaseName =
+            interfaceName.charAt(0).toLowerCase() + interfaceName.slice(1);
+        result.push({
+            name: interfaceName,
+            camelCaseName,
+            value: schemaText,
+            id: idValue,
+        });
     }
-    return schemas;
+    return result;
 }

ts/examples/chat/src/codeChat/commandTransformer.ts

@@ -56,7 +56,14 @@ export function createCommandTransformer(
             return undefined;
         } else {
             // console.log("[Success:]", JSON.stringify(result, null, 2));
-            // TODO: Validate that result.data is a valid NamedArgs?
+            if (
+                typeof result.data !== "object" ||
+                result.data === null ||
+                typeof (result.data as NamedArgs).name !== "string"
+            ) {
+                console.log("[Error:] result.data is not a valid NamedArgs");
+                return undefined;
+            }
             return result.data;
         }
     }

ts/examples/commandHistogram/src/main.ts

@@ -62,10 +62,10 @@ async function queryMongoDB(query: object) {
 
     try {
         await client.connect();
-        // TODO: move DB name to .env/config?
-        const database = client.db("telemetrydb");
-        // TODO: move collection name to .env/config?
-        const collection = database.collection("dispatcherlogs");
+        const database = client.db(process.env["DB_NAME"] ?? "telemetrydb");
+        const collection = database.collection(
+            process.env["COLLECTION_NAME"] ?? "dispatcherlogs",
+        );
 
         const documents = await collection.find(query).toArray();
         return documents;

ts/examples/docuProc/src/pdfImporter.ts

@@ -340,8 +340,3 @@ export async function exponentialBackoff<T extends any[], R>(
         }
     }
 }
-
-// Apply URL escaping to key. NOTE: Currently unused. TODO: Therefore remove.
-export function sanitizeKey(key: string): string {
-    return encodeURIComponent(key).replace(/%20/g, "+"); // Encode spaces as plus, others as %xx.
-}

ts/examples/docuProc/src/pdfQNAInteractiveApp.ts

@@ -1028,7 +1028,18 @@ async function findRecentAnswers(
     }
     // Assume the name field (the internal key) is a timestamp.
     recentAnswers.sort((a, b) => b.name.localeCompare(a.name));
-    recentAnswers.splice(20); // TODO: Cut off by total size, not count.
+    // Cut off by total size (character count), not count.
+    let totalSize = 0;
+    const maxTotalSize = 100_000;
+    let cutoff = recentAnswers.length;
+    for (let i = 0; i < recentAnswers.length; i++) {
+        totalSize += JSON.stringify(recentAnswers[i]).length;
+        if (totalSize > maxTotalSize) {
+            cutoff = i;
+            break;
+        }
+    }
+    recentAnswers.splice(cutoff);
     recentAnswers.reverse(); // Most recent last.
     return recentAnswers;
 }
@@ -1148,7 +1159,6 @@ export function writeChunkLines(
     io: iapp.InteractiveIo,
     lineBudget = 10,
 ): void {
-    // TODO: limit how much we write per blob too (if there are multiple).
     writeMain(io, `\nCHUNK ID: ${chunk.id}`);
     const formatContent = (content: string | string[]): string => {
         if (Array.isArray(content)) {
@@ -1164,9 +1174,18 @@ export function writeChunkLines(
         return content;
     };
 
-    if (chunk.blobs[0].content) {
-        const content = formatContent(chunk.blobs[0].content);
-        writeMain(io, `Content: ${content}`);
+    const perBlobBudget = Math.ceil(lineBudget / chunk.blobs.length);
+    for (const blob of chunk.blobs) {
+        if (!blob.content) continue;
+        const content = formatContent(blob.content);
+        const blobLines = content.split(/\r?\n/);
+        const limit = Math.min(blobLines.length, perBlobBudget);
+        for (let i = 0; i < limit; i++) {
+            writeMain(io, blobLines[i]);
+        }
+        if (blobLines.length > perBlobBudget) {
+            writeNote(io, "   ...");
+        }
     }
 }
 
@@ -1175,10 +1194,9 @@ export function wordWrap(text: string, wrapLength: number = 80): string {
     const lines: string[] = [];
     const prefixRegex = /^\s*((-|\*|\d+\.)\s+)?/;
     for (let line of text.split(/[ ]*\r?\n/)) {
-        if (line.startsWith("```")) inCodeBlock = !inCodeBlock; // TODO: Colorize code blocks.
-        if (line.length <= wrapLength || inCodeBlock) {
-            // The whole line is deemed to fit.
-            lines.push(line);
+        if (line.startsWith("```")) inCodeBlock = !inCodeBlock;
+        if (inCodeBlock || line.startsWith("```")) {
+            lines.push(chalk.cyan(line));
             continue;
         }
         // We must try to break.

ts/examples/memoryProviders/src/sqlite/textTable.ts

@@ -330,9 +330,21 @@ export async function createTextIndex<
     }
 
     function getIds(texts: string[]): Promise<(TTextId | undefined)[]> {
-        // TODO: use IN clause
+        if (texts.length === 0) {
+            return Promise.resolve([]);
+        }
+        const inClause = sql_makeInClause(texts);
+        const rows = db
+            .prepare(
+                `SELECT stringId, value FROM ${textTable.tableName} WHERE value IN (${inClause})`,
+            )
+            .all() as StringTableRow[];
+        const idByText = new Map<string, number>();
+        for (const row of rows) {
+            idByText.set(row.value, row.stringId);
+        }
         return Promise.resolve(
-            texts.map((t) => serializer.serialize(textTable.getId(t))),
+            texts.map((t) => serializer.serialize(idByText.get(t))),
         );
     }
 
@@ -378,13 +390,25 @@ export async function createTextIndex<
         values: string[],
         join?: string,
     ): IterableIterator<ScoredItem<TSourceId>> {
-        // TODO: use a JOIN
-        const textIds = [...textTable.getIds(values)];
-        const hits = postingsTable.getHits(textIds, join);
-        if (hits) {
-            for (const hit of hits) {
-                yield hit;
-            }
+        if (values.length === 0) {
+            return;
+        }
+        const inClause = sql_makeInClause(values);
+        const innerJoin = `INNER JOIN ${textTable.tableName} ON keyId = stringId`;
+        const sql = join
+            ? `SELECT valueId AS item, count(*) AS score FROM ${postingsTable.tableName}
+               ${innerJoin}
+               ${join} AND ${textTable.tableName}.value IN (${inClause})
+               GROUP BY valueId
+               ORDER BY score DESC`
+            : `SELECT valueId AS item, count(*) AS score FROM ${postingsTable.tableName}
+               ${innerJoin}
+               WHERE ${textTable.tableName}.value IN (${inClause})
+               GROUP BY valueId
+               ORDER BY score DESC`;
+        const stmt = db.prepare(sql);
+        for (const row of stmt.iterate()) {
+            yield row as ScoredItem<TSourceId>;
         }
     }
 

ts/examples/memoryProviders/src/sqlite/vectorTable.ts

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 import * as sqlite from "better-sqlite3";
+import { randomUUID } from "node:crypto";
 import {
     Embedding,
     ScoredItem,
@@ -40,6 +41,9 @@ export function createVectorTable<TKeyId extends ValueType = string>(
     const sql_add = db.prepare(
         `INSERT OR IGNORE INTO ${tableName} (keyId, embedding) VALUES (?, ?)`,
     );
+    const sql_add_auto = db.prepare(
+        `INSERT INTO ${tableName} (embedding) VALUES (?)`,
+    );
     const sql_remove = db.prepare(`DELETE FROM ${tableName} WHERE keyId = ?`);
     const sql_all = db.prepare(`SELECT * from ${tableName}`);
 
@@ -67,8 +71,16 @@ export function createVectorTable<TKeyId extends ValueType = string>(
 
     function put(value: Embedding, id?: TKeyId | undefined): Promise<TKeyId> {
         if (id === undefined) {
-            // TODO: ID generation
-            throw Error("id required");
+            const buffer = Buffer.from(value.buffer);
+            if (keyType === "INTEGER") {
+                const result = sql_add_auto.run(buffer);
+                return Promise.resolve(
+                    Number(result.lastInsertRowid) as TKeyId,
+                );
+            }
+            const newId = randomUUID() as TKeyId;
+            sql_add.run(newId, buffer);
+            return Promise.resolve(newId);
         }
         const buffer = Buffer.from(value.buffer);
         sql_add.run(id, buffer);