feat(indexer): introduce incremental indexing engine pipeline

.gitignore

@@ -1,3 +1,6 @@
+# Node.js
+node_modules/
+
 ## Core latex/pdflatex auxiliary files:
 *.aux
 *.lof

apps/indexer/.gitignore

@@ -0,0 +1,2 @@
+dist
+node_modules

apps/indexer/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "indexer",
+  "version": "1.0.0",
+  "description": "",
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "tsc",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "type": "commonjs",
+  "devDependencies": {
+    "@types/node": "^25.3.5",
+    "typescript": "^5.9.3"
+  }
+}

apps/indexer/src/IndexQueue.ts

@@ -0,0 +1,31 @@
+import { IndexJob } from "./types"
+
+/**
+ * Simple sequential job queue for indexing tasks.
+ * Ensures indexing operations run in order.
+ */
+export class IndexQueue {
+  private queue: IndexJob[] = []
+  private processing: Promise<void> | null = null
+
+  enqueue(job: IndexJob) {
+    this.queue.push(job)
+  }
+
+  process(handler: (job: IndexJob) => Promise<void>): Promise<void> {
+    if (this.processing) return this.processing
+
+    this.processing = (async () => {
+      while (this.queue.length > 0) {
+        const job = this.queue.shift()
+        if (!job) continue
+
+        await handler(job)
+      }
+    })().finally(() => {
+      this.processing = null
+    })
+
+    return this.processing
+  }
+}

apps/indexer/src/IndexingEngine.ts

@@ -0,0 +1,106 @@
+import { VaultAdapter } from "./adapters/VaultAdapter"
+import { EmbeddingAdapter } from "./adapters/EmbeddingAdapter"
+import { IndexStore } from "./adapters/IndexStore"
+import { NoteChunker } from "./NoteChunker"
+import { IndexQueue } from "./IndexQueue"
+import { IndexResult, IndexJob } from "./types"
+
+/**
+ * Coordinates the incremental indexing pipeline.
+ */
+export class IndexingEngine {
+  private vault: VaultAdapter
+  private embedder: EmbeddingAdapter
+  private store: IndexStore
+  private chunker: NoteChunker
+  private queue: IndexQueue
+
+  constructor(
+    vault: VaultAdapter,
+    embedder: EmbeddingAdapter,
+    store: IndexStore
+  ) {
+    this.vault = vault
+    this.embedder = embedder
+    this.store = store
+    this.chunker = new NoteChunker()
+    this.queue = new IndexQueue()
+  }
+
+  /**
+   * Schedule indexing for a note.
+   */
+  scheduleUpdate(notePath: string): Promise<void> {
+    const job: IndexJob = {
+      type: "update",
+      notePath,
+    }
+
+    this.queue.enqueue(job)
+
+    return this.queue.process(this.processJob.bind(this))
+  }
+
+  /**
+   * Schedule deletion of a note from the index.
+   */
+  scheduleDelete(notePath: string): Promise<void> {
+    const job: IndexJob = {
+      type: "delete",
+      notePath,
+    }
+
+    this.queue.enqueue(job)
+
+    return this.queue.process(this.processJob.bind(this))
+  }
+
+  /**
+   * Process jobs coming from the queue.
+   */
+  private async processJob(job: IndexJob) {
+    if (job.type === "update") {
+      await this.indexNote(job.notePath)
+    }
+
+    if (job.type === "delete") {
+      await this.removeNote(job.notePath)
+    }
+  }
+
+  /**
+   * Full indexing pipeline for a note.
+   */
+  private async indexNote(notePath: string): Promise<IndexResult> {
+    const markdown = await this.vault.readNote(notePath)
+
+    const chunks = this.chunker.split(notePath, markdown)
+
+    const chunkTexts = chunks.map((c) => c.text)
+
+    const embeddings = await this.embedder.embed(chunkTexts)
+
+    if (embeddings.length !== chunks.length) {
+      throw new Error(
+        `Embedding adapter returned ${embeddings.length} embeddings for ${chunks.length} chunks`
+      )
+    }
+
+    const result: IndexResult = {
+      notePath,
+      chunks,
+      embeddings,
+    }
+
+    await this.store.saveChunks(notePath, chunks, embeddings)
+
+    return result
+  }
+
+  /**
+   * Remove a note from the index.
+   */
+  private async removeNote(notePath: string) {
+    await this.store.deleteNote(notePath)
+  }
+}

apps/indexer/src/NoteChunker.ts

@@ -0,0 +1,31 @@
+import type { NoteChunk } from "./types"
+import crypto from "crypto"
+
+/**
+ * Splits markdown notes into chunks.
+ * Current implementation is simple paragraph-based splitting.
+ */
+export class NoteChunker {
+  split(notePath: string, markdown: string): NoteChunk[] {
+    const paragraphs = markdown
+      .split(/\n\s*\n/)
+      .map((p) => p.trim())
+      .filter((p) => p.length > 0)
+
+    const chunks: NoteChunk[] = paragraphs.map((text, index) => {
+      const id = crypto
+        .createHash("sha1")
+        .update(`${notePath}\0${index}\0${text}`)
+        .digest("hex")
+
+      return {
+        id,
+        notePath,
+        text,
+        position: index,
+      }
+    })
+
+    return chunks
+  }
+}

apps/indexer/src/adapters/EmbeddingAdapter.ts

@@ -0,0 +1,10 @@
+/**
+ * Adapter interface for embedding generation.
+ * Allows plugging different embedding models.
+ */
+export interface EmbeddingAdapter {
+  /**
+   * Generate embeddings for chunks.
+   */
+  embed(chunks: string[]): Promise<number[][]>
+}

apps/indexer/src/adapters/IndexStore.ts

@@ -0,0 +1,24 @@
+import { NoteChunk } from "../types"
+
+/**
+ * Storage abstraction for indexed notes.
+ * Allows plugging SQLite / vector DB / other stores.
+ */
+export interface IndexStore {
+  /**
+   * Atomically replace all indexed chunks and embeddings for the given notePath.
+   *
+   * Implementations must remove any previously stored chunks that no longer
+   * exist after a note edit.
+   */
+  saveChunks(
+    notePath: string,
+    chunks: NoteChunk[],
+    embeddings: number[][]
+  ): Promise<void>
+
+  /**
+   * Remove all chunks belonging to a note.
+   */
+  deleteNote(notePath: string): Promise<void>
+}

apps/indexer/src/adapters/VaultAdapter.ts

@@ -0,0 +1,16 @@
+/**
+ * Adapter interface for reading notes from the vault.
+ * This keeps the indexing engine independent of the
+ * underlying filesystem implementation.
+ */
+export interface VaultAdapter {
+  /**
+   * Read the contents of a note.
+   */
+  readNote(notePath: string): Promise<string>
+
+  /**
+   * List all notes in the vault.
+   */
+  listNotes(): Promise<string[]>
+}

apps/indexer/src/demo/DemoRunner.ts

@@ -0,0 +1,59 @@
+import { IndexingEngine } from "../IndexingEngine"
+import { VaultAdapter } from "../adapters/VaultAdapter"
+import { EmbeddingAdapter } from "../adapters/EmbeddingAdapter"
+import { IndexStore } from "../adapters/IndexStore"
+import { NoteChunk } from "../types"
+
+/**
+ * Simple in-memory demo implementations
+ */
+
+class DemoVault implements VaultAdapter {
+  async readNote(notePath: string): Promise<string> {
+    return `
+# Example Note
+
+This is the first paragraph.
+
+This is another paragraph about Smart Notes.
+`
+  }
+
+  async listNotes(): Promise<string[]> {
+    return ["demo.md"]
+  }
+}
+
+class DemoEmbedder implements EmbeddingAdapter {
+  async embed(chunks: string[]): Promise<number[][]> {
+    return chunks.map(() => [Math.random(), Math.random(), Math.random()])
+  }
+}
+
+class DemoStore implements IndexStore {
+  async saveChunks(
+    notePath: string,
+    chunks: NoteChunk[],
+    embeddings: number[][]
+  ): Promise<void> {
+    console.log("Indexed note:", notePath)
+    console.log("Chunks:", chunks.length)
+    console.log("Embeddings:", embeddings.length)
+  }
+
+  async deleteNote(notePath: string): Promise<void> {
+    console.log("Deleted note:", notePath)
+  }
+}
+
+async function runDemo() {
+  const engine = new IndexingEngine(
+    new DemoVault(),
+    new DemoEmbedder(),
+    new DemoStore()
+  )
+
+  engine.scheduleUpdate("demo.md")
+}
+
+runDemo()

apps/indexer/src/index.ts

@@ -0,0 +1,8 @@
+export * from "./types"
+export * from "./IndexingEngine"
+export * from "./IndexQueue"
+export * from "./NoteChunker"
+
+export * from "./adapters/VaultAdapter"
+export * from "./adapters/EmbeddingAdapter"
+export * from "./adapters/IndexStore"

apps/indexer/src/types.ts

@@ -0,0 +1,26 @@
+/**
+ * Represents a semantic chunk extracted from a note.
+ */
+export type NoteChunk = {
+  id: string
+  notePath: string
+  text: string
+  position: number
+}
+
+/**
+ * Job sent to the indexing queue.
+ */
+export type IndexJob = {
+  type: "update" | "delete"
+  notePath: string
+}
+
+/**
+ * Result produced by the indexing pipeline.
+ */
+export type IndexResult = {
+  notePath: string
+  chunks: NoteChunk[]
+  embeddings: number[][]
+}