diff --git a/.changeset/optional-indexing.md b/.changeset/optional-indexing.md new file mode 100644 index 000000000..2e72f3d92 --- /dev/null +++ b/.changeset/optional-indexing.md @@ -0,0 +1,66 @@ +--- +'@tanstack/db': minor +--- + +Make indexing explicit with two index types for different use cases + +**Breaking Changes:** + +- `autoIndex` now defaults to `off` instead of `eager` +- `BTreeIndex` is no longer exported from `@tanstack/db` main entry point +- To use `createIndex()` or `autoIndex: 'eager'`, you must set `defaultIndexType` on the collection + +**Changes:** + +- New `@tanstack/db/indexing` entry point for tree-shakeable indexing +- **BasicIndex** - Lightweight index using Map + sorted Array for both equality and range queries (`eq`, `in`, `gt`, `gte`, `lt`, `lte`). O(n) updates but fast reads. +- **BTreeIndex** - Full-featured index with O(log n) updates and sorted iteration for ORDER BY optimization on large collections (10k+ items) +- Dev mode suggestions (ON by default) warn when indexes would help + +**Migration:** + +If you were relying on auto-indexing, set `defaultIndexType` on your collections: + +1. **Lightweight indexing** (good for most use cases): + +```ts +import { BasicIndex } from '@tanstack/db/indexing' + +const collection = createCollection({ + defaultIndexType: BasicIndex, + autoIndex: 'eager', + // ... +}) +``` + +2. **Full BTree indexing** (for ORDER BY optimization on large collections): + +```ts +import { BTreeIndex } from '@tanstack/db/indexing' + +const collection = createCollection({ + defaultIndexType: BTreeIndex, + autoIndex: 'eager', + // ... +}) +``` + +3. **Per-index explicit type** (mix index types): + +```ts +import { BasicIndex, BTreeIndex } from '@tanstack/db/indexing' + +const collection = createCollection({ + defaultIndexType: BasicIndex, // Default for createIndex() + // ... +}) + +// Override for specific indexes +collection.createIndex((row) => row.date, { indexType: BTreeIndex }) +``` + +**Bundle Size Impact:** + +- No indexing: ~30% smaller bundle +- BasicIndex: ~5 KB (~1.3 KB gzipped) +- BTreeIndex: ~33 KB (~7.8 KB gzipped) diff --git a/packages/db-browser-wa-sqlite-persisted-collection/e2e/browser-single-tab-persisted-collection.e2e.test.ts b/packages/db-browser-wa-sqlite-persisted-collection/e2e/browser-single-tab-persisted-collection.e2e.test.ts index bca72eefd..d8f2effa3 100644 --- a/packages/db-browser-wa-sqlite-persisted-collection/e2e/browser-single-tab-persisted-collection.e2e.test.ts +++ b/packages/db-browser-wa-sqlite-persisted-collection/e2e/browser-single-tab-persisted-collection.e2e.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterAll, afterEach, beforeAll } from 'vitest' -import { createCollection } from '@tanstack/db' +import { createCollection, BTreeIndex } from '@tanstack/db' import { createBrowserWASQLitePersistence, persistedCollectionOptions, @@ -64,6 +64,8 @@ function createPersistedCollection( syncMode, getKey: (item) => item.id, persistence, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, }), ) diff --git a/packages/db-capacitor-sqlite-persisted-collection/e2e/shared/capacitor-persisted-collection-harness.ts b/packages/db-capacitor-sqlite-persisted-collection/e2e/shared/capacitor-persisted-collection-harness.ts index 0a935f980..aa4def0dd 100644 --- a/packages/db-capacitor-sqlite-persisted-collection/e2e/shared/capacitor-persisted-collection-harness.ts +++ b/packages/db-capacitor-sqlite-persisted-collection/e2e/shared/capacitor-persisted-collection-harness.ts @@ -1,4 +1,4 @@ -import { createCollection } from '@tanstack/db' +import { createCollection, BTreeIndex } from '@tanstack/db' import { persistedCollectionOptions } from '../../src' import { generateSeedData } from '../../../db-collection-e2e/src/fixtures/seed-data' import type { Collection } from '@tanstack/db' @@ -69,6 +69,8 @@ function createPersistedCollection( syncMode, getKey: (item) => item.id, persistence, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, }), ) diff --git a/packages/db-electron-sqlite-persisted-collection/tests/electron-persisted-collection.e2e.test.ts b/packages/db-electron-sqlite-persisted-collection/tests/electron-persisted-collection.e2e.test.ts index 1dc74a35f..3b4e70f2a 100644 --- a/packages/db-electron-sqlite-persisted-collection/tests/electron-persisted-collection.e2e.test.ts +++ b/packages/db-electron-sqlite-persisted-collection/tests/electron-persisted-collection.e2e.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterAll, afterEach, beforeAll } from 'vitest' -import { createCollection } from '@tanstack/db' +import { createCollection, BTreeIndex } from '@tanstack/db' import { persistedCollectionOptions } from '@tanstack/db-sqlite-persisted-collection-core' import { createNodeSQLitePersistence } from '@tanstack/db-node-sqlite-persisted-collection' import { BetterSqlite3SQLiteDriver } from '../../db-node-sqlite-persisted-collection/src/node-driver' @@ -142,6 +142,8 @@ function createPersistedCollection( syncMode, getKey: (item) => item.id, persistence, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, }), ) diff --git a/packages/db-expo-sqlite-persisted-collection/e2e/mobile-persisted-collection-conformance-suite.ts b/packages/db-expo-sqlite-persisted-collection/e2e/mobile-persisted-collection-conformance-suite.ts index ca2cf3f72..4d5f66740 100644 --- a/packages/db-expo-sqlite-persisted-collection/e2e/mobile-persisted-collection-conformance-suite.ts +++ b/packages/db-expo-sqlite-persisted-collection/e2e/mobile-persisted-collection-conformance-suite.ts @@ -2,7 +2,7 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterAll, afterEach, beforeAll } from 'vitest' -import { createCollection } from '@tanstack/db' +import { createCollection, BTreeIndex } from '@tanstack/db' import { persistedCollectionOptions } from '../src' import { generateSeedData } from '../../db-collection-e2e/src/fixtures/seed-data' import { runPersistedCollectionConformanceSuite } from '../../db-sqlite-persisted-collection-core/tests/contracts/persisted-collection-conformance-contract' @@ -62,6 +62,8 @@ function createPersistedCollection( syncMode, getKey: (item) => item.id, persistence, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, }), ) diff --git a/packages/db-node-sqlite-persisted-collection/e2e/node-persisted-collection.e2e.test.ts b/packages/db-node-sqlite-persisted-collection/e2e/node-persisted-collection.e2e.test.ts index 6331e31cf..393291434 100644 --- a/packages/db-node-sqlite-persisted-collection/e2e/node-persisted-collection.e2e.test.ts +++ b/packages/db-node-sqlite-persisted-collection/e2e/node-persisted-collection.e2e.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterAll, afterEach, beforeAll } from 'vitest' -import { createCollection } from '@tanstack/db' +import { createCollection, BTreeIndex } from '@tanstack/db' import BetterSqlite3 from 'better-sqlite3' import { createNodeSQLitePersistence, persistedCollectionOptions } from '../src' import { generateSeedData } from '../../db-collection-e2e/src/fixtures/seed-data' @@ -60,6 +60,8 @@ function createPersistedCollection( syncMode, getKey: (item) => item.id, persistence, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, }), ) diff --git a/packages/db-react-native-sqlite-persisted-collection/e2e/mobile-persisted-collection-conformance-suite.ts b/packages/db-react-native-sqlite-persisted-collection/e2e/mobile-persisted-collection-conformance-suite.ts index 4f56f158e..bf7080319 100644 --- a/packages/db-react-native-sqlite-persisted-collection/e2e/mobile-persisted-collection-conformance-suite.ts +++ b/packages/db-react-native-sqlite-persisted-collection/e2e/mobile-persisted-collection-conformance-suite.ts @@ -2,7 +2,7 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterAll, afterEach, beforeAll } from 'vitest' -import { createCollection } from '@tanstack/db' +import { createCollection, BTreeIndex } from '@tanstack/db' import { persistedCollectionOptions } from '../src' import { generateSeedData } from '../../db-collection-e2e/src/fixtures/seed-data' import { runPersistedCollectionConformanceSuite } from '../../db-sqlite-persisted-collection-core/tests/contracts/persisted-collection-conformance-contract' @@ -62,6 +62,8 @@ function createPersistedCollection( syncMode, getKey: (item) => item.id, persistence, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, }), ) diff --git a/packages/db-sqlite-persisted-collection-core/tests/persisted.test.ts b/packages/db-sqlite-persisted-collection-core/tests/persisted.test.ts index 64c488370..1613e0f3c 100644 --- a/packages/db-sqlite-persisted-collection-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persisted-collection-core/tests/persisted.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from 'vitest' -import { IR, createCollection, createTransaction } from '@tanstack/db' +import { + IR, + createCollection, + createTransaction, + BasicIndex, +} from '@tanstack/db' import { InvalidPersistedCollectionCoordinatorError, InvalidPersistedStorageKeyEncodingError, @@ -835,6 +840,7 @@ describe(`persistedCollectionOptions`, () => { persistedCollectionOptions({ id: `sync-present-indexes`, getKey: (item) => item.id, + defaultIndexType: BasicIndex, sync: { sync: ({ markReady }) => { markReady() diff --git a/packages/db-tauri-sqlite-persisted-collection/e2e/shared/tauri-persisted-collection-harness.ts b/packages/db-tauri-sqlite-persisted-collection/e2e/shared/tauri-persisted-collection-harness.ts index 3a46f5168..a4fcb3c0e 100644 --- a/packages/db-tauri-sqlite-persisted-collection/e2e/shared/tauri-persisted-collection-harness.ts +++ b/packages/db-tauri-sqlite-persisted-collection/e2e/shared/tauri-persisted-collection-harness.ts @@ -1,4 +1,4 @@ -import { createCollection } from '@tanstack/db' +import { createCollection, BTreeIndex } from '@tanstack/db' import { persistedCollectionOptions } from '../../src' import { generateSeedData } from '../../../db-collection-e2e/src/fixtures/seed-data' import type { Collection } from '@tanstack/db' @@ -69,6 +69,8 @@ function createPersistedCollection( syncMode, getKey: (item) => item.id, persistence, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, }), ) diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index 2cb975f91..e51eb998d 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -1,4 +1,5 @@ import { + CollectionConfigurationError, CollectionRequiresConfigError, CollectionRequiresSyncConfigError, } from '../errors' @@ -17,7 +18,7 @@ import type { CollectionEventHandler, CollectionIndexMetadata, } from './events.js' -import type { BaseIndex, IndexResolver } from '../indexes/base-index.js' +import type { BaseIndex, IndexConstructor } from '../indexes/base-index.js' import type { IndexOptions } from '../indexes/index-options.js' import type { ChangeMessage, @@ -39,8 +40,6 @@ import type { } from '../types' import type { SingleRowRefProxy } from '../query/builder/ref-proxy' import type { StandardSchemaV1 } from '@standard-schema/spec' -import type { BTreeIndex } from '../indexes/btree-index.js' -import type { IndexProxy } from '../indexes/lazy-index.js' import type { WithVirtualProps } from '../virtual-props.js' export type { CollectionIndexMetadata } from './events.js' @@ -336,7 +335,16 @@ export class CollectionImpl< // Set default values for optional config properties this.config = { ...config, - autoIndex: config.autoIndex ?? `eager`, + autoIndex: config.autoIndex ?? `off`, + } + + if (this.config.autoIndex === `eager` && !config.defaultIndexType) { + throw new CollectionConfigurationError( + `autoIndex: 'eager' requires defaultIndexType to be set. ` + + `Import an index type and set it:\n` + + ` import { BasicIndex } from '@tanstack/db'\n` + + ` createCollection({ defaultIndexType: BasicIndex, autoIndex: 'eager', ... })`, + ) } this._changes = new CollectionChangesManager() @@ -362,6 +370,7 @@ export class CollectionImpl< this._indexes.setDeps({ state: this._state, lifecycle: this._lifecycle, + defaultIndexType: config.defaultIndexType, events: this._events, }) this._lifecycle.setDeps({ @@ -568,38 +577,27 @@ export class CollectionImpl< * Indexes significantly improve query performance by allowing constant time lookups * and logarithmic time range queries instead of full scans. * - * @template TResolver - The type of the index resolver (constructor or async loader) * @param indexCallback - Function that extracts the indexed value from each item * @param config - Configuration including index type and type-specific options - * @returns An index proxy that provides access to the index when ready + * @returns The created index * * @example - * // Create a default B+ tree index - * const ageIndex = collection.createIndex((row) => row.age) + * ```ts + * import { BasicIndex } from '@tanstack/db' * - * // Create a ordered index with custom options + * // Create an index with explicit type * const ageIndex = collection.createIndex((row) => row.age, { - * indexType: BTreeIndex, - * options: { - * compareFn: customComparator, - * compareOptions: { direction: 'asc', nulls: 'first', stringSort: 'lexical' } - * }, - * name: 'age_btree' + * indexType: BasicIndex * }) * - * // Create an async-loaded index - * const textIndex = collection.createIndex((row) => row.content, { - * indexType: async () => { - * const { FullTextIndex } = await import('./indexes/fulltext.js') - * return FullTextIndex - * }, - * options: { language: 'en' } - * }) + * // Create an index with collection's default type + * const nameIndex = collection.createIndex((row) => row.name) + * ``` */ - public createIndex = typeof BTreeIndex>( + public createIndex>( indexCallback: (row: SingleRowRefProxy) => any, - config: IndexOptions = {}, - ): IndexProxy { + config: IndexOptions = {}, + ): BaseIndex { return this._indexes.createIndex(indexCallback, config) } @@ -611,7 +609,7 @@ export class CollectionImpl< * collection query planning. Existing index proxy references should be treated * as invalid after removal. */ - public removeIndex(indexOrId: IndexProxy | number): boolean { + public removeIndex(indexOrId: BaseIndex | number): boolean { return this._indexes.removeIndex(indexOrId) } diff --git a/packages/db/src/collection/indexes.ts b/packages/db/src/collection/indexes.ts index ce3988ead..84e45d6fc 100644 --- a/packages/db/src/collection/indexes.ts +++ b/packages/db/src/collection/indexes.ts @@ -1,15 +1,10 @@ -import { - IndexProxy, - IndexRemovedError, - LazyIndexWrapper, -} from '../indexes/lazy-index' import { createSingleRowRefProxy, toExpression, } from '../query/builder/ref-proxy' -import { BTreeIndex } from '../indexes/btree-index' +import { CollectionConfigurationError } from '../errors' import type { StandardSchemaV1 } from '@standard-schema/spec' -import type { BaseIndex, IndexResolver } from '../indexes/base-index' +import type { BaseIndex, IndexConstructor } from '../indexes/base-index' import type { ChangeMessage } from '../types' import type { IndexOptions } from '../indexes/index-options' import type { SingleRowRefProxy } from '../query/builder/ref-proxy' @@ -33,24 +28,12 @@ function compareStringsCodePoint(left: string, right: string): number { return left < right ? -1 : 1 } -function isConstructorResolver( - resolver: IndexResolver, -): boolean { - return typeof resolver === `function` && resolver.prototype !== undefined -} - function resolveResolverMetadata( - resolver: IndexResolver, + resolver: IndexConstructor, ): CollectionIndexResolverMetadata { - if (isConstructorResolver(resolver)) { - return { - kind: `constructor`, - ...(resolver.name ? { name: resolver.name } : {}), - } - } - return { - kind: `async`, + kind: `constructor`, + ...(resolver.name ? { name: resolver.name } : {}), } } @@ -174,7 +157,7 @@ function createCollectionIndexMetadata( indexId: number, expression: BasicExpression, name: string | undefined, - resolver: IndexResolver, + resolver: IndexConstructor, options: unknown, ): CollectionIndexMetadata { const resolverMetadata = resolveResolverMetadata(resolver) @@ -223,10 +206,6 @@ function cloneExpression(expression: BasicExpression): BasicExpression { return JSON.parse(JSON.stringify(expression)) as BasicExpression } -function isIndexRemovedError(error: unknown): boolean { - return error instanceof IndexRemovedError -} - export class CollectionIndexesManager< TOutput extends object = Record, TKey extends string | number = string | number, @@ -235,12 +214,11 @@ export class CollectionIndexesManager< > { private lifecycle!: CollectionLifecycleManager private state!: CollectionStateManager + private defaultIndexType: IndexConstructor | undefined private events!: CollectionEventsManager - public lazyIndexes = new Map>() - public resolvedIndexes = new Map>() + public indexes = new Map>() public indexMetadata = new Map() - public isIndexesResolved = false public indexCounter = 0 constructor() {} @@ -248,20 +226,32 @@ export class CollectionIndexesManager< setDeps(deps: { state: CollectionStateManager lifecycle: CollectionLifecycleManager + defaultIndexType?: IndexConstructor events: CollectionEventsManager }) { this.state = deps.state this.lifecycle = deps.lifecycle + this.defaultIndexType = deps.defaultIndexType this.events = deps.events } /** * Creates an index on a collection for faster queries. + * + * @example + * ```ts + * // With explicit index type (recommended for tree-shaking) + * import { BasicIndex } from '@tanstack/db' + * collection.createIndex((row) => row.userId, { indexType: BasicIndex }) + * + * // With collection's default index type + * collection.createIndex((row) => row.userId) + * ``` */ - public createIndex = typeof BTreeIndex>( + public createIndex>( indexCallback: (row: SingleRowRefProxy) => any, - config: IndexOptions = {}, - ): IndexProxy { + config: IndexOptions = {}, + ): BaseIndex { this.lifecycle.validateCollectionUsable(`createIndex`) const indexId = ++this.indexCounter @@ -269,90 +259,63 @@ export class CollectionIndexesManager< const indexExpression = indexCallback(singleRowRefProxy) const expression = toExpression(indexExpression) - // Default to BTreeIndex if no type specified - const resolver = config.indexType ?? (BTreeIndex as unknown as TResolver) + // Use provided index type, or fall back to collection's default + const IndexType = config.indexType ?? this.defaultIndexType + if (!IndexType) { + throw new CollectionConfigurationError( + `No index type specified and no defaultIndexType set on collection. ` + + `Either pass indexType in config, or set defaultIndexType on the collection:\n` + + ` import { BasicIndex } from '@tanstack/db'\n` + + ` createCollection({ defaultIndexType: BasicIndex, ... })`, + ) + } - // Create lazy wrapper - const lazyIndex = new LazyIndexWrapper( + // Create index synchronously + const index = new IndexType( indexId, expression, config.name, - resolver, config.options, - this.state.entries(), ) - this.lazyIndexes.set(indexId, lazyIndex) + // Build with current data + index.build(this.state.entries()) + + this.indexes.set(indexId, index) + + // Track metadata and emit event const metadata = createCollectionIndexMetadata( indexId, expression, config.name, - resolver, + IndexType, config.options, ) this.indexMetadata.set(indexId, metadata) - - // For BTreeIndex, resolve immediately and synchronously - if ((resolver as unknown) === BTreeIndex) { - try { - const resolvedIndex = lazyIndex.getResolved() - this.resolvedIndexes.set(indexId, resolvedIndex) - } catch (error) { - console.warn(`Failed to resolve BTreeIndex:`, error) - } - } else if (typeof resolver === `function` && resolver.prototype) { - // Other synchronous constructors - resolve immediately - try { - const resolvedIndex = lazyIndex.getResolved() - this.resolvedIndexes.set(indexId, resolvedIndex) - } catch { - // Fallback to async resolution - this.resolveSingleIndex(indexId, lazyIndex).catch((error) => { - if (isIndexRemovedError(error)) { - return - } - console.warn(`Failed to resolve single index:`, error) - }) - } - } else if (this.isIndexesResolved) { - // Async loader but indexes are already resolved - resolve this one - this.resolveSingleIndex(indexId, lazyIndex).catch((error) => { - if (isIndexRemovedError(error)) { - return - } - console.warn(`Failed to resolve single index:`, error) - }) - } - this.events.emitIndexAdded(metadata) - return new IndexProxy(indexId, lazyIndex) + return index } /** * Removes an index from this collection. * Returns true when an index existed and was removed, false otherwise. */ - public removeIndex(indexOrId: IndexProxy | number): boolean { + public removeIndex(indexOrId: BaseIndex | number): boolean { this.lifecycle.validateCollectionUsable(`removeIndex`) const indexId = typeof indexOrId === `number` ? indexOrId : indexOrId.id - const lazyIndex = this.lazyIndexes.get(indexId) - if (!lazyIndex) { + const index = this.indexes.get(indexId) + if (!index) { return false } - if ( - indexOrId instanceof IndexProxy && - lazyIndex !== indexOrId._getLazyWrapper() - ) { - // Same numeric id from another collection should not remove this index. + if (typeof indexOrId !== `number` && index !== indexOrId) { + // Passed a different index instance with the same id — do not remove. return false } - lazyIndex.markRemoved() - this.lazyIndexes.delete(indexId) - this.resolvedIndexes.delete(indexId) + this.indexes.delete(indexId) const metadata = this.indexMetadata.get(indexId) this.indexMetadata.delete(indexId) @@ -363,61 +326,6 @@ export class CollectionIndexesManager< return true } - /** - * Resolve all lazy indexes (called when collection first syncs) - */ - public async resolveAllIndexes(): Promise { - if (this.isIndexesResolved) return - - const resolutionPromises = Array.from(this.lazyIndexes.entries()).map( - async ([indexId, lazyIndex]) => { - let resolvedIndex: BaseIndex - try { - resolvedIndex = await lazyIndex.resolve() - } catch (error) { - if (isIndexRemovedError(error)) { - return { indexId, resolvedIndex: undefined } - } - - throw error - } - - // Build index with current data - resolvedIndex.build(this.state.entries()) - - if (this.lazyIndexes.has(indexId)) { - this.resolvedIndexes.set(indexId, resolvedIndex) - } - return { indexId, resolvedIndex } - }, - ) - - await Promise.all(resolutionPromises) - this.isIndexesResolved = true - } - - /** - * Resolve a single index immediately - */ - private async resolveSingleIndex( - indexId: number, - lazyIndex: LazyIndexWrapper, - ): Promise> { - const resolvedIndex = await lazyIndex.resolve() - resolvedIndex.build(this.state.entries()) - if (this.lazyIndexes.has(indexId)) { - this.resolvedIndexes.set(indexId, resolvedIndex) - } - return resolvedIndex - } - - /** - * Get resolved indexes for query optimization - */ - get indexes(): Map> { - return this.resolvedIndexes - } - /** * Returns a sorted snapshot of index metadata. * This allows persisted wrappers to bootstrap from indexes that were created @@ -440,7 +348,7 @@ export class CollectionIndexesManager< * Updates all indexes when the collection changes */ public updateIndexes(changes: Array>): void { - for (const index of this.resolvedIndexes.values()) { + for (const index of this.indexes.values()) { for (const change of changes) { switch (change.type) { case `insert`: @@ -462,15 +370,10 @@ export class CollectionIndexesManager< } /** - * Clean up the collection by stopping sync and clearing data - * This can be called manually or automatically by garbage collection + * Clean up indexes */ public cleanup(): void { - for (const lazyIndex of this.lazyIndexes.values()) { - lazyIndex.markRemoved() - } - this.lazyIndexes.clear() - this.resolvedIndexes.clear() + this.indexes.clear() this.indexMetadata.clear() } } diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts index 3010e3467..a9454ddca 100644 --- a/packages/db/src/collection/lifecycle.ts +++ b/packages/db/src/collection/lifecycle.ts @@ -106,17 +106,6 @@ export class CollectionLifecycleManager< const previousStatus = this.status this.status = newStatus - // Resolve indexes when collection becomes ready - if (newStatus === `ready` && !this.indexes.isIndexesResolved) { - // Resolve indexes asynchronously without blocking - this.indexes.resolveAllIndexes().catch((error) => { - console.warn( - `${this.config.id ? `[${this.config.id}] ` : ``}Failed to resolve indexes:`, - error, - ) - }) - } - // Emit event this.events.emitStatusChange(newStatus, previousStatus) } diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index ae13e1295..2d48add4b 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -256,6 +256,13 @@ export class CollectionSubscription this.orderByIndex = index } + /** + * Check if an orderBy index has been set for this subscription + */ + hasOrderByIndex(): boolean { + return this.orderByIndex !== undefined + } + /** * Set subscription status and emit events if changed */ diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 2c2b380be..ec1e22966 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -27,11 +27,45 @@ export { } from './virtual-props.js' // Index system exports -export * from './indexes/base-index.js' -export * from './indexes/btree-index.js' -export * from './indexes/lazy-index.js' +export { BaseIndex } from './indexes/base-index.js' +export type { + IndexInterface, + IndexConstructor, + IndexStats, + IndexOperation, +} from './indexes/base-index.js' export { type IndexOptions } from './indexes/index-options.js' +// Index implementations +export { BasicIndex } from './indexes/basic-index.js' +export type { + BasicIndexOptions, + RangeQueryOptions, +} from './indexes/basic-index.js' +export { BTreeIndex } from './indexes/btree-index.js' +export type { RangeQueryOptions as BTreeRangeQueryOptions } from './indexes/btree-index.js' +export { ReverseIndex } from './indexes/reverse-index.js' + +// Index optimization utilities +export { + optimizeExpressionWithIndexes, + findIndexForField, +} from './utils/index-optimization.js' + +// Dev mode utilities +export { + configureIndexDevMode, + isDevModeEnabled, + getIndexDevModeConfig, + trackQuery, + clearQueryPatterns, + getQueryPatterns, +} from './indexes/index-registry.js' +export type { + IndexDevModeConfig, + IndexSuggestion, +} from './indexes/index-registry.js' + // Expression helpers export * from './query/expression-helpers.js' diff --git a/packages/db/src/indexes/auto-index.ts b/packages/db/src/indexes/auto-index.ts index d58c626e5..303267cec 100644 --- a/packages/db/src/indexes/auto-index.ts +++ b/packages/db/src/indexes/auto-index.ts @@ -1,6 +1,6 @@ import { DEFAULT_COMPARE_OPTIONS } from '../utils' +import { checkCollectionSizeForIndex, isDevModeEnabled } from './index-registry' import { hasVirtualPropPath } from '../virtual-props' -import { BTreeIndex } from './btree-index' import type { CompareOptions } from '../query/builder/types' import type { BasicExpression } from '../query/ir' import type { CollectionImpl } from '../collection/index.js' @@ -11,11 +11,9 @@ export interface AutoIndexConfig { function shouldAutoIndex(collection: CollectionImpl) { // Only proceed if auto-indexing is enabled - if (collection.config.autoIndex !== `eager`) { - return false - } - - return true + // Note: autoIndex: 'eager' without defaultIndexType is caught at construction time + // in CollectionImpl, so we don't need to check for it here. + return collection.config.autoIndex === `eager` } export function ensureIndexForField< @@ -50,9 +48,18 @@ export function ensureIndexForField< return // Index already exists } + // Dev mode: check if collection size warrants an index suggestion + if (isDevModeEnabled()) { + checkCollectionSizeForIndex( + collection.id || `unknown`, + collection.size, + fieldPath, + ) + } + // Create a new index for this field using the collection's createIndex method + // The collection will use its defaultIndexType try { - // Use the proxy-based approach to create the proper accessor for nested paths collection.createIndex( (row) => { // Navigate through the field path @@ -64,7 +71,6 @@ export function ensureIndexForField< }, { name: `auto:${fieldPath.join(`.`)}`, - indexType: BTreeIndex, options: compareFn ? { compareFn, compareOptions: compareOpts } : {}, }, ) diff --git a/packages/db/src/indexes/base-index.ts b/packages/db/src/indexes/base-index.ts index ac44ad0ae..4346450f9 100644 --- a/packages/db/src/indexes/base-index.ts +++ b/packages/db/src/indexes/base-index.ts @@ -26,7 +26,7 @@ export interface IndexStats { } export interface IndexInterface< - TKey extends string | number | undefined = string | number | undefined, + TKey extends string | number = string | number, > { add: (key: TKey, item: any) => void remove: (key: TKey, item: any) => void @@ -79,7 +79,7 @@ export interface IndexInterface< * Base abstract class that all index types extend */ export abstract class BaseIndex< - TKey extends string | number | undefined = string | number | undefined, + TKey extends string | number = string | number, > implements IndexInterface { public readonly id: number public readonly name?: string @@ -189,7 +189,6 @@ export abstract class BaseIndex< } } - // Protected methods for subclasses protected abstract initialize(options?: any): void protected evaluateIndexExpression(item: any): any { @@ -218,10 +217,3 @@ export type IndexConstructor = name?: string, options?: any, ) => BaseIndex - -/** - * Index resolver can be either a class constructor or async loader - */ -export type IndexResolver = - | IndexConstructor - | (() => Promise>) diff --git a/packages/db/src/indexes/basic-index.ts b/packages/db/src/indexes/basic-index.ts new file mode 100644 index 000000000..b80f7fb43 --- /dev/null +++ b/packages/db/src/indexes/basic-index.ts @@ -0,0 +1,507 @@ +import { defaultComparator, normalizeValue } from '../utils/comparison.js' +import { + deleteInSortedArray, + findInsertPositionInArray, +} from '../utils/array-utils.js' +import { BaseIndex } from './base-index.js' +import type { CompareOptions } from '../query/builder/types.js' +import type { BasicExpression } from '../query/ir.js' +import type { IndexOperation } from './base-index.js' + +/** + * Options for range queries + */ +export interface RangeQueryOptions { + from?: any + to?: any + fromInclusive?: boolean + toInclusive?: boolean +} + +/** + * Options for Basic index + */ +export interface BasicIndexOptions { + compareFn?: (a: any, b: any) => number + compareOptions?: CompareOptions +} + +/** + * Basic index using Map + sorted Array. + * + * - Map for O(1) equality lookups + * - Sorted Array for O(log n) range queries via binary search + * - O(n) updates to maintain sort order + * + * Simpler and smaller than BTreeIndex, good for read-heavy workloads. + * Use BTreeIndex for write-heavy workloads with large collections. + */ +export class BasicIndex< + TKey extends string | number = string | number, +> extends BaseIndex { + public readonly supportedOperations = new Set([ + `eq`, + `gt`, + `gte`, + `lt`, + `lte`, + `in`, + ]) + + // Map for O(1) equality lookups: indexedValue -> Set of PKs + private valueMap = new Map>() + // Sorted array of unique indexed values for range queries + private sortedValues: Array = [] + // Set of all indexed PKs + private indexedKeys = new Set() + // Comparator function + private compareFn: (a: any, b: any) => number = defaultComparator + + constructor( + id: number, + expression: BasicExpression, + name?: string, + options?: any, + ) { + super(id, expression, name, options) + this.compareFn = options?.compareFn ?? defaultComparator + if (options?.compareOptions) { + this.compareOptions = options!.compareOptions + } + } + + protected initialize(_options?: BasicIndexOptions): void {} + + /** + * Adds a value to the index + */ + add(key: TKey, item: any): void { + let indexedValue: any + try { + indexedValue = this.evaluateIndexExpression(item) + } catch (error) { + throw new Error( + `Failed to evaluate index expression for key ${key}: ${error}`, + { cause: error }, + ) + } + + const normalizedValue = normalizeValue(indexedValue) + + if (this.valueMap.has(normalizedValue)) { + // Value already exists, just add the key to the set + this.valueMap.get(normalizedValue)!.add(key) + } else { + // New value - add to map and insert into sorted array + this.valueMap.set(normalizedValue, new Set([key])) + + // Insert into sorted position + const insertIdx = findInsertPositionInArray( + this.sortedValues, + normalizedValue, + this.compareFn, + ) + this.sortedValues.splice(insertIdx, 0, normalizedValue) + } + + this.indexedKeys.add(key) + this.updateTimestamp() + } + + /** + * Removes a value from the index + */ + remove(key: TKey, item: any): void { + let indexedValue: any + try { + indexedValue = this.evaluateIndexExpression(item) + } catch (error) { + console.warn( + `Failed to evaluate index expression for key ${key} during removal:`, + error, + ) + this.indexedKeys.delete(key) + this.updateTimestamp() + return + } + + const normalizedValue = normalizeValue(indexedValue) + + if (this.valueMap.has(normalizedValue)) { + const keySet = this.valueMap.get(normalizedValue)! + keySet.delete(key) + + if (keySet.size === 0) { + // No more keys for this value, remove from map and sorted array + this.valueMap.delete(normalizedValue) + deleteInSortedArray(this.sortedValues, normalizedValue, this.compareFn) + } + } + + this.indexedKeys.delete(key) + this.updateTimestamp() + } + + /** + * Updates a value in the index + */ + update(key: TKey, oldItem: any, newItem: any): void { + this.remove(key, oldItem) + this.add(key, newItem) + } + + /** + * Builds the index from a collection of entries + */ + build(entries: Iterable<[TKey, any]>): void { + this.clear() + + // Collect all entries first + const entriesArray: Array<{ key: TKey; value: any }> = [] + for (const [key, item] of entries) { + let indexedValue: any + try { + indexedValue = this.evaluateIndexExpression(item) + } catch (error) { + throw new Error( + `Failed to evaluate index expression for key ${key}: ${error}`, + { cause: error }, + ) + } + entriesArray.push({ key, value: normalizeValue(indexedValue) }) + this.indexedKeys.add(key) + } + + // Group by value + for (const { key, value } of entriesArray) { + if (this.valueMap.has(value)) { + this.valueMap.get(value)!.add(key) + } else { + this.valueMap.set(value, new Set([key])) + } + } + + // Build sorted array from unique values + this.sortedValues = Array.from(this.valueMap.keys()).sort(this.compareFn) + + this.updateTimestamp() + } + + /** + * Clears all data from the index + */ + clear(): void { + this.valueMap.clear() + this.sortedValues = [] + this.indexedKeys.clear() + this.updateTimestamp() + } + + /** + * Performs a lookup operation + */ + lookup(operation: IndexOperation, value: any): Set { + const startTime = performance.now() + + let result: Set + + switch (operation) { + case `eq`: + result = this.equalityLookup(value) + break + case `gt`: + result = this.rangeQuery({ from: value, fromInclusive: false }) + break + case `gte`: + result = this.rangeQuery({ from: value, fromInclusive: true }) + break + case `lt`: + result = this.rangeQuery({ to: value, toInclusive: false }) + break + case `lte`: + result = this.rangeQuery({ to: value, toInclusive: true }) + break + case `in`: + result = this.inArrayLookup(value) + break + default: + throw new Error(`Operation ${operation} not supported by BasicIndex`) + } + + this.trackLookup(startTime) + return result + } + + /** + * Gets the number of indexed keys + */ + get keyCount(): number { + return this.indexedKeys.size + } + + /** + * Performs an equality lookup - O(1) + */ + equalityLookup(value: any): Set { + const normalizedValue = normalizeValue(value) + return this.valueMap.get(normalizedValue) ?? new Set() + } + + /** + * Performs a range query using binary search - O(log n + m) + */ + rangeQuery(options: RangeQueryOptions = {}): Set { + const { from, to, fromInclusive = true, toInclusive = true } = options + const result = new Set() + + if (this.sortedValues.length === 0) { + return result + } + + const normalizedFrom = normalizeValue(from) + const normalizedTo = normalizeValue(to) + + // Find start index + let startIdx = 0 + if (normalizedFrom !== undefined) { + startIdx = findInsertPositionInArray( + this.sortedValues, + normalizedFrom, + this.compareFn, + ) + // If not inclusive and we found exact match, skip it + if ( + !fromInclusive && + startIdx < this.sortedValues.length && + this.compareFn(this.sortedValues[startIdx], normalizedFrom) === 0 + ) { + startIdx++ + } + } + + // Find end index + let endIdx = this.sortedValues.length + if (normalizedTo !== undefined) { + endIdx = findInsertPositionInArray( + this.sortedValues, + normalizedTo, + this.compareFn, + ) + // If inclusive and we found the value, include it + if ( + toInclusive && + endIdx < this.sortedValues.length && + this.compareFn(this.sortedValues[endIdx], normalizedTo) === 0 + ) { + endIdx++ + } + } + + // Collect all keys in range + for (let i = startIdx; i < endIdx; i++) { + const keys = this.valueMap.get(this.sortedValues[i]) + if (keys) { + keys.forEach((key) => result.add(key)) + } + } + + return result + } + + /** + * Performs a reversed range query + */ + rangeQueryReversed(options: RangeQueryOptions = {}): Set { + const { from, to, fromInclusive = true, toInclusive = true } = options + + // Swap from/to and fromInclusive/toInclusive to handle reversed ranges + // If to is undefined, we want to start from the end (max value) + // If from is undefined, we want to end at the beginning (min value) + const swappedFrom = + to ?? + (this.sortedValues.length > 0 + ? this.sortedValues[this.sortedValues.length - 1] + : undefined) + const swappedTo = + from ?? (this.sortedValues.length > 0 ? this.sortedValues[0] : undefined) + + return this.rangeQuery({ + from: swappedFrom, + to: swappedTo, + fromInclusive: toInclusive, + toInclusive: fromInclusive, + }) + } + + /** + * Returns the next n items in sorted order + */ + take(n: number, from?: any, filterFn?: (key: TKey) => boolean): Array { + const result: Array = [] + + let startIdx = 0 + if (from !== undefined) { + const normalizedFrom = normalizeValue(from) + startIdx = findInsertPositionInArray( + this.sortedValues, + normalizedFrom, + this.compareFn, + ) + // Skip past the 'from' value (exclusive) + while ( + startIdx < this.sortedValues.length && + this.compareFn(this.sortedValues[startIdx], normalizedFrom) <= 0 + ) { + startIdx++ + } + } + + for ( + let i = startIdx; + i < this.sortedValues.length && result.length < n; + i++ + ) { + const keys = this.valueMap.get(this.sortedValues[i]) + if (keys) { + for (const key of keys) { + if (result.length >= n) break + if (!filterFn || filterFn(key)) { + result.push(key) + } + } + } + } + + return result + } + + /** + * Returns the next n items in reverse sorted order + */ + takeReversed( + n: number, + from?: any, + filterFn?: (key: TKey) => boolean, + ): Array { + const result: Array = [] + + let startIdx = this.sortedValues.length - 1 + if (from !== undefined) { + const normalizedFrom = normalizeValue(from) + startIdx = + findInsertPositionInArray( + this.sortedValues, + normalizedFrom, + this.compareFn, + ) - 1 + // Skip past the 'from' value (exclusive) + while ( + startIdx >= 0 && + this.compareFn(this.sortedValues[startIdx], normalizedFrom) >= 0 + ) { + startIdx-- + } + } + + for (let i = startIdx; i >= 0 && result.length < n; i--) { + const keys = this.valueMap.get(this.sortedValues[i]) + if (keys) { + for (const key of keys) { + if (result.length >= n) break + if (!filterFn || filterFn(key)) { + result.push(key) + } + } + } + } + + return result + } + + /** + * Returns the first n items in sorted order (from the start) + */ + takeFromStart(n: number, filterFn?: (key: TKey) => boolean): Array { + const result: Array = [] + for (let i = 0; i < this.sortedValues.length && result.length < n; i++) { + const keys = this.valueMap.get(this.sortedValues[i]) + if (keys) { + for (const key of keys) { + if (result.length >= n) break + if (!filterFn || filterFn(key)) { + result.push(key) + } + } + } + } + return result + } + + /** + * Returns the first n items in reverse sorted order (from the end) + */ + takeReversedFromEnd( + n: number, + filterFn?: (key: TKey) => boolean, + ): Array { + const result: Array = [] + for ( + let i = this.sortedValues.length - 1; + i >= 0 && result.length < n; + i-- + ) { + const keys = this.valueMap.get(this.sortedValues[i]) + if (keys) { + for (const key of keys) { + if (result.length >= n) break + if (!filterFn || filterFn(key)) { + result.push(key) + } + } + } + } + return result + } + + /** + * Performs an IN array lookup - O(k) where k is values.length + */ + inArrayLookup(values: Array): Set { + const result = new Set() + + for (const value of values) { + const normalizedValue = normalizeValue(value) + const keys = this.valueMap.get(normalizedValue) + if (keys) { + keys.forEach((key) => result.add(key)) + } + } + + return result + } + + // Getter methods for testing/compatibility + get indexedKeysSet(): Set { + return this.indexedKeys + } + + get orderedEntriesArray(): Array<[any, Set]> { + return this.sortedValues.map((value) => [ + value, + this.valueMap.get(value) ?? new Set(), + ]) + } + + get orderedEntriesArrayReversed(): Array<[any, Set]> { + const result: Array<[any, Set]> = [] + for (let i = this.sortedValues.length - 1; i >= 0; i--) { + const value = this.sortedValues[i] + result.push([value, this.valueMap.get(value) ?? new Set()]) + } + return result + } + + get valueMapData(): Map> { + return this.valueMap + } +} diff --git a/packages/db/src/indexes/btree-index.ts b/packages/db/src/indexes/btree-index.ts index 2106dabf9..17608950a 100644 --- a/packages/db/src/indexes/btree-index.ts +++ b/packages/db/src/indexes/btree-index.ts @@ -33,7 +33,7 @@ export interface RangeQueryOptions { * This maintains items in sorted order and provides efficient range operations */ export class BTreeIndex< - TKey extends string | number | undefined = string | number | undefined, + TKey extends string | number = string | number, > extends BaseIndex { public readonly supportedOperations = new Set([ `eq`, diff --git a/packages/db/src/indexes/index-options.ts b/packages/db/src/indexes/index-options.ts index 6dfbc137b..7ea980fb0 100644 --- a/packages/db/src/indexes/index-options.ts +++ b/packages/db/src/indexes/index-options.ts @@ -1,42 +1,22 @@ -import type { IndexConstructor, IndexResolver } from './base-index.js' +import type { IndexConstructor } from './base-index.js' /** - * Enhanced index options that support both sync and async resolvers + * Options for creating an index */ -export interface IndexOptions { +export interface IndexOptions< + TIndexType extends IndexConstructor = IndexConstructor, +> { + /** Optional name for the index */ name?: string - indexType?: TResolver - options?: TResolver extends IndexConstructor - ? TResolver extends new ( - id: number, - expr: any, - name?: string, - options?: infer O, - ) => any - ? O - : never - : TResolver extends () => Promise - ? TCtor extends new ( - id: number, - expr: any, - name?: string, - options?: infer O, - ) => any - ? O - : never - : never + /** Index type to use (e.g., BasicIndex, BTreeIndex) */ + indexType?: TIndexType + /** Options passed to the index constructor */ + options?: TIndexType extends new ( + id: number, + expr: any, + name?: string, + options?: infer O, + ) => any + ? O + : never } - -/** - * Utility type to extract the constructed index type from a resolver - */ -export type ResolvedIndexType = - TResolver extends IndexConstructor - ? InstanceType - : TResolver extends () => Promise> - ? TResolver extends () => Promise - ? TCtor extends IndexConstructor - ? InstanceType - : never - : never - : never diff --git a/packages/db/src/indexes/index-registry.ts b/packages/db/src/indexes/index-registry.ts new file mode 100644 index 000000000..5fce43ddf --- /dev/null +++ b/packages/db/src/indexes/index-registry.ts @@ -0,0 +1,174 @@ +/** + * Index Dev Mode - Helps developers identify when indexes would improve performance + * + * Dev mode suggestions are ON by default in non-production builds. + */ + +// Dev mode detection settings - ON by default in non-production +let devModeConfig: IndexDevModeConfig = { + enabled: true, + collectionSizeThreshold: 1000, + slowQueryThresholdMs: 10, + onSuggestion: null, +} + +export interface IndexDevModeConfig { + /** Enable dev mode index suggestions */ + enabled: boolean + /** Suggest indexes when collection has more than this many items */ + collectionSizeThreshold: number + /** Suggest indexes when queries take longer than this (ms) */ + slowQueryThresholdMs: number + /** Custom handler for index suggestions */ + onSuggestion: ((suggestion: IndexSuggestion) => void) | null +} + +export interface IndexSuggestion { + type: `collection-size` | `slow-query` | `frequent-field` + collectionId: string + fieldPath: Array + message: string + collectionSize?: number + queryTimeMs?: number + queryCount?: number +} + +// Track query patterns for dev mode +const queryPatterns = new Map< + string, + { + fieldPath: Array + queryCount: number + totalTimeMs: number + avgTimeMs: number + } +>() + +/** + * Configure dev mode for index suggestions + */ +export function configureIndexDevMode( + config: Partial, +): void { + devModeConfig = { ...devModeConfig, ...config } +} + +/** + * Get current dev mode configuration + */ +export function getIndexDevModeConfig(): IndexDevModeConfig { + return devModeConfig +} + +/** + * Check if dev mode is enabled + */ +export function isDevModeEnabled(): boolean { + return devModeConfig.enabled && process.env.NODE_ENV !== `production` +} + +/** + * Emit an index suggestion (dev mode only) + */ +export function emitIndexSuggestion(suggestion: IndexSuggestion): void { + if (!isDevModeEnabled()) return + + if (devModeConfig.onSuggestion) { + try { + devModeConfig.onSuggestion(suggestion) + } catch { + // Don't let a buggy callback crash query execution + } + } else { + // Default: log to console with helpful formatting + console.warn( + `[TanStack DB] Index suggestion for "${suggestion.collectionId}":\n` + + ` ${suggestion.message}\n` + + ` Field: ${suggestion.fieldPath.join(`.`)}\n` + + ` Add index: collection.createIndex((row) => row.${suggestion.fieldPath.join(`.`)})`, + ) + } +} + +/** + * Track a query for dev mode analysis + */ +export function trackQuery( + collectionId: string, + fieldPath: Array, + executionTimeMs: number, +): void { + if (!isDevModeEnabled()) return + + const key = `${collectionId}:${fieldPath.join(`.`)}` + const existing = queryPatterns.get(key) + + if (existing) { + existing.queryCount++ + existing.totalTimeMs += executionTimeMs + existing.avgTimeMs = existing.totalTimeMs / existing.queryCount + } else { + queryPatterns.set(key, { + fieldPath, + queryCount: 1, + totalTimeMs: executionTimeMs, + avgTimeMs: executionTimeMs, + }) + } + + // Check if we should suggest an index + const pattern = queryPatterns.get(key)! + if (pattern.avgTimeMs > devModeConfig.slowQueryThresholdMs) { + emitIndexSuggestion({ + type: `slow-query`, + collectionId, + fieldPath, + message: `Queries on "${fieldPath.join(`.`)}" are slow (avg ${pattern.avgTimeMs.toFixed(1)}ms). Consider adding an index.`, + queryTimeMs: pattern.avgTimeMs, + queryCount: pattern.queryCount, + }) + } +} + +/** + * Check collection size and suggest index if needed (dev mode) + */ +export function checkCollectionSizeForIndex( + collectionId: string, + collectionSize: number, + fieldPath: Array, +): void { + if (!isDevModeEnabled()) return + + if (collectionSize > devModeConfig.collectionSizeThreshold) { + emitIndexSuggestion({ + type: `collection-size`, + collectionId, + fieldPath, + message: `Collection has ${collectionSize} items. Queries on "${fieldPath.join(`.`)}" may benefit from an index.`, + collectionSize, + }) + } +} + +/** + * Clear query pattern tracking (useful for tests) + */ +export function clearQueryPatterns(): void { + queryPatterns.clear() +} + +/** + * Get query patterns (useful for debugging/testing) + */ +export function getQueryPatterns(): Map< + string, + { + fieldPath: Array + queryCount: number + totalTimeMs: number + avgTimeMs: number + } +> { + return new Map(queryPatterns) +} diff --git a/packages/db/src/indexes/lazy-index.ts b/packages/db/src/indexes/lazy-index.ts deleted file mode 100644 index a025632cc..000000000 --- a/packages/db/src/indexes/lazy-index.ts +++ /dev/null @@ -1,276 +0,0 @@ -import type { - BaseIndex, - IndexConstructor, - IndexResolver, -} from './base-index.js' -import type { BasicExpression } from '../query/ir.js' - -export class IndexRemovedError extends Error { - constructor(indexId: number) { - super(`Index ${indexId} has been removed from its collection.`) - this.name = `IndexRemovedError` - } -} - -/** - * Utility to determine if a resolver is a constructor or async loader - */ -function isConstructor( - resolver: IndexResolver, -): resolver is IndexConstructor { - // Check if it's a function with a prototype (constructor) - return ( - typeof resolver === `function` && - resolver.prototype !== undefined && - resolver.prototype.constructor === resolver - ) -} - -/** - * Resolve index constructor from resolver - */ -async function resolveIndexConstructor( - resolver: IndexResolver, -): Promise> { - if (isConstructor(resolver)) { - return resolver - } else { - // It's an async loader function - return await resolver() - } -} - -/** - * Wrapper that defers index creation until first sync - */ -export class LazyIndexWrapper { - private indexPromise: Promise> | null = null - private resolvedIndex: BaseIndex | null = null - private removed = false - - constructor( - private id: number, - private expression: BasicExpression, - private name: string | undefined, - private resolver: IndexResolver, - private options: any, - private collectionEntries?: Iterable<[TKey, any]>, - ) { - // For synchronous constructors, resolve immediately - if (isConstructor(this.resolver)) { - this.resolvedIndex = new this.resolver( - this.id, - this.expression, - this.name, - this.options, - ) - // Build with initial data if provided - if (this.collectionEntries) { - this.resolvedIndex.build(this.collectionEntries) - } - } - } - - /** - * Resolve the actual index - */ - async resolve(): Promise> { - this.throwIfRemoved() - - if (this.resolvedIndex) { - return this.resolvedIndex - } - - if (!this.indexPromise) { - this.indexPromise = this.createIndex() - } - - this.resolvedIndex = await this.indexPromise - this.throwIfRemoved() - return this.resolvedIndex - } - - /** - * Check if already resolved - */ - isResolved(): boolean { - return !this.removed && this.resolvedIndex !== null - } - - /** - * Get resolved index (throws if not ready) - */ - getResolved(): BaseIndex { - this.throwIfRemoved() - - if (!this.resolvedIndex) { - throw new Error( - `Index ${this.id} has not been resolved yet. Ensure collection is synced.`, - ) - } - return this.resolvedIndex - } - - /** - * Get the index ID - */ - getId(): number { - return this.id - } - - markRemoved(): void { - this.removed = true - this.resolvedIndex = null - this.indexPromise = null - } - - /** - * Get the index name - */ - getName(): string | undefined { - return this.name - } - - /** - * Get the index expression - */ - getExpression(): BasicExpression { - return this.expression - } - - private async createIndex(): Promise> { - const IndexClass = await resolveIndexConstructor(this.resolver) - return new IndexClass(this.id, this.expression, this.name, this.options) - } - - private throwIfRemoved(): void { - if (this.removed) { - throw new IndexRemovedError(this.id) - } - } -} - -/** - * Proxy that provides synchronous interface while index loads asynchronously - */ -export class IndexProxy { - constructor( - private indexId: number, - private lazyIndex: LazyIndexWrapper, - ) {} - - /** - * Get the resolved index (throws if not ready) - */ - get index(): BaseIndex { - return this.lazyIndex.getResolved() - } - - /** - * Check if index is ready - */ - get isReady(): boolean { - return this.lazyIndex.isResolved() - } - - /** - * Wait for index to be ready - */ - async whenReady(): Promise> { - return await this.lazyIndex.resolve() - } - - /** - * Get the index ID - */ - get id(): number { - return this.indexId - } - - /** - * Get the index name (throws if not ready) - */ - get name(): string | undefined { - if (this.isReady) { - return this.index.name - } - return this.lazyIndex.getName() - } - - /** - * Get the index expression (available immediately) - */ - get expression(): BasicExpression { - return this.lazyIndex.getExpression() - } - - /** - * Check if index supports an operation (throws if not ready) - */ - supports(operation: any): boolean { - return this.index.supports(operation) - } - - /** - * Get index statistics (throws if not ready) - */ - getStats() { - return this.index.getStats() - } - - /** - * Check if index matches a field path (available immediately) - */ - matchesField(fieldPath: Array): boolean { - const expr = this.expression - return ( - expr.type === `ref` && - expr.path.length === fieldPath.length && - expr.path.every((part, i) => part === fieldPath[i]) - ) - } - - /** - * Get the key count (throws if not ready) - */ - get keyCount(): number { - return this.index.keyCount - } - - // Test compatibility properties - delegate to resolved index - get indexedKeysSet(): Set { - const resolved = this.index as any - return resolved.indexedKeysSet - } - - get orderedEntriesArray(): Array<[any, Set]> { - const resolved = this.index as any - return resolved.orderedEntriesArray - } - - get valueMapData(): Map> { - const resolved = this.index as any - return resolved.valueMapData - } - - // BTreeIndex compatibility methods - equalityLookup(value: any): Set { - const resolved = this.index as any - return resolved.equalityLookup?.(value) ?? new Set() - } - - rangeQuery(options: any): Set { - const resolved = this.index as any - return resolved.rangeQuery?.(options) ?? new Set() - } - - inArrayLookup(values: Array): Set { - const resolved = this.index as any - return resolved.inArrayLookup?.(values) ?? new Set() - } - - // Internal method for the collection to get the lazy wrapper - _getLazyWrapper(): LazyIndexWrapper { - return this.lazyIndex - } -} diff --git a/packages/db/src/query/compiler/order-by.ts b/packages/db/src/query/compiler/order-by.ts index 0d07a1a41..ec8fa1839 100644 --- a/packages/db/src/query/compiler/order-by.ts +++ b/packages/db/src/query/compiler/order-by.ts @@ -290,15 +290,14 @@ export function processOrderBy( optimizableOrderByCollections[targetCollectionId] = orderByOptimizationInfo - // Set up lazy loading callback if we have an index - if (index) { - setSizeCallback = (getSize: () => number) => { - optimizableOrderByCollections[targetCollectionId]![`dataNeeded`] = - () => { - const size = getSize() - return Math.max(0, orderByOptimizationInfo!.limit - size) - } - } + // Set up lazy loading callback to track how much more data is needed + // This is used by loadMoreIfNeeded to determine if more data should be loaded + setSizeCallback = (getSize: () => number) => { + optimizableOrderByCollections[targetCollectionId]![`dataNeeded`] = + () => { + const size = getSize() + return Math.max(0, orderByOptimizationInfo!.limit - size) + } } } } diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 0dbd01780..6087e234e 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -4,6 +4,7 @@ import type { StandardSchemaV1 } from '@standard-schema/spec' import type { Transaction } from './transactions' import type { BasicExpression, OrderBy } from './query/ir.js' import type { EventEmitter } from './event-emitter.js' +import type { IndexConstructor } from './indexes/base-index.js' import type { SingleRowRefProxy } from './query/builder/ref-proxy.js' import type { WithVirtualProps } from './virtual-props.js' @@ -561,12 +562,27 @@ export interface BaseCollectionConfig< /** * Auto-indexing mode for the collection. * When enabled, indexes will be automatically created for simple where expressions. - * @default "eager" + * @default "off" * @description - * - "off": No automatic indexing - * - "eager": Automatically create indexes for simple where expressions in subscribeChanges (default) + * - "off": No automatic indexing (default). Use explicit indexes for better bundle size. + * - "eager": Automatically create indexes for simple where expressions in subscribeChanges. + * Requires setting defaultIndexType. */ autoIndex?: `off` | `eager` + /** + * Default index type to use when creating indexes without an explicit type. + * Required for auto-indexing. Import from '@tanstack/db'. + * @example + * ```ts + * import { BasicIndex } from '@tanstack/db' + * const collection = createCollection({ + * defaultIndexType: BasicIndex, + * autoIndex: 'eager', + * // ... + * }) + * ``` + */ + defaultIndexType?: IndexConstructor /** * Optional function to compare two items. * This is used to order the items in the collection. diff --git a/packages/db/src/utils/array-utils.ts b/packages/db/src/utils/array-utils.ts index 8b2cdbefd..47569cacb 100644 --- a/packages/db/src/utils/array-utils.ts +++ b/packages/db/src/utils/array-utils.ts @@ -5,6 +5,35 @@ * @param compareFn Comparison function to use for ordering * @returns The index where the value should be inserted to maintain order */ +export function findInsertPositionInArray( + sortedArray: Array, + value: T, + compareFn: (a: T, b: T) => number, +): number { + let left = 0 + let right = sortedArray.length + + while (left < right) { + const mid = Math.floor((left + right) / 2) + const comparison = compareFn(sortedArray[mid]!, value) + + if (comparison < 0) { + left = mid + 1 + } else { + right = mid + } + } + + return left +} + +/** + * Finds the correct insert position for a value in a sorted tuple array using binary search + * @param sortedArray The sorted tuple array to search in + * @param value The value to find the position for + * @param compareFn Comparison function to use for ordering + * @returns The index where the value should be inserted to maintain order + */ export function findInsertPosition( sortedArray: Array<[T, any]>, value: T, @@ -26,3 +55,23 @@ export function findInsertPosition( return left } + +/** + * Deletes a value from a sorted array while maintaining sort order + * @param sortedArray The sorted array to delete from + * @param value The value to delete + * @param compareFn Comparison function to use for ordering + * @returns True if the value was found and deleted, false otherwise + */ +export function deleteInSortedArray( + sortedArray: Array, + value: T, + compareFn: (a: T, b: T) => number, +): boolean { + const idx = findInsertPositionInArray(sortedArray, value, compareFn) + if (idx < sortedArray.length && compareFn(sortedArray[idx]!, value) === 0) { + sortedArray.splice(idx, 1) + return true + } + return false +} diff --git a/packages/db/tests/btree-index-undefined-values.test.ts b/packages/db/tests/btree-index-undefined-values.test.ts index 0d29678c3..11e29690c 100644 --- a/packages/db/tests/btree-index-undefined-values.test.ts +++ b/packages/db/tests/btree-index-undefined-values.test.ts @@ -34,6 +34,7 @@ describe(`BTreeIndex - Issue #1186: Infinite loop with undefined indexed values` collection = createCollection({ id: `test-collection`, getKey: (item) => item.id, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { diff --git a/packages/db/tests/collection-auto-index.test.ts b/packages/db/tests/collection-auto-index.test.ts index f40db1e61..4fdaac012 100644 --- a/packages/db/tests/collection-auto-index.test.ts +++ b/packages/db/tests/collection-auto-index.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest' +import { CollectionConfigurationError } from '../src/errors' import { createCollection } from '../src/collection/index.js' import { and, @@ -12,6 +13,7 @@ import { import { createSingleRowRefProxy } from '../src/query/builder/ref-proxy' import { createLiveQueryCollection } from '../src' import { PropRef } from '../src/query/ir' +import { BTreeIndex } from '../src/indexes/btree-index' import { createIndexUsageTracker, expectIndexUsage, @@ -121,7 +123,7 @@ describe(`Collection Auto-Indexing`, () => { subscription.unsubscribe() }) - it(`should create auto-indexes by default when autoIndex is not specified`, async () => { + it(`should NOT create auto-indexes by default when autoIndex is not specified`, async () => { const autoIndexCollection = createCollection({ getKey: (item) => item.id, startSync: true, @@ -157,20 +159,54 @@ describe(`Collection Auto-Indexing`, () => { }, ) - // Should have created an auto-index for the status field (default is eager) - expect(autoIndexCollection.indexes.size).toBe(1) - - const autoIndex = Array.from(autoIndexCollection.indexes.values())[0]! - expect(autoIndex.expression.type).toBe(`ref`) - expect((autoIndex.expression as any).path).toEqual([`status`]) + // Should NOT have created an auto-index (default is off) + expect(autoIndexCollection.indexes.size).toBe(0) subscription.unsubscribe() }) + it(`should throw CollectionConfigurationError when autoIndex is "eager" without defaultIndexType`, () => { + expect(() => + createCollection({ + getKey: (item) => item.id, + autoIndex: `eager`, + startSync: true, + sync: { + sync: ({ begin, commit, markReady }) => { + begin() + commit() + markReady() + }, + }, + }), + ).toThrow(CollectionConfigurationError) + }) + + it(`should throw CollectionConfigurationError when createIndex is called without indexType or defaultIndexType`, async () => { + const collection = createCollection({ + getKey: (item) => item.id, + startSync: true, + sync: { + sync: ({ begin, commit, markReady }) => { + begin() + commit() + markReady() + }, + }, + }) + + await collection.stateWhenReady() + + expect(() => collection.createIndex((row) => row.age)).toThrow( + CollectionConfigurationError, + ) + }) + it(`should create auto-indexes for simple where expressions when autoIndex is "eager"`, async () => { const autoIndexCollection = createCollection({ getKey: (item) => item.id, autoIndex: `eager`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -220,6 +256,7 @@ describe(`Collection Auto-Indexing`, () => { const autoIndexCollection = createCollection({ getKey: (item) => item.id, autoIndex: `eager`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -267,6 +304,7 @@ describe(`Collection Auto-Indexing`, () => { const autoIndexCollection = createCollection({ getKey: (item) => item.id, autoIndex: `eager`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -318,6 +356,7 @@ describe(`Collection Auto-Indexing`, () => { const autoIndexCollection = createCollection({ getKey: (item) => item.id, autoIndex: `eager`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -358,6 +397,7 @@ describe(`Collection Auto-Indexing`, () => { const autoIndexCollection = createCollection({ getKey: (item) => item.id, autoIndex: `eager`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -391,6 +431,7 @@ describe(`Collection Auto-Indexing`, () => { const autoIndexCollection = createCollection({ getKey: (item) => item.id, autoIndex: `eager`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -436,6 +477,7 @@ describe(`Collection Auto-Indexing`, () => { const leftCollection = createCollection({ getKey: (item) => item.id, autoIndex: `eager`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -456,6 +498,7 @@ describe(`Collection Auto-Indexing`, () => { const rightCollection = createCollection({ getKey: (item) => item.id2, autoIndex: `eager`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -548,6 +591,7 @@ describe(`Collection Auto-Indexing`, () => { const leftCollection = createCollection({ getKey: (item) => item.id, autoIndex: `eager`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -568,6 +612,7 @@ describe(`Collection Auto-Indexing`, () => { const rightCollection = createCollection({ getKey: (item) => item.id2, autoIndex: `eager`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -667,6 +712,7 @@ describe(`Collection Auto-Indexing`, () => { const autoIndexCollection = createCollection({ getKey: (item) => item.id, autoIndex: `eager`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -705,6 +751,7 @@ describe(`Collection Auto-Indexing`, () => { const autoIndexCollection = createCollection({ getKey: (item) => item.id, autoIndex: `eager`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -801,6 +848,7 @@ describe(`Collection Auto-Indexing`, () => { const collection = createCollection({ getKey: (item) => item.id, autoIndex: `eager`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { diff --git a/packages/db/tests/collection-change-events.test.ts b/packages/db/tests/collection-change-events.test.ts index be59dde14..085af31f0 100644 --- a/packages/db/tests/collection-change-events.test.ts +++ b/packages/db/tests/collection-change-events.test.ts @@ -3,6 +3,7 @@ import { createCollection } from '../src/collection/index.js' import { currentStateAsChanges } from '../src/collection/change-events.js' import { Func, PropRef, Value } from '../src/query/ir.js' import { DEFAULT_COMPARE_OPTIONS } from '../src/utils.js' +import { BTreeIndex } from '../src/indexes/btree-index.js' interface TestUser { id: string @@ -39,6 +40,7 @@ describe(`currentStateAsChanges`, () => { id: `test-collection-${autoIndex}`, getKey: (user) => user.id, autoIndex, + defaultIndexType: autoIndex === `eager` ? BTreeIndex : undefined, sync: { sync: mockSync, }, @@ -379,6 +381,7 @@ describe(`currentStateAsChanges`, () => { id: `test-collection-empty-${autoIndex}`, getKey: (user) => user.id, autoIndex: autoIndex as `eager` | `off`, + defaultIndexType: autoIndex === `eager` ? BTreeIndex : undefined, sync: { sync: mockSync, }, diff --git a/packages/db/tests/collection-events.test.ts b/packages/db/tests/collection-events.test.ts index 8514ecc06..97c6538ff 100644 --- a/packages/db/tests/collection-events.test.ts +++ b/packages/db/tests/collection-events.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { createCollection } from '../src/collection/index.js' import type { Collection } from '../src/collection/index.js' +import { BTreeIndex } from '../src/indexes/btree-index.js' describe(`Collection Events System`, () => { let collection: Collection @@ -11,6 +12,7 @@ describe(`Collection Events System`, () => { collection = createCollection({ id: `test-collection`, getKey: (item: any) => item.id, + defaultIndexType: BTreeIndex, sync: { sync: mockSync, }, diff --git a/packages/db/tests/collection-indexes.test.ts b/packages/db/tests/collection-indexes.test.ts index ea6c953eb..a441a5520 100644 --- a/packages/db/tests/collection-indexes.test.ts +++ b/packages/db/tests/collection-indexes.test.ts @@ -102,6 +102,8 @@ describe(`Collection Indexes`, () => { collection = createCollection({ getKey: (item) => item.id, startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, sync: { sync: ({ begin, write, commit, markReady }) => { // Provide initial data through sync @@ -213,6 +215,7 @@ describe(`Collection Indexes`, () => { const otherCollection = createCollection({ getKey: (item) => item.id, startSync: true, + defaultIndexType: BTreeIndex, sync: { sync: ({ begin, write, commit, markReady }) => { begin() @@ -278,6 +281,7 @@ describe(`Collection Indexes`, () => { const preSyncIndex = lazyCollection.createIndex((row) => row.status, { name: `statusIndex`, + indexType: BTreeIndex, }) const snapshot = lazyCollection.getIndexMetadata() @@ -306,34 +310,15 @@ describe(`Collection Indexes`, () => { expect(snapshotB[0]!.resolver.kind).toBe(`constructor`) }) - it(`should invalidate removed index proxies`, async () => { + it(`should remove index from collection`, () => { const statusIndex = collection.createIndex((row) => row.status) expect(collection.removeIndex(statusIndex)).toBe(true) - expect(statusIndex.isReady).toBe(false) - expect(() => statusIndex.indexedKeysSet).toThrow( - `has been removed from its collection`, - ) - await expect(statusIndex.whenReady()).rejects.toThrow( - `has been removed from its collection`, - ) + expect(collection.indexes.has(statusIndex.id)).toBe(false) }) - it(`should not resurrect async indexes removed before they resolve`, async () => { - const delayedIndex = collection.createIndex((row) => row.age, { - indexType: async () => { - await new Promise((resolve) => setTimeout(resolve, 20)) - return BTreeIndex - }, - }) - - expect(collection.removeIndex(delayedIndex)).toBe(true) - await new Promise((resolve) => setTimeout(resolve, 35)) - - expect(collection.indexes.has(delayedIndex.id)).toBe(false) - await expect(delayedIndex.whenReady()).rejects.toThrow( - `has been removed from its collection`, - ) + it(`should return false when removing non-existent index`, () => { + expect(collection.removeIndex(999)).toBe(false) }) }) @@ -1482,6 +1467,7 @@ describe(`Collection Indexes`, () => { const specialCollection = createCollection({ getKey: (item) => item.id, startSync: true, + defaultIndexType: BTreeIndex, sync: { sync: ({ begin, write, commit }) => { begin() @@ -1543,6 +1529,7 @@ describe(`Collection Indexes`, () => { it(`should handle index creation on empty collection`, () => { const emptyCollection = createCollection({ getKey: (item) => item.id, + defaultIndexType: BTreeIndex, sync: { sync: () => {} }, }) diff --git a/packages/db/tests/collection-subscriber-duplicate-inserts.test.ts b/packages/db/tests/collection-subscriber-duplicate-inserts.test.ts index 774d600d6..8b9d6be57 100644 --- a/packages/db/tests/collection-subscriber-duplicate-inserts.test.ts +++ b/packages/db/tests/collection-subscriber-duplicate-inserts.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' import { createCollection } from '../src/collection/index.js' +import { BTreeIndex } from '../src/indexes/btree-index.js' import { createLiveQueryCollection, eq } from '../src/query/index.js' import { mockSyncCollectionOptions } from './utils.js' import type { ChangeMessage } from '../src/types.js' @@ -279,6 +280,8 @@ describe(`CollectionSubscriber duplicate insert prevention`, () => { id: `duplicate-d2-mutation-during-setup`, getKey: (item: TestItem) => item.id, initialData, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, }), ) diff --git a/packages/db/tests/deterministic-ordering.test.ts b/packages/db/tests/deterministic-ordering.test.ts index 8011e339e..9ce9a326d 100644 --- a/packages/db/tests/deterministic-ordering.test.ts +++ b/packages/db/tests/deterministic-ordering.test.ts @@ -260,6 +260,8 @@ describe(`Deterministic Ordering`, () => { { id: `multi-col-orderby-messages`, getKey: (item) => item.id, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { diff --git a/packages/db/tests/query/indexes.test.ts b/packages/db/tests/query/indexes.test.ts index 49b4c2c11..6abc065f6 100644 --- a/packages/db/tests/query/indexes.test.ts +++ b/packages/db/tests/query/indexes.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it } from 'vitest' import { createCollection } from '../../src/collection/index.js' +import { BTreeIndex } from '../../src/indexes/btree-index' import { createLiveQueryCollection } from '../../src/query/live-query-collection' import { @@ -234,6 +235,7 @@ function createTestItemCollection(autoIndex: `off` | `eager` = `off`) { getKey: (item) => item.id, initialData: testData, autoIndex, + defaultIndexType: BTreeIndex, }), ) } @@ -600,6 +602,7 @@ describe(`Query Index Optimization`, () => { const secondCollection = createCollection({ getKey: (item) => item.id, autoIndex: `off`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit }) => { @@ -895,6 +898,7 @@ describe(`Query Index Optimization`, () => { const secondCollection = createCollection({ getKey: (item) => item.id2, autoIndex: `off`, + defaultIndexType: BTreeIndex, startSync: true, sync: { sync: ({ begin, write, commit }) => { diff --git a/packages/db/tests/query/live-query-collection.test.ts b/packages/db/tests/query/live-query-collection.test.ts index 03a59b1d9..d56e5a715 100644 --- a/packages/db/tests/query/live-query-collection.test.ts +++ b/packages/db/tests/query/live-query-collection.test.ts @@ -17,6 +17,7 @@ import { stripVirtualProps, } from '../utils.js' import { createDeferred } from '../../src/deferred' +import { BTreeIndex } from '../../src/indexes/btree-index' import type { ChangeMessage, LoadSubsetOptions } from '../../src/types.js' // Sample user type for tests @@ -1784,6 +1785,7 @@ describe(`createLiveQueryCollection`, () => { mockSyncCollectionOptions({ id: `limited-users`, getKey: (user) => user.id, + autoIndex: `eager`, initialData: [ { id: 1, name: `Alice`, active: true }, { id: 2, name: `Bob`, active: true }, @@ -2029,6 +2031,8 @@ describe(`createLiveQueryCollection`, () => { getKey: (item) => item.id, syncMode: `on-demand`, startSync: true, + autoIndex: `eager`, // Enable auto-indexing for orderBy optimization + defaultIndexType: BTreeIndex, sync: { sync: ({ markReady, begin, write, commit }) => { // Provide minimal initial data @@ -2150,6 +2154,7 @@ describe(`createLiveQueryCollection`, () => { getKey: (item) => item.id, syncMode: `on-demand`, startSync: true, + defaultIndexType: BTreeIndex, autoIndex: `eager`, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -2215,6 +2220,7 @@ describe(`createLiveQueryCollection`, () => { getKey: (item) => item.id, syncMode: `on-demand`, startSync: true, + defaultIndexType: BTreeIndex, autoIndex: `eager`, sync: { sync: ({ begin, write, commit, markReady }) => { diff --git a/packages/db/tests/query/order-by.test.ts b/packages/db/tests/query/order-by.test.ts index dffdb9904..131c9d1ef 100644 --- a/packages/db/tests/query/order-by.test.ts +++ b/packages/db/tests/query/order-by.test.ts @@ -255,6 +255,10 @@ function createEmployeesWithNullableCollection( function createOrderByTests(autoIndex: `off` | `eager`): void { describe(`with autoIndex ${autoIndex}`, () => { + // Some tests require an index for incremental updates (loadMoreIfNeeded). + // These only work with autoIndex: 'eager' which auto-creates the needed indexes. + const itWhenAutoIndexEager = autoIndex === `eager` ? it : it.skip + let employeesCollection: ReturnType let departmentsCollection: ReturnType @@ -557,56 +561,59 @@ function createOrderByTests(autoIndex: `off` | `eager`): void { ]) }) - it(`applies incremental insert of a new row inside the topK but after max sent value correctly`, async () => { - const collection = createLiveQueryCollection((q) => - q - .from({ employees: employeesCollection }) - .orderBy(({ employees }) => employees.salary, `asc`) - .offset(1) - .limit(10) - .select(({ employees }) => ({ - id: employees.id, - name: employees.name, - salary: employees.salary, - })), - ) - await collection.preload() - - const results = Array.from(collection.values()) - - expect(results.map((r) => r.salary)).toEqual([ - 52_000, 55_000, 60_000, 65_000, - ]) + itWhenAutoIndexEager( + `applies incremental insert of a new row inside the topK but after max sent value correctly`, + async () => { + const collection = createLiveQueryCollection((q) => + q + .from({ employees: employeesCollection }) + .orderBy(({ employees }) => employees.salary, `asc`) + .offset(1) + .limit(10) + .select(({ employees }) => ({ + id: employees.id, + name: employees.name, + salary: employees.salary, + })), + ) + await collection.preload() - // Now insert a new employee with highest salary - // this should now become part of the topK because - // the topK isn't full yet, so even though it's after the max sent value - // it should still be part of the topK - const newEmployee = { - id: 6, - name: `George`, - department_id: 1, - salary: 72_000, - hire_date: `2023-01-01`, - } + const results = Array.from(collection.values()) - employeesCollection.utils.begin() - employeesCollection.utils.write({ - type: `insert`, - value: newEmployee, - }) - employeesCollection.utils.commit() + expect(results.map((r) => r.salary)).toEqual([ + 52_000, 55_000, 60_000, 65_000, + ]) - const newResults = Array.from(collection.values()) + // Now insert a new employee with highest salary + // this should now become part of the topK because + // the topK isn't full yet, so even though it's after the max sent value + // it should still be part of the topK + const newEmployee = { + id: 6, + name: `George`, + department_id: 1, + salary: 72_000, + hire_date: `2023-01-01`, + } - expect(newResults.map((r) => [r.id, r.salary])).toEqual([ - [5, 52_000], - [3, 55_000], - [2, 60_000], - [4, 65_000], - [6, 72_000], - ]) - }) + employeesCollection.utils.begin() + employeesCollection.utils.write({ + type: `insert`, + value: newEmployee, + }) + employeesCollection.utils.commit() + + const newResults = Array.from(collection.values()) + + expect(newResults.map((r) => [r.id, r.salary])).toEqual([ + [5, 52_000], + [3, 55_000], + [2, 60_000], + [4, 65_000], + [6, 72_000], + ]) + }, + ) it(`applies incremental insert of a new row after the topK correctly`, async () => { const collection = createLiveQueryCollection((q) => @@ -734,37 +741,40 @@ function createOrderByTests(autoIndex: `off` | `eager`): void { ]) }) - it(`handles deletion from partial page with limit larger than data`, async () => { - const collection = createLiveQueryCollection((q) => - q - .from({ employees: employeesCollection }) - .orderBy(({ employees }) => employees.salary, `desc`) - .limit(20) // Limit larger than number of employees (5) - .select(({ employees }) => ({ - id: employees.id, - name: employees.name, - salary: employees.salary, - })), - ) - await collection.preload() - - const results = Array.from(collection.values()) - expect(results).toHaveLength(5) - expect(results[0]!.name).toBe(`Diana`) - - // Delete Diana (the highest paid employee, first in DESC order) - const dianaData = employeeData.find((e) => e.id === 4)! - employeesCollection.utils.begin() - employeesCollection.utils.write({ - type: `delete`, - value: dianaData, - }) - employeesCollection.utils.commit() - - const newResults = Array.from(collection.values()) - expect(newResults).toHaveLength(4) - expect(newResults[0]!.name).toBe(`Bob`) - }) + itWhenAutoIndexEager( + `handles deletion from partial page with limit larger than data`, + async () => { + const collection = createLiveQueryCollection((q) => + q + .from({ employees: employeesCollection }) + .orderBy(({ employees }) => employees.salary, `desc`) + .limit(20) // Limit larger than number of employees (5) + .select(({ employees }) => ({ + id: employees.id, + name: employees.name, + salary: employees.salary, + })), + ) + await collection.preload() + + const results = Array.from(collection.values()) + expect(results).toHaveLength(5) + expect(results[0]!.name).toBe(`Diana`) + + // Delete Diana (the highest paid employee, first in DESC order) + const dianaData = employeeData.find((e) => e.id === 4)! + employeesCollection.utils.begin() + employeesCollection.utils.write({ + type: `delete`, + value: dianaData, + }) + employeesCollection.utils.commit() + + const newResults = Array.from(collection.values()) + expect(newResults).toHaveLength(4) + expect(newResults[0]!.name).toBe(`Bob`) + }, + ) }) describe(`OrderBy with Joins`, () => { diff --git a/packages/db/tests/query/query-while-syncing.test.ts b/packages/db/tests/query/query-while-syncing.test.ts index 5e4720cce..a1ea4591a 100644 --- a/packages/db/tests/query/query-while-syncing.test.ts +++ b/packages/db/tests/query/query-while-syncing.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest' import { createLiveQueryCollection, eq, gt } from '../../src/query/index.js' import { createCollection } from '../../src/collection/index.js' import { createTransaction } from '../../src/transactions.js' +import { BTreeIndex } from '../../src/indexes/btree-index.js' import { stripVirtualProps } from '../utils.js' // Sample user type for tests @@ -54,6 +55,7 @@ describe(`Query while syncing`, () => { id: `test-users-delayed-sync`, getKey: (user) => user.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -140,6 +142,7 @@ describe(`Query while syncing`, () => { id: `test-users-where-sync`, getKey: (user) => user.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -212,6 +215,7 @@ describe(`Query while syncing`, () => { id: `test-users-select-sync`, getKey: (user) => user.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -270,6 +274,7 @@ describe(`Query while syncing`, () => { id: `test-users-join-sync`, getKey: (user) => user.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -285,6 +290,7 @@ describe(`Query while syncing`, () => { id: `test-departments-join-sync`, getKey: (dept) => dept.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -392,6 +398,7 @@ describe(`Query while syncing`, () => { id: `test-users-left-join-sync`, getKey: (user) => user.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -407,6 +414,7 @@ describe(`Query while syncing`, () => { id: `test-departments-left-join-sync`, getKey: (dept) => dept.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -502,6 +510,7 @@ describe(`Query while syncing`, () => { id: `multi-source-1`, getKey: (user) => user.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -517,6 +526,7 @@ describe(`Query while syncing`, () => { id: `multi-source-2`, getKey: (user) => user.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -532,6 +542,7 @@ describe(`Query while syncing`, () => { id: `multi-source-3`, getKey: (user) => user.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -614,6 +625,7 @@ describe(`Query while syncing`, () => { id: `test-users-error-sync`, getKey: (user) => user.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit }) => { @@ -672,6 +684,7 @@ describe(`Query while syncing`, () => { id: `test-users-preload-sync`, getKey: (user) => user.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -766,6 +779,7 @@ describe(`Query while syncing`, () => { id: `test-users-where-preload`, getKey: (user) => user.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -849,6 +863,7 @@ describe(`Query while syncing`, () => { id: `test-users-join-preload`, getKey: (user) => user.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -864,6 +879,7 @@ describe(`Query while syncing`, () => { id: `test-departments-join-preload`, getKey: (dept) => dept.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -940,6 +956,7 @@ describe(`Query while syncing`, () => { id: `test-users-state-when-ready`, getKey: (user) => user.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -1009,6 +1026,7 @@ describe(`Query while syncing`, () => { id: `test-users-optimistic-mutations`, getKey: (user) => user.id, autoIndex, + ...(autoIndex === `eager` ? { defaultIndexType: BTreeIndex } : {}), startSync: false, sync: { sync: ({ begin, write, commit, markReady }) => { diff --git a/packages/db/tests/test-setup.ts b/packages/db/tests/test-setup.ts index a9d0dd31a..98bea44d1 100644 --- a/packages/db/tests/test-setup.ts +++ b/packages/db/tests/test-setup.ts @@ -1 +1,4 @@ import '@testing-library/jest-dom/vitest' + +// BTreeIndex is available via: import { BTreeIndex } from "../src/indexes/btree-index" +// Tests should pass defaultIndexType explicitly to collections when needed diff --git a/packages/db/tests/transactions.test.ts b/packages/db/tests/transactions.test.ts index a0c716052..d9f27667d 100644 --- a/packages/db/tests/transactions.test.ts +++ b/packages/db/tests/transactions.test.ts @@ -556,25 +556,54 @@ describe(`Transactions`, () => { describe(`duplicate instance detection`, () => { it(`sets a global marker in dev mode when in browser top window`, () => { - // The duplicate instance marker should be set when the module loads in dev mode + // The duplicate instance marker is set when the module loads in dev mode + // AND in a browser top-level window (not a worker, SSR, or iframe). const marker = Symbol.for(`@tanstack/db/instance-marker`) - // This will only be true if we're in dev mode AND in a browser top window - // In test environment (vitest), we should have these conditions met - expect((globalThis as any)[marker]).toBe(true) + const w = (globalThis as any).window + const isBrowserTopWindow = + w && + `document` in w && + (() => { + try { + return w === w.top + } catch { + return true + } + })() + + if (isBrowserTopWindow) { + expect((globalThis as any)[marker]).toBe(true) + } else { + // In Node.js / vitest (not a browser top window), the marker is not set + expect((globalThis as any)[marker]).toBeUndefined() + } }) it(`marker is only set in development mode`, () => { - // This test verifies the marker exists in our test environment - // In production (NODE_ENV=production), the marker would NOT be set + // This test verifies the marker behavior in our test environment. + // In production (NODE_ENV=production), the marker would NOT be set. + // In non-browser environments (Node.js / vitest), the marker is also not set. const marker = Symbol.for(`@tanstack/db/instance-marker`) const isDev = typeof process !== `undefined` && process.env.NODE_ENV !== `production` - - if (isDev) { + const w = (globalThis as any).window + const isBrowserTopWindow = + w && + `document` in w && + (() => { + try { + return w === w.top + } catch { + return true + } + })() + + if (isDev && isBrowserTopWindow) { expect((globalThis as any)[marker]).toBe(true) + } else { + // Either not dev mode or not a browser top window + expect((globalThis as any)[marker]).toBeUndefined() } - // Note: We can't easily test the production case without changing NODE_ENV - // which would affect the entire test suite }) it(`can be disabled with environment variable`, () => { diff --git a/packages/db/tests/utils.ts b/packages/db/tests/utils.ts index 8de71982a..2b5c5d0c8 100644 --- a/packages/db/tests/utils.ts +++ b/packages/db/tests/utils.ts @@ -1,10 +1,12 @@ import { expect } from 'vitest' +import { BTreeIndex } from '../src/indexes/btree-index' import type { CollectionConfig, MutationFnParams, StringCollationConfig, SyncConfig, } from '../src/index.js' +import type { IndexConstructor } from '../src/indexes/base-index' import type { WithVirtualProps } from '../src/virtual-props.js' export type OutputWithVirtual< @@ -214,6 +216,7 @@ type MockSyncCollectionConfig> = { sync?: SyncConfig syncMode?: `eager` | `on-demand` defaultStringCollation?: StringCollationConfig + defaultIndexType?: IndexConstructor } export function mockSyncCollectionOptions< @@ -295,6 +298,10 @@ export function mockSyncCollectionOptions< utils, ...config, autoIndex: config.autoIndex, + // When autoIndex is 'eager', we need a defaultIndexType + defaultIndexType: + config.defaultIndexType ?? + (config.autoIndex === `eager` ? BTreeIndex : undefined), } return options @@ -304,6 +311,7 @@ type MockSyncCollectionConfigNoInitialState = { id: string getKey: (item: T) => string | number autoIndex?: `off` | `eager` + defaultIndexType?: IndexConstructor } export function mockSyncCollectionOptionsNoInitialState< @@ -377,6 +385,10 @@ export function mockSyncCollectionOptionsNoInitialState< utils, ...config, autoIndex: config.autoIndex, + // When autoIndex is 'eager', we need a defaultIndexType + defaultIndexType: + config.defaultIndexType ?? + (config.autoIndex === `eager` ? BTreeIndex : undefined), } return options diff --git a/packages/db/vite.config.ts b/packages/db/vite.config.ts index 62e1a9df2..a56d9ca7e 100644 --- a/packages/db/vite.config.ts +++ b/packages/db/vite.config.ts @@ -16,7 +16,7 @@ const config = defineConfig({ export default mergeConfig( config, tanstackViteConfig({ - entry: `./src/index.ts`, + entry: [`./src/index.ts`], srcDir: `./src`, }), ) diff --git a/packages/electric-db-collection/e2e/electric.e2e.test.ts b/packages/electric-db-collection/e2e/electric.e2e.test.ts index cb41e26da..3e36ece78 100644 --- a/packages/electric-db-collection/e2e/electric.e2e.test.ts +++ b/packages/electric-db-collection/e2e/electric.e2e.test.ts @@ -5,7 +5,7 @@ */ import { afterAll, afterEach, beforeAll, describe, inject } from 'vitest' -import { createCollection } from '@tanstack/db' +import { createCollection, BasicIndex } from '@tanstack/db' import { ELECTRIC_TEST_HOOKS, electricCollectionOptions } from '../src/electric' import { makePgClient } from '../../db-collection-e2e/support/global-setup' import { @@ -290,6 +290,8 @@ describe(`Electric Collection E2E Tests`, () => { syncMode: `on-demand`, getKey: (item: any) => item.id, startSync: true, + autoIndex: `eager`, + defaultIndexType: BasicIndex, }), ) @@ -305,6 +307,8 @@ describe(`Electric Collection E2E Tests`, () => { syncMode: `on-demand`, getKey: (item: any) => item.id, startSync: true, + autoIndex: `eager`, + defaultIndexType: BasicIndex, }), ) diff --git a/packages/electric-db-collection/tests/electric-live-query.test.ts b/packages/electric-db-collection/tests/electric-live-query.test.ts index 2606bbfaf..8bd5ac7b8 100644 --- a/packages/electric-db-collection/tests/electric-live-query.test.ts +++ b/packages/electric-db-collection/tests/electric-live-query.test.ts @@ -5,6 +5,7 @@ import { eq, gt, lt, + BasicIndex, } from '@tanstack/db' import { electricCollectionOptions } from '../src/electric' import type { ElectricCollectionUtils } from '../src/electric' @@ -145,6 +146,7 @@ describe.each([ return createCollection({ ...options, startSync: true, + ...(autoIndex === `eager` ? { defaultIndexType: BasicIndex } : {}), }) as unknown as Collection< User, string | number, @@ -352,6 +354,7 @@ describe.each([ }), autoIndex, startSync: true, + ...(autoIndex === `eager` ? { defaultIndexType: BasicIndex } : {}), }) // Send initial data but don't complete sync (no up-to-date) @@ -494,8 +497,8 @@ describe.each([ return () => {} }) - const testElectricCollection = createCollection( - electricCollectionOptions({ + const testElectricCollection = createCollection({ + ...electricCollectionOptions({ id: `test-incremental-loading`, shapeOptions: { url: `http://test-url`, @@ -506,7 +509,8 @@ describe.each([ startSync: true, autoIndex: `eager` as const, }), - ) + defaultIndexType: BasicIndex, + }) mockRequestSnapshot.mockResolvedValue({ data: [], @@ -675,6 +679,7 @@ describe(`Electric Collection with Live Query - syncMode integration`, () => { ...options, startSync: true, autoIndex: `eager` as const, + defaultIndexType: BasicIndex, }) } @@ -982,6 +987,7 @@ describe(`Electric Collection - loadSubset deduplication`, () => { ...options, startSync: true, autoIndex: `eager` as const, + defaultIndexType: BasicIndex, }) } diff --git a/packages/query-db-collection/e2e/offline-refresh.e2e.test.ts b/packages/query-db-collection/e2e/offline-refresh.e2e.test.ts index db07f7048..ae25dd85f 100644 --- a/packages/query-db-collection/e2e/offline-refresh.e2e.test.ts +++ b/packages/query-db-collection/e2e/offline-refresh.e2e.test.ts @@ -6,7 +6,7 @@ */ import { describe, expect, it, vi } from 'vitest' -import { createCollection } from '@tanstack/db' +import { createCollection, BTreeIndex } from '@tanstack/db' import { QueryClient } from '@tanstack/query-core' import { startOfflineExecutor } from '@tanstack/offline-transactions' import { queryCollectionOptions } from '../src/query' @@ -186,6 +186,8 @@ describe(`offline transactions + query collection refresh`, () => { queryFn, getKey: (item: TestItem) => item.id, startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, }), ) diff --git a/packages/query-db-collection/e2e/query.e2e.test.ts b/packages/query-db-collection/e2e/query.e2e.test.ts index 1bda6ce26..00863707d 100644 --- a/packages/query-db-collection/e2e/query.e2e.test.ts +++ b/packages/query-db-collection/e2e/query.e2e.test.ts @@ -5,7 +5,7 @@ */ import { afterAll, afterEach, beforeAll, describe } from 'vitest' -import { createCollection } from '@tanstack/db' +import { createCollection, BTreeIndex } from '@tanstack/db' import { QueryClient } from '@tanstack/query-core' import { queryCollectionOptions } from '../src/query' import { @@ -55,6 +55,8 @@ describe(`Query Collection E2E Tests`, () => { }, getKey: (item: E2EUser) => item.id, startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, }), ) @@ -68,6 +70,8 @@ describe(`Query Collection E2E Tests`, () => { }, getKey: (item: E2EPost) => item.id, startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, }), ) @@ -81,6 +85,8 @@ describe(`Query Collection E2E Tests`, () => { }, getKey: (item: E2EComment) => item.id, startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, }), ) @@ -99,6 +105,8 @@ describe(`Query Collection E2E Tests`, () => { }, getKey: (item: E2EUser) => item.id, startSync: false, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, }), ) @@ -115,6 +123,8 @@ describe(`Query Collection E2E Tests`, () => { }, getKey: (item: E2EPost) => item.id, startSync: false, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, }), ) @@ -131,6 +141,8 @@ describe(`Query Collection E2E Tests`, () => { }, getKey: (item: E2EComment) => item.id, startSync: false, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, }), ) diff --git a/packages/query-db-collection/tests/query.test.ts b/packages/query-db-collection/tests/query.test.ts index 32c552c5d..5d2f9dbc1 100644 --- a/packages/query-db-collection/tests/query.test.ts +++ b/packages/query-db-collection/tests/query.test.ts @@ -7,6 +7,7 @@ import { ilike, inArray, or, + BTreeIndex, } from '@tanstack/db' import { stripVirtualProps } from '../../db/tests/utils' import { persistedCollectionOptions } from '../../db-sqlite-persisted-collection-core/src' @@ -4240,6 +4241,8 @@ describe(`QueryCollection`, () => { getKey, startSync: true, syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, } const options = queryCollectionOptions(config) diff --git a/packages/react-db/tests/useLiveInfiniteQuery.test.tsx b/packages/react-db/tests/useLiveInfiniteQuery.test.tsx index d020d79d1..9aa63244e 100644 --- a/packages/react-db/tests/useLiveInfiniteQuery.test.tsx +++ b/packages/react-db/tests/useLiveInfiniteQuery.test.tsx @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { act, renderHook, waitFor } from '@testing-library/react' import { createCollection, createLiveQueryCollection, eq } from '@tanstack/db' +import { BTreeIndex } from '@tanstack/db' import { useLiveInfiniteQuery } from '../src/useLiveInfiniteQuery' import { mockSyncCollectionOptions } from '../../db/tests/utils' import { createFilterFunctionFromExpression } from '../../db/src/collection/change-events' @@ -49,7 +50,8 @@ function createOnDemandCollection(opts: OnDemandCollectionOptions) { getKey: (post: Post) => post.id, syncMode: `on-demand`, startSync: true, - ...(autoIndex ? { autoIndex } : {}), + autoIndex: autoIndex ?? `eager`, + defaultIndexType: BTreeIndex, sync: { sync: ({ markReady, begin, write, commit }) => { markReady() @@ -106,6 +108,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(50) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `initial-page-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -156,6 +159,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(50) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `multiple-pages-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -215,6 +219,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(25) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `no-more-pages-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -274,6 +279,7 @@ describe(`useLiveInfiniteQuery`, () => { it(`should handle empty results`, async () => { const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `empty-results-test`, getKey: (post: Post) => post.id, initialData: [], @@ -309,6 +315,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(30) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `live-updates-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -380,6 +387,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(25) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `deletions-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -446,6 +454,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(5) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `partial-page-deletion-desc-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -513,6 +522,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(5) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `partial-page-deletion-asc-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -576,6 +586,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(50) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `where-clause-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -629,6 +640,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(50) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `deps-change-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -687,6 +699,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(30) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `page-params-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -737,6 +750,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(20) // Exactly 2 pages const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `exact-boundary-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -793,6 +807,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(50) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `concurrent-fetch-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -854,6 +869,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(5) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `no-fetch-when-done-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -896,6 +912,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(30) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `initial-param-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -937,6 +954,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(20) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `sync-detection-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -1022,6 +1040,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(50) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `immediate-data-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -1284,6 +1303,8 @@ describe(`useLiveInfiniteQuery`, () => { getKey: (post: Post) => post.id, syncMode: `on-demand`, startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, sync: { sync: ({ markReady, begin, write, commit }) => { // Provide initial data by slicing the first 15 elements @@ -1436,6 +1457,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(50) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `pre-created-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -1481,6 +1503,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(50) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `pre-created-multi-page-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -1532,6 +1555,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts1 = createMockPosts(30) const collection1 = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `pre-created-reset-1`, getKey: (post: Post) => post.id, initialData: posts1, @@ -1552,6 +1576,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts2 = createMockPosts(40) const collection2 = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `pre-created-reset-2`, getKey: (post: Post) => post.id, initialData: posts2, @@ -1612,6 +1637,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(50) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `no-orderby-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -1674,6 +1700,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(50) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `mismatched-window-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -1715,6 +1742,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(30) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `pre-created-live-updates-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -1788,6 +1816,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(50) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `router-loader-test`, getKey: (post: Post) => post.id, initialData: posts, @@ -1842,6 +1871,7 @@ describe(`useLiveInfiniteQuery`, () => { const posts = createMockPosts(10) const collection = createCollection( mockSyncCollectionOptions({ + autoIndex: `eager`, id: `circular-deps-test`, getKey: (post: Post) => post.id, initialData: posts, diff --git a/packages/trailbase-db-collection/e2e/trailbase.e2e.test.ts b/packages/trailbase-db-collection/e2e/trailbase.e2e.test.ts index 5a9e272a4..47cdf0cea 100644 --- a/packages/trailbase-db-collection/e2e/trailbase.e2e.test.ts +++ b/packages/trailbase-db-collection/e2e/trailbase.e2e.test.ts @@ -6,7 +6,7 @@ */ import { describe, expect, inject } from 'vitest' -import { createCollection } from '@tanstack/db' +import { createCollection, BTreeIndex } from '@tanstack/db' import { initClient } from 'trailbase' import { trailBaseCollectionOptions } from '../src/trailbase' import { @@ -171,6 +171,8 @@ function createCollectionsForSyncMode( getKey: (item: User) => item.id, startSync: true, syncMode, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, parse: { id: parseTrailBaseId, isActive: (isActive) => Boolean(isActive), @@ -195,6 +197,8 @@ function createCollectionsForSyncMode( getKey: (item: Post) => item.id, startSync: true, syncMode, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, parse: { id: parseTrailBaseId, largeViewCount: (l) => BigInt(l), @@ -217,6 +221,8 @@ function createCollectionsForSyncMode( getKey: (item: Comment) => item.id, startSync: true, syncMode, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, parse: { id: parseTrailBaseId, createdAt: (v) => new Date(v),