Building offline-capable web applications
Jack Arens

Data drift: stale data from failed propagations
| Approach | Offline | Complexity | Scale |
|---|---|---|---|
| Direct DB Calls | ❌ | Low | Poor |
| Propagation | ❌ | Medium | Poor |
| On-Demand Reducer | ❌ | High | Variable |
| Local-first | ✅ | Medium | Excellent |

“What if all the data was just always loaded?”

Local-first needs a sync engine to handle data flow

Emerging options (ElectricSQL, PowerSync)
Released after we started implementation - not designed for NoSQL
function pull(cookie): // Get changes since client's last sync point changes = getChangesSince(cookie)
// Build patches from changes patches = buildPatches(changes)
newCookie = getCurrentVersionCookie()
// Return patches and new sync point return { patches, cookie: newCookie }function push(clientId, mutations): for each mutation in mutations: // Skip if already processed (idempotency) if alreadyProcessed(clientId, mutation.id): if !hasPermission(clientId, mutation): continue
// Apply mutation to server state applyMutation(mutation) recordMutationProcessed(clientId, mutation.id)
return { success: true }Client: Replicache service wraps all data access
rcVersion) rcVersion) Updates applied in a bulk transaction on the server, which adds versioning

Problem: 40k+ element projects = ~150 - 200MB, slow initial load
Pull is called repeatedly until it returns no patches (complete)
function handlePull(cookie): changesSinceCookie = getChangesSince(cookie)
if cookie.fromVersion === 0: // Fresh client - serve a pre-built bundle instead bundleUrl = getLatestBundleUrl() return { bundleUrl, cookie: bundleCookie } else: // Few changes - return incremental patches patches = buildPatches(changesSinceCookie) return { patches, cookie: currentCookie }function pull(): lastCookie = getStoredCookie()
response = POST /pull { cookie: lastCookie }
if response.hasBundleUrl: // Bundle exists in storage - load full snapshot bundleData = downloadFromStorage(response.bundleUrl) applyPatches(bundleData.patches) storeCookie(bundleData.cookie)
// Pull again for changes since bundle was created pull() else: // No bundle - apply incremental patches applyPatches(response.patches) storeCookie(response.cookie)Problem: Loading all data into memory for sort is slow
export interface LayerElement { autoGenerateName?: LayerAutoGenerateNameOptions; autoIncrementId?: number; category: ContextItem<R>; completed: boolean; createdAt: number | T; createdBy: R | string; createdByRef: { email: string; id: string; name: string }; createdPhaseId?: string; deletedAt?: number | T; family: string; fields: Record<string, any | LayerElementField<T, R>>; id?: string; modelRevitId: string; name: string; params: Record<string, LayerRevitParameterValue<T, R>>; rcVersion?: number; references?: { [key: string]: LayerReference<T, R> }; revitExternalId?: string; revitId: null | string; revitKind?: 'Forge' | 'Instance' | 'Type'; searchableIndex: string[]; skipInitialization?: boolean; spatialRelationships?: LayerForgeInstanceElement['spatialRelationships']; starred: boolean; status: 'active' | 'archived'; templateName?: string; type?: string; typeId?: string; updatedAt: T; updatedBy: R; updatedByRef: { email: string; id: string; name: string }; versionHistory?: LayerForgeInstanceElement['versionHistory']; viewables?: LayerForgeViewable[];}export interface LokiDBElement { _sortName: string; /** * Dynamic fields stored as key-value pairs * * Keys (IDs) represent one the following: * - A regular layer field ID (unique) * - A Revit parameter ID * - A spatial relationship ID + a categoryID (ex. 'spatialRelationships.<categoryId>') */ [id: string]: unknown; autoIncrementId: null | number; categoryId: string; completed: boolean; createdAt: number; createdBy: string; createdPhaseId?: string; id: string; modelRevitId?: string; name: string; references: string[]; spatialRelationships: string[]; starred: boolean; status: string; type?: string; updatedAt: number; updatedBy: string;}Problem: Backend calls to update data can fail if made in bulk due to contention
Client doesn’t run into this issue - can retry at will/long lived sessions
Slides & Writeup @ jackarens.com