chore: improvements to caching strategies (#5488)

frontend/appflowy_web_app/src/application/database-yjs/selector.ts

@@ -71,7 +71,6 @@ export function useDatabaseViewsSelector(iidIndex: string) {
         const [key] = viewItem;
         const view = folderViews?.get(key);
 
-        console.log('view', view?.get(YjsFolderKey.bid), iidIndex);
         if (
           visibleViewsId.includes(key) &&
           view &&
@@ -81,7 +80,6 @@ export function useDatabaseViewsSelector(iidIndex: string) {
         }
       }
 
-      console.log('viewsId', viewsId);
       setViewIds(viewsId);
     };
 

frontend/appflowy_web_app/src/application/services/js-services/__tests__/cache.test.ts

@@ -0,0 +1,310 @@
+import { CollabType } from '@/application/collab.type';
+import * as Y from 'yjs';
+import { withTestingYDoc } from '@/application/slate-yjs/__tests__/withTestingYjsEditor';
+import { expect } from '@jest/globals';
+import { getCollab, batchCollab, collabTypeToDBType } from '../cache';
+import { applyYDoc } from '@/application/ydoc/apply';
+import { getCollabDBName, openCollabDB } from '../cache/db';
+import { StrategyType } from '../cache/types';
+
+jest.mock('@/application/ydoc/apply', () => ({
+  applyYDoc: jest.fn(),
+}));
+jest.mock('../cache/db', () => ({
+  openCollabDB: jest.fn(),
+  getCollabDBName: jest.fn(),
+}));
+
+const emptyDoc = new Y.Doc();
+const normalDoc = withTestingYDoc('1');
+const mockFetcher = jest.fn();
+const mockBatchFetcher = jest.fn();
+
+describe('Cache functions', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('getCollab', () => {
+    describe('with CACHE_ONLY strategy', () => {
+      it('should throw error when no cache', async () => {
+        (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
+        (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+        await expect(
+          getCollab(
+            mockFetcher,
+            {
+              collabId: 'id1',
+              collabType: CollabType.Document,
+            },
+            StrategyType.CACHE_ONLY
+          )
+        ).rejects.toThrow('No cache found');
+      });
+      it('should fetch collab with CACHE_ONLY strategy and existing cache', async () => {
+        (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
+        (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+        const result = await getCollab(
+          mockFetcher,
+          {
+            collabId: 'id1',
+            collabType: CollabType.Document,
+          },
+          StrategyType.CACHE_ONLY
+        );
+
+        expect(result).toBe(normalDoc);
+        expect(mockFetcher).not.toHaveBeenCalled();
+        expect(applyYDoc).not.toHaveBeenCalled();
+      });
+    });
+
+    describe('with CACHE_FIRST strategy', () => {
+      it('should fetch collab with CACHE_FIRST strategy and existing cache', async () => {
+        (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
+        (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+        mockFetcher.mockResolvedValue({ state: new Uint8Array() });
+
+        const result = await getCollab(
+          mockFetcher,
+          {
+            collabId: 'id1',
+            collabType: CollabType.Document,
+          },
+          StrategyType.CACHE_FIRST
+        );
+
+        expect(result).toBe(normalDoc);
+        expect(mockFetcher).not.toHaveBeenCalled();
+        expect(applyYDoc).not.toHaveBeenCalled();
+      });
+
+      it('should fetch collab with CACHE_FIRST strategy and no cache', async () => {
+        (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
+        (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+        mockFetcher.mockResolvedValue({ state: new Uint8Array() });
+
+        const result = await getCollab(
+          mockFetcher,
+          {
+            collabId: 'id1',
+            collabType: CollabType.Document,
+          },
+          StrategyType.CACHE_FIRST
+        );
+
+        expect(result).toBe(emptyDoc);
+        expect(mockFetcher).toHaveBeenCalled();
+        expect(applyYDoc).toHaveBeenCalled();
+      });
+    });
+
+    describe('with CACHE_AND_NETWORK strategy', () => {
+      it('should fetch collab with CACHE_AND_NETWORK strategy and existing cache', async () => {
+        (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
+        (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+        mockFetcher.mockResolvedValue({ state: new Uint8Array() });
+
+        const result = await getCollab(
+          mockFetcher,
+          {
+            collabId: 'id1',
+            collabType: CollabType.Document,
+          },
+          StrategyType.CACHE_AND_NETWORK
+        );
+
+        expect(result).toBe(normalDoc);
+        expect(mockFetcher).toHaveBeenCalled();
+        expect(applyYDoc).toHaveBeenCalled();
+      });
+
+      it('should fetch collab with CACHE_AND_NETWORK strategy and no cache', async () => {
+        (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
+        (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+        mockFetcher.mockResolvedValue({ state: new Uint8Array() });
+
+        const result = await getCollab(
+          mockFetcher,
+          {
+            collabId: 'id1',
+            collabType: CollabType.Document,
+          },
+          StrategyType.CACHE_AND_NETWORK
+        );
+
+        expect(result).toBe(emptyDoc);
+        expect(mockFetcher).toHaveBeenCalled();
+        expect(applyYDoc).toHaveBeenCalled();
+      });
+    });
+
+    describe('with default strategy', () => {
+      it('should fetch collab with default strategy', async () => {
+        (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
+        (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+        mockFetcher.mockResolvedValue({ state: new Uint8Array() });
+
+        const result = await getCollab(
+          mockFetcher,
+          {
+            collabId: 'id1',
+            collabType: CollabType.Document,
+          },
+          StrategyType.NETWORK_ONLY
+        );
+
+        expect(result).toBe(normalDoc);
+        expect(mockFetcher).toHaveBeenCalled();
+        expect(applyYDoc).toHaveBeenCalled();
+      });
+    });
+  });
+
+  describe('batchCollab', () => {
+    describe('with CACHE_ONLY strategy', () => {
+      it('should batch fetch collabs with CACHE_ONLY strategy and no cache', async () => {
+        (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
+
+        (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+        await expect(
+          batchCollab(
+            mockBatchFetcher,
+            [
+              {
+                collabId: 'id1',
+                collabType: CollabType.Document,
+              },
+            ],
+            StrategyType.CACHE_ONLY
+          )
+        ).rejects.toThrow('No cache found');
+      });
+
+      it('should batch fetch collabs with CACHE_ONLY strategy and existing cache', async () => {
+        (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
+
+        (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+        await batchCollab(
+          mockBatchFetcher,
+          [
+            {
+              collabId: 'id1',
+              collabType: CollabType.Document,
+            },
+          ],
+          StrategyType.CACHE_ONLY
+        );
+
+        expect(mockBatchFetcher).not.toHaveBeenCalled();
+      });
+    });
+
+    describe('with CACHE_FIRST strategy', () => {
+      it('should batch fetch collabs with CACHE_FIRST strategy and existing cache', async () => {
+        (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
+
+        (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+        await batchCollab(
+          mockBatchFetcher,
+          [
+            {
+              collabId: 'id1',
+              collabType: CollabType.Document,
+            },
+          ],
+          StrategyType.CACHE_FIRST
+        );
+
+        expect(mockBatchFetcher).not.toHaveBeenCalled();
+      });
+
+      it('should batch fetch collabs with CACHE_FIRST strategy and no cache', async () => {
+        (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
+
+        (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+        mockBatchFetcher.mockResolvedValue({ id1: [1, 2, 3] });
+
+        await batchCollab(
+          mockBatchFetcher,
+          [
+            {
+              collabId: 'id1',
+              collabType: CollabType.Document,
+            },
+          ],
+          StrategyType.CACHE_FIRST
+        );
+
+        expect(mockBatchFetcher).toHaveBeenCalled();
+        expect(applyYDoc).toHaveBeenCalled();
+      });
+    });
+
+    describe('with CACHE_AND_NETWORK strategy', () => {
+      it('should batch fetch collabs with CACHE_AND_NETWORK strategy', async () => {
+        (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
+
+        (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+        mockBatchFetcher.mockResolvedValue({ id1: [1, 2, 3] });
+
+        await batchCollab(
+          mockBatchFetcher,
+          [
+            {
+              collabId: 'id1',
+              collabType: CollabType.Document,
+            },
+          ],
+          StrategyType.CACHE_AND_NETWORK
+        );
+
+        expect(mockBatchFetcher).toHaveBeenCalled();
+        expect(applyYDoc).toHaveBeenCalled();
+      });
+
+      it('should batch fetch collabs with CACHE_AND_NETWORK strategy and no cache', async () => {
+        (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
+
+        (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+        mockBatchFetcher.mockResolvedValue({ id1: [1, 2, 3] });
+
+        await batchCollab(
+          mockBatchFetcher,
+          [
+            {
+              collabId: 'id1',
+              collabType: CollabType.Document,
+            },
+          ],
+          StrategyType.CACHE_AND_NETWORK
+        );
+
+        expect(mockBatchFetcher).toHaveBeenCalled();
+        expect(applyYDoc).toHaveBeenCalled();
+      });
+    });
+  });
+});
+
+describe('collabTypeToDBType', () => {
+  it('should return correct DB type', () => {
+    expect(collabTypeToDBType(CollabType.Document)).toBe('document');
+    expect(collabTypeToDBType(CollabType.Folder)).toBe('folder');
+    expect(collabTypeToDBType(CollabType.Database)).toBe('database');
+    expect(collabTypeToDBType(CollabType.WorkspaceDatabase)).toBe('databases');
+    expect(collabTypeToDBType(CollabType.DatabaseRow)).toBe('database_row');
+    expect(collabTypeToDBType(CollabType.UserAwareness)).toBe('user_awareness');
+    expect(collabTypeToDBType(CollabType.Empty)).toBe('');
+  });
+});

frontend/appflowy_web_app/src/application/services/js-services/__tests__/fetch.test.ts

@@ -0,0 +1,57 @@
+import { expect } from '@jest/globals';
+import { fetchCollab, batchFetchCollab } from '../fetch';
+import { CollabType } from '@/application/collab.type';
+import { APIService } from '@/application/services/js-services/wasm';
+
+jest.mock('@/application/services/js-services/wasm', () => {
+  return {
+    APIService: {
+      getCollab: jest.fn(),
+      batchGetCollab: jest.fn(),
+    },
+  };
+});
+
+describe('Collab fetch functions with deduplication', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('fetchCollab', () => {
+    it('should fetch collab without duplicating requests', async () => {
+      const workspaceId = 'workspace1';
+      const id = 'id1';
+      const type = CollabType.Document;
+      const mockResponse = { data: 'mockData' };
+
+      (APIService.getCollab as jest.Mock).mockResolvedValue(mockResponse);
+
+      const result1 = fetchCollab(workspaceId, id, type);
+      const result2 = fetchCollab(workspaceId, id, type);
+
+      expect(result1).toBe(result2);
+      await expect(result1).resolves.toEqual(mockResponse);
+      expect(APIService.getCollab).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('batchFetchCollab', () => {
+    it('should batch fetch collabs without duplicating requests', async () => {
+      const workspaceId = 'workspace1';
+      const params = [
+        { collabId: 'id1', collabType: CollabType.Document },
+        { collabId: 'id2', collabType: CollabType.Folder },
+      ];
+      const mockResponse = { data: 'mockData' };
+
+      (APIService.batchGetCollab as jest.Mock).mockResolvedValue(mockResponse);
+
+      const result1 = batchFetchCollab(workspaceId, params);
+      const result2 = batchFetchCollab(workspaceId, params);
+
+      expect(result1).toBe(result2);
+      await expect(result1).resolves.toEqual(mockResponse);
+      expect(APIService.batchGetCollab).toHaveBeenCalledTimes(1);
+    });
+  });
+});

frontend/appflowy_web_app/src/application/services/js-services/auth.service.ts

@@ -1,8 +1,8 @@
 import { AuthService } from '@/application/services/services.type';
 import { ProviderType, SignUpWithEmailPasswordParams } from '@/application/user.type';
 import { APIService } from 'src/application/services/js-services/wasm';
-import { signInSuccess } from '@/application/services/js-services/storage/auth';
+import { signInSuccess } from '@/application/services/js-services/session/auth';
-import { invalidToken } from '@/application/services/js-services/storage';
+import { invalidToken } from 'src/application/services/js-services/session';
 import { afterSignInDecorator } from '@/application/services/js-services/decorator';
 
 export class JSAuthService implements AuthService {

frontend/appflowy_web_app/src/application/services/js-services/cache/db.ts

@@ -1,9 +1,10 @@
 import { YDoc } from '@/application/collab.type';
 import { databasePrefix } from '@/application/constants';
-import { getAuthInfo } from '@/application/services/js-services/storage';
 import { IndexeddbPersistence } from 'y-indexeddb';
 import * as Y from 'yjs';
 
+const openedSet = new Set<string>();
+
 /**
  * Open the collaboration database, and return a function to close it
  */
@@ -19,6 +20,10 @@ export async function openCollabDB(docName: string): Promise<YDoc> {
   });
 
   provider.on('synced', () => {
+    if (!openedSet.has(name)) {
+      openedSet.add(name);
+    }
+
     resolve(true);
   });
 
@@ -27,9 +32,10 @@ export async function openCollabDB(docName: string): Promise<YDoc> {
   return doc as YDoc;
 }
 
-export function getDBName(id: string, type: string) {
+export function getCollabDBName(id: string, type: string, uuid?: string) {
-  const { uuid } = getAuthInfo() || {};
+  if (!uuid) {
+    return `${type}_${id}`;
+  }
 
-  if (!uuid) throw new Error('No user found');
   return `${uuid}_${type}_${id}`;
 }

frontend/appflowy_web_app/src/application/services/js-services/cache/index.ts

@@ -0,0 +1,165 @@
+import { CollabType, YDoc, YjsEditorKey, YSharedRoot } from '@/application/collab.type';
+import { applyYDoc } from '@/application/ydoc/apply';
+import { getCollabDBName, openCollabDB } from './db';
+import { Fetcher, StrategyType } from './types';
+
+export function collabTypeToDBType(type: CollabType) {
+  switch (type) {
+    case CollabType.Folder:
+      return 'folder';
+    case CollabType.Document:
+      return 'document';
+    case CollabType.Database:
+      return 'database';
+    case CollabType.WorkspaceDatabase:
+      return 'databases';
+    case CollabType.DatabaseRow:
+      return 'database_row';
+    case CollabType.UserAwareness:
+      return 'user_awareness';
+    default:
+      return '';
+  }
+}
+
+const collabSharedRootKeyMap = {
+  [CollabType.Folder]: YjsEditorKey.folder,
+  [CollabType.Document]: YjsEditorKey.document,
+  [CollabType.Database]: YjsEditorKey.database,
+  [CollabType.WorkspaceDatabase]: YjsEditorKey.workspace_database,
+  [CollabType.DatabaseRow]: YjsEditorKey.database_row,
+  [CollabType.UserAwareness]: YjsEditorKey.user_awareness,
+  [CollabType.Empty]: YjsEditorKey.empty,
+};
+
+export function hasCache(doc: YDoc, type: CollabType) {
+  const data = doc.getMap(YjsEditorKey.data_section) as YSharedRoot;
+
+  return data.has(collabSharedRootKeyMap[type] as string);
+}
+
+export async function getCollab(
+  fetcher: Fetcher<{
+    state: Uint8Array;
+  }>,
+  {
+    collabId,
+    collabType,
+    uuid,
+  }: {
+    uuid?: string;
+    collabId: string;
+    collabType: CollabType;
+  },
+  strategy: StrategyType = StrategyType.CACHE_AND_NETWORK
+) {
+  const name = getCollabDBName(collabId, collabTypeToDBType(collabType), uuid);
+  const collab = await openCollabDB(name);
+  const exist = hasCache(collab, collabType);
+
+  switch (strategy) {
+    case StrategyType.CACHE_ONLY: {
+      if (!exist) {
+        throw new Error('No cache found');
+      }
+
+      return collab;
+    }
+
+    case StrategyType.CACHE_FIRST: {
+      if (!exist) {
+        await revalidateCollab(fetcher, collab);
+      }
+
+      return collab;
+    }
+
+    case StrategyType.CACHE_AND_NETWORK: {
+      if (!exist) {
+        await revalidateCollab(fetcher, collab);
+      } else {
+        void revalidateCollab(fetcher, collab);
+      }
+
+      return collab;
+    }
+
+    default: {
+      await revalidateCollab(fetcher, collab);
+
+      return collab;
+    }
+  }
+}
+
+async function revalidateCollab(
+  fetcher: Fetcher<{
+    state: Uint8Array;
+  }>,
+  collab: YDoc
+) {
+  const { state } = await fetcher();
+
+  applyYDoc(collab, state);
+}
+
+export async function batchCollab(
+  batchFetcher: Fetcher<Record<string, number[]>>,
+  collabs: {
+    collabId: string;
+    collabType: CollabType;
+    uuid?: string;
+  }[],
+  strategy: StrategyType = StrategyType.CACHE_AND_NETWORK,
+  itemCallback?: (id: string, doc: YDoc) => void
+) {
+  const collabMap = new Map<string, YDoc>();
+
+  for (const { collabId, collabType, uuid } of collabs) {
+    const name = getCollabDBName(collabId, collabTypeToDBType(collabType), uuid);
+    const collab = await openCollabDB(name);
+    const exist = hasCache(collab, collabType);
+
+    collabMap.set(collabId, collab);
+    if (exist) {
+      itemCallback?.(collabId, collab);
+    }
+  }
+
+  const notCacheIds = collabs.filter(({ collabId, collabType }) => {
+    const id = collabMap.get(collabId);
+
+    if (!id) return false;
+
+    return !hasCache(id, collabType);
+  });
+
+  if (strategy === StrategyType.CACHE_ONLY) {
+    if (notCacheIds.length > 0) {
+      throw new Error('No cache found');
+    }
+
+    return;
+  }
+
+  if (strategy === StrategyType.CACHE_FIRST && notCacheIds.length === 0) {
+    return;
+  }
+
+  const states = await batchFetcher();
+
+  for (const [collabId, data] of Object.entries(states)) {
+    const info = collabs.find((item) => item.collabId === collabId);
+    const collab = collabMap.get(collabId);
+
+    if (!info || !collab) {
+      continue;
+    }
+
+    const state = new Uint8Array(data);
+
+    applyYDoc(collab, state);
+
+    itemCallback?.(collabId, collab);
+  }
+}

frontend/appflowy_web_app/src/application/services/js-services/cache/types.ts

@@ -0,0 +1,12 @@
+export enum StrategyType {
+  // Cache only: return the cache if it exists, otherwise throw an error
+  CACHE_ONLY = 'CACHE_ONLY',
+  // Cache first: return the cache if it exists, otherwise fetch from the network
+  CACHE_FIRST = 'CACHE_FIRST',
+  // Cache and network: return the cache if it exists, otherwise fetch from the network and update the cache
+  CACHE_AND_NETWORK = 'CACHE_AND_NETWORK',
+  // Network only: fetch from the network and update the cache
+  NETWORK_ONLY = 'NETWORK_ONLY',
+}
+
+export type Fetcher<T> = () => Promise<T>;

frontend/appflowy_web_app/src/application/services/js-services/database.service.ts

@@ -1,16 +1,16 @@
 import { CollabType, YDatabase, YDoc, YjsDatabaseKey, YjsEditorKey } from '@/application/collab.type';
-import {
+import { batchCollab, getCollab } from '@/application/services/js-services/cache';
-  batchCollabs,
+import { StrategyType } from '@/application/services/js-services/cache/types';
-  getCollabStorage,
+import { batchFetchCollab, fetchCollab } from '@/application/services/js-services/fetch';
-  getCollabStorageWithAPICall,
+import { getCurrentWorkspace } from 'src/application/services/js-services/session';
-  getCurrentWorkspace,
-} from '@/application/services/js-services/storage';
 import { DatabaseService } from '@/application/services/services.type';
 import * as Y from 'yjs';
 
 export class JSDatabaseService implements DatabaseService {
   private loadedDatabaseId: Set<string> = new Set();
 
+  private loadedWorkspaceId: Set<string> = new Set();
+
   private cacheDatabaseRowDocMap: Map<string, Y.Doc> = new Map();
 
   constructor() {
@@ -28,22 +28,30 @@ export class JSDatabaseService implements DatabaseService {
       throw new Error('Workspace database not found');
     }
 
-    const workspaceDatabase = await getCollabStorageWithAPICall(
+    const isLoaded = this.loadedWorkspaceId.has(workspace.id);
-      workspace.id,
+
-      workspace.workspaceDatabaseId,
+    const workspaceDatabase = await getCollab(
-      CollabType.WorkspaceDatabase
+      () => {
+        return fetchCollab(workspace.id, workspace.workspaceDatabaseId, CollabType.WorkspaceDatabase);
+      },
+      {
+        collabId: workspace.workspaceDatabaseId,
+        collabType: CollabType.WorkspaceDatabase,
+      },
+      isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK
     );
 
+    if (!isLoaded) {
+      this.loadedWorkspaceId.add(workspace.id);
+    }
+
     return workspaceDatabase.getMap(YjsEditorKey.data_section).get(YjsEditorKey.workspace_database).toJSON() as {
       views: string[];
       database_id: string;
     }[];
   }
 
-  async openDatabase(
+  async openDatabase(databaseId: string): Promise<{
-    databaseId: string,
-    rowIds?: string[]
-  ): Promise<{
     databaseDoc: YDoc;
     rows: Y.Map<YDoc>;
   }> {
@@ -68,13 +76,18 @@ export class JSDatabaseService implements DatabaseService {
 
     const rowsFolder: Y.Map<YDoc> = rootRowsDoc.getMap();
 
-    let databaseDoc: YDoc | undefined = undefined;
+    const databaseDoc = await getCollab(
+      () => {
+        return fetchCollab(workspaceId, databaseId, CollabType.Database);
+      },
+      {
+        collabId: databaseId,
+        collabType: CollabType.Database,
+      },
+      isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK
+    );
 
-    if (isLoaded) {
+    if (!isLoaded) this.loadedDatabaseId.add(databaseId);
-      databaseDoc = (await getCollabStorage(databaseId, CollabType.Database)).doc;
-    } else {
-      databaseDoc = await getCollabStorageWithAPICall(workspaceId, databaseId, CollabType.Database);
-    }
 
     const database = databaseDoc.getMap(YjsEditorKey.data_section)?.get(YjsEditorKey.database) as YDatabase;
     const viewId = database.get(YjsDatabaseKey.metas)?.get(YjsDatabaseKey.iid)?.toString();
@@ -87,47 +100,50 @@ export class JSDatabaseService implements DatabaseService {
       throw new Error('Database rows not found');
     }
 
-    const ids = rowIds ? rowIds : rowOrdersIds.map((item) => item.id);
+    const rowsParams = rowOrdersIds.map((item) => ({
-
+      collabId: item.id,
-    if (isLoaded) {
+      collabType: CollabType.DatabaseRow,
-      for (const id of ids) {
+    }));
-        const { doc } = await getCollabStorage(id, CollabType.DatabaseRow);
 
+    void batchCollab(
+      () => {
+        return batchFetchCollab(workspaceId, rowsParams);
+      },
+      rowsParams,
+      isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK,
+      (id: string, doc: YDoc) => {
         if (!rowsFolder.has(id)) {
           rowsFolder.set(id, doc);
         }
       }
-    } else {
+    );
-      void this.loadDatabaseRows(workspaceId, ids, (id, row) => {
-        if (!rowsFolder.has(id)) {
-          rowsFolder.set(id, row);
-        }
-      });
-    }
 
-    this.loadedDatabaseId.add(databaseId);
+    // Update rows if there are new rows added after the database has been loaded
-
-    if (!rowIds) {
-      // Update rows if new rows are added
     rowOrders?.observe((event) => {
       if (event.changes.added.size > 0) {
         const rowIds = rowOrders.toJSON() as {
           id: string;
         }[];
 
-          console.log('Update rows', rowIds);
+        const params = rowIds.map((item) => ({
-          void this.loadDatabaseRows(
+          collabId: item.id,
-            workspaceId,
+          collabType: CollabType.DatabaseRow,
-            rowIds.map((item) => item.id),
+        }));
-            (rowId: string, rowDoc) => {
+
-              if (!rowsFolder.has(rowId)) {
+        void batchCollab(
-                rowsFolder.set(rowId, rowDoc);
+          () => {
+            return batchFetchCollab(workspaceId, params);
+          },
+          params,
+          StrategyType.CACHE_AND_NETWORK,
+          (id: string, doc: YDoc) => {
+            if (!rowsFolder.has(id)) {
+              rowsFolder.set(id, doc);
             }
           }
         );
       }
     });
-    }
 
     return {
       databaseDoc,
@@ -135,21 +151,6 @@ export class JSDatabaseService implements DatabaseService {
     };
   }
 
-  async loadDatabaseRows(workspaceId: string, rowIds: string[], rowCallback: (rowId: string, rowDoc: YDoc) => void) {
-    try {
-      await batchCollabs(
-        workspaceId,
-        rowIds.map((id) => ({
-          object_id: id,
-          collab_type: CollabType.DatabaseRow,
-        })),
-        rowCallback
-      );
-    } catch (e) {
-      console.error(e);
-    }
-  }
-
   async closeDatabase(databaseId: string) {
     this.cacheDatabaseRowDocMap.delete(databaseId);
   }

frontend/appflowy_web_app/src/application/services/js-services/document.service.ts

@@ -1,8 +1,13 @@
 import { CollabOrigin, CollabType, YDoc } from '@/application/collab.type';
-import { getCollabStorageWithAPICall, getCurrentWorkspace } from '@/application/services/js-services/storage';
+import { getCollab } from '@/application/services/js-services/cache';
+import { StrategyType } from '@/application/services/js-services/cache/types';
+import { fetchCollab } from '@/application/services/js-services/fetch';
+import { getCurrentWorkspace } from 'src/application/services/js-services/session';
 import { DocumentService } from '@/application/services/services.type';
 
 export class JSDocumentService implements DocumentService {
+  private loaded: Set<string> = new Set();
+
   constructor() {
     //
   }
@@ -14,8 +19,20 @@ export class JSDocumentService implements DocumentService {
       throw new Error('Workspace database not found');
     }
 
-    const doc = await getCollabStorageWithAPICall(workspace.id, docId, CollabType.Document);
+    const isLoaded = this.loaded.has(docId);
 
+    const doc = await getCollab(
+      () => {
+        return fetchCollab(workspace.id, docId, CollabType.Document);
+      },
+      {
+        collabId: docId,
+        collabType: CollabType.Document,
+      },
+      isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK
+    );
+
+    if (!isLoaded) this.loaded.add(docId);
     const handleUpdate = (update: Uint8Array, origin: CollabOrigin) => {
       if (origin === CollabOrigin.LocalSync) {
         // Send the update to the server

frontend/appflowy_web_app/src/application/services/js-services/fetch.ts

@@ -0,0 +1,66 @@
+import { CollabType } from '@/application/collab.type';
+import { APIService } from '@/application/services/js-services/wasm';
+
+const pendingRequests = new Map();
+
+function generateRequestKey<T>(url: string, params: T) {
+  if (!params) return url;
+
+  try {
+    return `${url}_${JSON.stringify(params)}`;
+  } catch (_e) {
+    return `${url}_${params}`;
+  }
+}
+
+// Deduplication fetch requests
+// When multiple requests are made to the same URL with the same params, only one request is made
+// and the result is shared with all the requests
+function fetchWithDeduplication<Req, Res>(url: string, params: Req, fetchFunction: () => Promise<Res>): Promise<Res> {
+  const requestKey = generateRequestKey<Req>(url, params);
+
+  if (pendingRequests.has(requestKey)) {
+    return pendingRequests.get(requestKey);
+  }
+
+  const fetchPromise = fetchFunction().finally(() => {
+    pendingRequests.delete(requestKey);
+  });
+
+  pendingRequests.set(requestKey, fetchPromise);
+  return fetchPromise;
+}
+
+/**
+ * Fetch collab
+ * @param workspaceId
+ * @param id
+ * @param type [CollabType]
+ */
+export function fetchCollab(workspaceId: string, id: string, type: CollabType) {
+  const fetchFunction = () => APIService.getCollab(workspaceId, id, type);
+
+  return fetchWithDeduplication(`fetchCollab_${workspaceId}`, { id, type }, fetchFunction);
+}
+
+/**
+ * Batch fetch collab
+ * Usage:
+ *   // load database rows
+ *   const rows = await batchFetchCollab(workspaceId, databaseRows.map((row) => ({ collabId: row.id, collabType: CollabType.DatabaseRow })));
+ *
+ * @param workspaceId
+ * @param params [{ collabId: string; collabType: CollabType }]
+ */
+export function batchFetchCollab(workspaceId: string, params: { collabId: string; collabType: CollabType }[]) {
+  const fetchFunction = () =>
+    APIService.batchGetCollab(
+      workspaceId,
+      params.map(({ collabId, collabType }) => ({
+        object_id: collabId,
+        collab_type: collabType,
+      }))
+    );
+
+  return fetchWithDeduplication(`batchFetchCollab_${workspaceId}`, params, fetchFunction);
+}

frontend/appflowy_web_app/src/application/services/js-services/folder.service.ts

@@ -1,14 +1,30 @@
 import { CollabOrigin, CollabType, YDoc } from '@/application/collab.type';
-import { getCollabStorageWithAPICall } from '@/application/services/js-services/storage';
+import { getCollab } from '@/application/services/js-services/cache';
+import { StrategyType } from '@/application/services/js-services/cache/types';
+import { fetchCollab } from '@/application/services/js-services/fetch';
 import { FolderService } from '@/application/services/services.type';
 
 export class JSFolderService implements FolderService {
+  private loaded: Set<string> = new Set();
+
   constructor() {
     //
   }
 
   async openWorkspace(workspaceId: string): Promise<YDoc> {
-    const doc = await getCollabStorageWithAPICall(workspaceId, workspaceId, CollabType.Folder);
+    const isLoaded = this.loaded.has(workspaceId);
+    const doc = await getCollab(
+      () => {
+        return fetchCollab(workspaceId, workspaceId, CollabType.Folder);
+      },
+      {
+        collabId: workspaceId,
+        collabType: CollabType.Folder,
+      },
+      isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK
+    );
+
+    if (!isLoaded) this.loaded.add(workspaceId);
     const handleUpdate = (update: Uint8Array, origin: CollabOrigin) => {
       if (origin === CollabOrigin.LocalSync) {
         // Send the update to the server

frontend/appflowy_web_app/src/application/services/js-services/session/index.ts

@@ -1,4 +1,3 @@
 export * from './token';
 export * from './user';
-export * from './collab';
 export * from './auth';

frontend/appflowy_web_app/src/application/services/js-services/storage/collab.ts

@@ -1,118 +0,0 @@
-import { CollabType, YDoc, YjsEditorKey, YSharedRoot } from '@/application/collab.type';
-import { getDBName, openCollabDB } from '@/application/services/js-services/db';
-import { APIService } from '@/application/services/js-services/wasm';
-import { applyYDoc } from '@/application/ydoc/apply';
-
-export function fetchCollab(workspaceId: string, id: string, type: CollabType) {
-  return APIService.getCollab(workspaceId, id, type);
-}
-
-export function batchFetchCollab(workspaceId: string, params: { object_id: string; collab_type: CollabType }[]) {
-  return APIService.batchGetCollab(workspaceId, params);
-}
-
-function collabTypeToDBType(type: CollabType) {
-  switch (type) {
-    case CollabType.Folder:
-      return 'folder';
-    case CollabType.Document:
-      return 'document';
-    case CollabType.Database:
-      return 'database';
-    case CollabType.WorkspaceDatabase:
-      return 'databases';
-    case CollabType.DatabaseRow:
-      return 'database_row';
-    case CollabType.UserAwareness:
-      return 'user_awareness';
-    default:
-      return '';
-  }
-}
-
-const collabSharedRootKeyMap = {
-  [CollabType.Folder]: YjsEditorKey.folder,
-  [CollabType.Document]: YjsEditorKey.document,
-  [CollabType.Database]: YjsEditorKey.database,
-  [CollabType.WorkspaceDatabase]: YjsEditorKey.workspace_database,
-  [CollabType.DatabaseRow]: YjsEditorKey.database_row,
-  [CollabType.UserAwareness]: YjsEditorKey.user_awareness,
-  [CollabType.Empty]: YjsEditorKey.empty,
-};
-
-export async function getCollabStorage(id: string, type: CollabType) {
-  const name = getDBName(id, collabTypeToDBType(type));
-
-  const doc = await openCollabDB(name);
-  let localExist = false;
-  const existData = doc.share.has(YjsEditorKey.data_section);
-
-  if (existData) {
-    const data = doc.getMap(YjsEditorKey.data_section) as YSharedRoot;
-
-    localExist = data.has(collabSharedRootKeyMap[type] as string);
-  }
-
-  return {
-    doc,
-    localExist,
-  };
-}
-
-export async function getCollabStorageWithAPICall(workspaceId: string, id: string, type: CollabType) {
-  const { doc, localExist } = await getCollabStorage(id, type);
-  const asyncApply = async () => {
-    const res = await fetchCollab(workspaceId, id, type);
-
-    applyYDoc(doc, res.state);
-  };
-
-  // If the document exists locally, apply the state asynchronously,
-  // otherwise, apply the state synchronously
-  if (localExist) {
-    void asyncApply();
-  } else {
-    await asyncApply();
-  }
-
-  return doc;
-}
-
-export async function batchCollabs(
-  workspaceId: string,
-  params: {
-    object_id: string;
-    collab_type: CollabType;
-  }[],
-  rowCallback?: (id: string, doc: YDoc) => void
-) {
-  console.log('Fetching collab data:', params);
-  // Create or get Y.Doc from local storage
-  for (const item of params) {
-    const { object_id, collab_type } = item;
-
-    const { doc, localExist } = await getCollabStorage(object_id, collab_type);
-
-    if (rowCallback && localExist) {
-      rowCallback(object_id, doc);
-    }
-  }
-
-  const res = await batchFetchCollab(workspaceId, params);
-
-  console.log('Fetched collab data:', res);
-  for (const id of Object.keys(res)) {
-    const type = params.find((param) => param.object_id === id)?.collab_type;
-    const data = res[id];
-
-    if (type === undefined || !data) {
-      continue;
-    }
-
-    const { doc } = await getCollabStorage(id, type);
-
-    applyYDoc(doc, new Uint8Array(data));
-
-    rowCallback?.(id, doc);
-  }
-}

frontend/appflowy_web_app/src/application/services/js-services/user.service.ts

@@ -8,7 +8,7 @@ import {
   invalidToken,
   setSignInUser,
   setUserWorkspace,
-} from '@/application/services/js-services/storage';
+} from 'src/application/services/js-services/session';
 import { asyncDataDecorator } from '@/application/services/js-services/decorator';
 
 async function getUser() {

frontend/appflowy_web_app/src/application/services/js-services/wasm/client_api.ts

@@ -2,7 +2,7 @@ import { CollabType } from '@/application/collab.type';
 import { ClientAPI } from '@appflowyinc/client-api-wasm';
 import { UserProfile, UserWorkspace } from '@/application/user.type';
 import { AFCloudConfig } from '@/application/services/services.type';
-import { invalidToken, readTokenStr, writeToken } from '@/application/services/js-services/storage';
+import { invalidToken, readTokenStr, writeToken } from 'src/application/services/js-services/session';
 
 let client: ClientAPI;
 

frontend/appflowy_web_app/src/components/_shared/not-found/RecordNotFound.tsx

@@ -1,4 +1,4 @@
-import { getCurrentWorkspace } from '@/application/services/js-services/storage';
+import { getCurrentWorkspace } from 'src/application/services/js-services/session';
 import { Button, Dialog, DialogActions, DialogContent, DialogContentText, DialogTitle } from '@mui/material';
 import React from 'react';
 import { useNavigate } from 'react-router-dom';

frontend/appflowy_web_app/src/components/database/Database.tsx

@@ -12,7 +12,6 @@ export const Database = memo(
     viewId: string;
     onNavigateToView: (viewId: string) => void;
   }) => {
-    console.log('Database', viewId, iidIndex);
     return (
       <div className='appflowy-database relative flex w-full flex-1 select-text flex-col overflow-y-hidden'>
         <DatabaseViews iidIndex={iidIndex} onChangeView={onNavigateToView} viewId={viewId} />

frontend/appflowy_web_app/src/components/editor/components/blocks/database/DatabaseBlock.tsx

@@ -1,6 +1,6 @@
 import { ReactComponent as ExpandMoreIcon } from '$icons/16x/full_view.svg';
 import { useNavigateToView } from '@/application/folder-yjs';
-import { getCurrentWorkspace } from '@/application/services/js-services/storage';
+import { getCurrentWorkspace } from 'src/application/services/js-services/session';
 import { IdProvider } from '@/components/_shared/context-provider/IdProvider';
 import { Database } from '@/components/database';
 import { useGetDatabaseId, useLoadDatabase } from '@/components/database/Database.hooks';

frontend/appflowy_web_app/src/components/layout/breadcrumb/Item.tsx

@@ -9,19 +9,19 @@ import { ReactComponent as CalendarSvg } from '$icons/16x/date.svg';
 
 const renderCrumbIcon = (icon: string) => {
   if (Number(icon) === ViewLayout.Grid) {
-    return <GridSvg />;
+    return <GridSvg className={'h-4 w-4'} />;
   }
 
   if (Number(icon) === ViewLayout.Board) {
-    return <BoardSvg />;
+    return <BoardSvg className={'h-4 w-4'} />;
   }
 
   if (Number(icon) === ViewLayout.Calendar) {
-    return <CalendarSvg />;
+    return <CalendarSvg className={'h-4 w-4'} />;
   }
 
   if (Number(icon) === ViewLayout.Document) {
-    return <DocumentSvg />;
+    return <DocumentSvg className={'h-4 w-4'} />;
   }
 
   return icon;
@@ -41,7 +41,7 @@ function Item({ crumb, disableClick = false }: { crumb: Crumb; disableClick?: bo
         onNavigateToView?.(viewId);
       }}
     >
-      <span className={'h-4 w-4'}>{renderCrumbIcon(icon)}</span>
+      {renderCrumbIcon(icon)}
       <span
         className={!disableClick ? 'max-w-[250px] truncate hover:text-fill-default hover:underline' : 'flex-1 truncate'}
       >