mirror of
https://github.com/AppFlowy-IO/AppFlowy.git
synced 2024-08-30 18:12:39 +00:00
chore: improvements to caching strategies (#5488)
This commit is contained in:
parent
76fde00cc4
commit
86696b271e
@ -71,7 +71,6 @@ export function useDatabaseViewsSelector(iidIndex: string) {
|
||||
const [key] = viewItem;
|
||||
const view = folderViews?.get(key);
|
||||
|
||||
console.log('view', view?.get(YjsFolderKey.bid), iidIndex);
|
||||
if (
|
||||
visibleViewsId.includes(key) &&
|
||||
view &&
|
||||
@ -81,7 +80,6 @@ export function useDatabaseViewsSelector(iidIndex: string) {
|
||||
}
|
||||
}
|
||||
|
||||
console.log('viewsId', viewsId);
|
||||
setViewIds(viewsId);
|
||||
};
|
||||
|
||||
|
@ -0,0 +1,310 @@
|
||||
import { CollabType } from '@/application/collab.type';
|
||||
import * as Y from 'yjs';
|
||||
import { withTestingYDoc } from '@/application/slate-yjs/__tests__/withTestingYjsEditor';
|
||||
import { expect } from '@jest/globals';
|
||||
import { getCollab, batchCollab, collabTypeToDBType } from '../cache';
|
||||
import { applyYDoc } from '@/application/ydoc/apply';
|
||||
import { getCollabDBName, openCollabDB } from '../cache/db';
|
||||
import { StrategyType } from '../cache/types';
|
||||
|
||||
jest.mock('@/application/ydoc/apply', () => ({
|
||||
applyYDoc: jest.fn(),
|
||||
}));
|
||||
jest.mock('../cache/db', () => ({
|
||||
openCollabDB: jest.fn(),
|
||||
getCollabDBName: jest.fn(),
|
||||
}));
|
||||
|
||||
const emptyDoc = new Y.Doc();
|
||||
const normalDoc = withTestingYDoc('1');
|
||||
const mockFetcher = jest.fn();
|
||||
const mockBatchFetcher = jest.fn();
|
||||
|
||||
describe('Cache functions', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('getCollab', () => {
|
||||
describe('with CACHE_ONLY strategy', () => {
|
||||
it('should throw error when no cache', async () => {
|
||||
(openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
|
||||
(getCollabDBName as jest.Mock).mockReturnValue('testDB');
|
||||
|
||||
await expect(
|
||||
getCollab(
|
||||
mockFetcher,
|
||||
{
|
||||
collabId: 'id1',
|
||||
collabType: CollabType.Document,
|
||||
},
|
||||
StrategyType.CACHE_ONLY
|
||||
)
|
||||
).rejects.toThrow('No cache found');
|
||||
});
|
||||
it('should fetch collab with CACHE_ONLY strategy and existing cache', async () => {
|
||||
(openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
|
||||
(getCollabDBName as jest.Mock).mockReturnValue('testDB');
|
||||
|
||||
const result = await getCollab(
|
||||
mockFetcher,
|
||||
{
|
||||
collabId: 'id1',
|
||||
collabType: CollabType.Document,
|
||||
},
|
||||
StrategyType.CACHE_ONLY
|
||||
);
|
||||
|
||||
expect(result).toBe(normalDoc);
|
||||
expect(mockFetcher).not.toHaveBeenCalled();
|
||||
expect(applyYDoc).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('with CACHE_FIRST strategy', () => {
|
||||
it('should fetch collab with CACHE_FIRST strategy and existing cache', async () => {
|
||||
(openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
|
||||
(getCollabDBName as jest.Mock).mockReturnValue('testDB');
|
||||
|
||||
mockFetcher.mockResolvedValue({ state: new Uint8Array() });
|
||||
|
||||
const result = await getCollab(
|
||||
mockFetcher,
|
||||
{
|
||||
collabId: 'id1',
|
||||
collabType: CollabType.Document,
|
||||
},
|
||||
StrategyType.CACHE_FIRST
|
||||
);
|
||||
|
||||
expect(result).toBe(normalDoc);
|
||||
expect(mockFetcher).not.toHaveBeenCalled();
|
||||
expect(applyYDoc).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should fetch collab with CACHE_FIRST strategy and no cache', async () => {
|
||||
(openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
|
||||
(getCollabDBName as jest.Mock).mockReturnValue('testDB');
|
||||
|
||||
mockFetcher.mockResolvedValue({ state: new Uint8Array() });
|
||||
|
||||
const result = await getCollab(
|
||||
mockFetcher,
|
||||
{
|
||||
collabId: 'id1',
|
||||
collabType: CollabType.Document,
|
||||
},
|
||||
StrategyType.CACHE_FIRST
|
||||
);
|
||||
|
||||
expect(result).toBe(emptyDoc);
|
||||
expect(mockFetcher).toHaveBeenCalled();
|
||||
expect(applyYDoc).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('with CACHE_AND_NETWORK strategy', () => {
|
||||
it('should fetch collab with CACHE_AND_NETWORK strategy and existing cache', async () => {
|
||||
(openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
|
||||
(getCollabDBName as jest.Mock).mockReturnValue('testDB');
|
||||
|
||||
mockFetcher.mockResolvedValue({ state: new Uint8Array() });
|
||||
|
||||
const result = await getCollab(
|
||||
mockFetcher,
|
||||
{
|
||||
collabId: 'id1',
|
||||
collabType: CollabType.Document,
|
||||
},
|
||||
StrategyType.CACHE_AND_NETWORK
|
||||
);
|
||||
|
||||
expect(result).toBe(normalDoc);
|
||||
expect(mockFetcher).toHaveBeenCalled();
|
||||
expect(applyYDoc).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should fetch collab with CACHE_AND_NETWORK strategy and no cache', async () => {
|
||||
(openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
|
||||
(getCollabDBName as jest.Mock).mockReturnValue('testDB');
|
||||
|
||||
mockFetcher.mockResolvedValue({ state: new Uint8Array() });
|
||||
|
||||
const result = await getCollab(
|
||||
mockFetcher,
|
||||
{
|
||||
collabId: 'id1',
|
||||
collabType: CollabType.Document,
|
||||
},
|
||||
StrategyType.CACHE_AND_NETWORK
|
||||
);
|
||||
|
||||
expect(result).toBe(emptyDoc);
|
||||
expect(mockFetcher).toHaveBeenCalled();
|
||||
expect(applyYDoc).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('with default strategy', () => {
|
||||
it('should fetch collab with default strategy', async () => {
|
||||
(openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
|
||||
(getCollabDBName as jest.Mock).mockReturnValue('testDB');
|
||||
|
||||
mockFetcher.mockResolvedValue({ state: new Uint8Array() });
|
||||
|
||||
const result = await getCollab(
|
||||
mockFetcher,
|
||||
{
|
||||
collabId: 'id1',
|
||||
collabType: CollabType.Document,
|
||||
},
|
||||
StrategyType.NETWORK_ONLY
|
||||
);
|
||||
|
||||
expect(result).toBe(normalDoc);
|
||||
expect(mockFetcher).toHaveBeenCalled();
|
||||
expect(applyYDoc).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('batchCollab', () => {
|
||||
describe('with CACHE_ONLY strategy', () => {
|
||||
it('should batch fetch collabs with CACHE_ONLY strategy and no cache', async () => {
|
||||
(openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
|
||||
|
||||
(getCollabDBName as jest.Mock).mockReturnValue('testDB');
|
||||
|
||||
await expect(
|
||||
batchCollab(
|
||||
mockBatchFetcher,
|
||||
[
|
||||
{
|
||||
collabId: 'id1',
|
||||
collabType: CollabType.Document,
|
||||
},
|
||||
],
|
||||
StrategyType.CACHE_ONLY
|
||||
)
|
||||
).rejects.toThrow('No cache found');
|
||||
});
|
||||
|
||||
it('should batch fetch collabs with CACHE_ONLY strategy and existing cache', async () => {
|
||||
(openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
|
||||
|
||||
(getCollabDBName as jest.Mock).mockReturnValue('testDB');
|
||||
|
||||
await batchCollab(
|
||||
mockBatchFetcher,
|
||||
[
|
||||
{
|
||||
collabId: 'id1',
|
||||
collabType: CollabType.Document,
|
||||
},
|
||||
],
|
||||
StrategyType.CACHE_ONLY
|
||||
);
|
||||
|
||||
expect(mockBatchFetcher).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('with CACHE_FIRST strategy', () => {
|
||||
it('should batch fetch collabs with CACHE_FIRST strategy and existing cache', async () => {
|
||||
(openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
|
||||
|
||||
(getCollabDBName as jest.Mock).mockReturnValue('testDB');
|
||||
|
||||
await batchCollab(
|
||||
mockBatchFetcher,
|
||||
[
|
||||
{
|
||||
collabId: 'id1',
|
||||
collabType: CollabType.Document,
|
||||
},
|
||||
],
|
||||
StrategyType.CACHE_FIRST
|
||||
);
|
||||
|
||||
expect(mockBatchFetcher).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should batch fetch collabs with CACHE_FIRST strategy and no cache', async () => {
|
||||
(openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
|
||||
|
||||
(getCollabDBName as jest.Mock).mockReturnValue('testDB');
|
||||
mockBatchFetcher.mockResolvedValue({ id1: [1, 2, 3] });
|
||||
|
||||
await batchCollab(
|
||||
mockBatchFetcher,
|
||||
[
|
||||
{
|
||||
collabId: 'id1',
|
||||
collabType: CollabType.Document,
|
||||
},
|
||||
],
|
||||
StrategyType.CACHE_FIRST
|
||||
);
|
||||
|
||||
expect(mockBatchFetcher).toHaveBeenCalled();
|
||||
expect(applyYDoc).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('with CACHE_AND_NETWORK strategy', () => {
|
||||
it('should batch fetch collabs with CACHE_AND_NETWORK strategy', async () => {
|
||||
(openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
|
||||
|
||||
(getCollabDBName as jest.Mock).mockReturnValue('testDB');
|
||||
mockBatchFetcher.mockResolvedValue({ id1: [1, 2, 3] });
|
||||
|
||||
await batchCollab(
|
||||
mockBatchFetcher,
|
||||
[
|
||||
{
|
||||
collabId: 'id1',
|
||||
collabType: CollabType.Document,
|
||||
},
|
||||
],
|
||||
StrategyType.CACHE_AND_NETWORK
|
||||
);
|
||||
|
||||
expect(mockBatchFetcher).toHaveBeenCalled();
|
||||
expect(applyYDoc).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should batch fetch collabs with CACHE_AND_NETWORK strategy and no cache', async () => {
|
||||
(openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
|
||||
|
||||
(getCollabDBName as jest.Mock).mockReturnValue('testDB');
|
||||
mockBatchFetcher.mockResolvedValue({ id1: [1, 2, 3] });
|
||||
|
||||
await batchCollab(
|
||||
mockBatchFetcher,
|
||||
[
|
||||
{
|
||||
collabId: 'id1',
|
||||
collabType: CollabType.Document,
|
||||
},
|
||||
],
|
||||
StrategyType.CACHE_AND_NETWORK
|
||||
);
|
||||
|
||||
expect(mockBatchFetcher).toHaveBeenCalled();
|
||||
expect(applyYDoc).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('collabTypeToDBType', () => {
|
||||
it('should return correct DB type', () => {
|
||||
expect(collabTypeToDBType(CollabType.Document)).toBe('document');
|
||||
expect(collabTypeToDBType(CollabType.Folder)).toBe('folder');
|
||||
expect(collabTypeToDBType(CollabType.Database)).toBe('database');
|
||||
expect(collabTypeToDBType(CollabType.WorkspaceDatabase)).toBe('databases');
|
||||
expect(collabTypeToDBType(CollabType.DatabaseRow)).toBe('database_row');
|
||||
expect(collabTypeToDBType(CollabType.UserAwareness)).toBe('user_awareness');
|
||||
expect(collabTypeToDBType(CollabType.Empty)).toBe('');
|
||||
});
|
||||
});
|
@ -0,0 +1,57 @@
|
||||
import { expect } from '@jest/globals';
|
||||
import { fetchCollab, batchFetchCollab } from '../fetch';
|
||||
import { CollabType } from '@/application/collab.type';
|
||||
import { APIService } from '@/application/services/js-services/wasm';
|
||||
|
||||
jest.mock('@/application/services/js-services/wasm', () => {
|
||||
return {
|
||||
APIService: {
|
||||
getCollab: jest.fn(),
|
||||
batchGetCollab: jest.fn(),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
describe('Collab fetch functions with deduplication', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('fetchCollab', () => {
|
||||
it('should fetch collab without duplicating requests', async () => {
|
||||
const workspaceId = 'workspace1';
|
||||
const id = 'id1';
|
||||
const type = CollabType.Document;
|
||||
const mockResponse = { data: 'mockData' };
|
||||
|
||||
(APIService.getCollab as jest.Mock).mockResolvedValue(mockResponse);
|
||||
|
||||
const result1 = fetchCollab(workspaceId, id, type);
|
||||
const result2 = fetchCollab(workspaceId, id, type);
|
||||
|
||||
expect(result1).toBe(result2);
|
||||
await expect(result1).resolves.toEqual(mockResponse);
|
||||
expect(APIService.getCollab).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('batchFetchCollab', () => {
|
||||
it('should batch fetch collabs without duplicating requests', async () => {
|
||||
const workspaceId = 'workspace1';
|
||||
const params = [
|
||||
{ collabId: 'id1', collabType: CollabType.Document },
|
||||
{ collabId: 'id2', collabType: CollabType.Folder },
|
||||
];
|
||||
const mockResponse = { data: 'mockData' };
|
||||
|
||||
(APIService.batchGetCollab as jest.Mock).mockResolvedValue(mockResponse);
|
||||
|
||||
const result1 = batchFetchCollab(workspaceId, params);
|
||||
const result2 = batchFetchCollab(workspaceId, params);
|
||||
|
||||
expect(result1).toBe(result2);
|
||||
await expect(result1).resolves.toEqual(mockResponse);
|
||||
expect(APIService.batchGetCollab).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
});
|
@ -1,8 +1,8 @@
|
||||
import { AuthService } from '@/application/services/services.type';
|
||||
import { ProviderType, SignUpWithEmailPasswordParams } from '@/application/user.type';
|
||||
import { APIService } from 'src/application/services/js-services/wasm';
|
||||
import { signInSuccess } from '@/application/services/js-services/storage/auth';
|
||||
import { invalidToken } from '@/application/services/js-services/storage';
|
||||
import { signInSuccess } from '@/application/services/js-services/session/auth';
|
||||
import { invalidToken } from 'src/application/services/js-services/session';
|
||||
import { afterSignInDecorator } from '@/application/services/js-services/decorator';
|
||||
|
||||
export class JSAuthService implements AuthService {
|
||||
|
@ -1,9 +1,10 @@
|
||||
import { YDoc } from '@/application/collab.type';
|
||||
import { databasePrefix } from '@/application/constants';
|
||||
import { getAuthInfo } from '@/application/services/js-services/storage';
|
||||
import { IndexeddbPersistence } from 'y-indexeddb';
|
||||
import * as Y from 'yjs';
|
||||
|
||||
const openedSet = new Set<string>();
|
||||
|
||||
/**
|
||||
* Open the collaboration database, and return a function to close it
|
||||
*/
|
||||
@ -19,6 +20,10 @@ export async function openCollabDB(docName: string): Promise<YDoc> {
|
||||
});
|
||||
|
||||
provider.on('synced', () => {
|
||||
if (!openedSet.has(name)) {
|
||||
openedSet.add(name);
|
||||
}
|
||||
|
||||
resolve(true);
|
||||
});
|
||||
|
||||
@ -27,9 +32,10 @@ export async function openCollabDB(docName: string): Promise<YDoc> {
|
||||
return doc as YDoc;
|
||||
}
|
||||
|
||||
export function getDBName(id: string, type: string) {
|
||||
const { uuid } = getAuthInfo() || {};
|
||||
export function getCollabDBName(id: string, type: string, uuid?: string) {
|
||||
if (!uuid) {
|
||||
return `${type}_${id}`;
|
||||
}
|
||||
|
||||
if (!uuid) throw new Error('No user found');
|
||||
return `${uuid}_${type}_${id}`;
|
||||
}
|
165
frontend/appflowy_web_app/src/application/services/js-services/cache/index.ts
vendored
Normal file
165
frontend/appflowy_web_app/src/application/services/js-services/cache/index.ts
vendored
Normal file
@ -0,0 +1,165 @@
|
||||
import { CollabType, YDoc, YjsEditorKey, YSharedRoot } from '@/application/collab.type';
|
||||
import { applyYDoc } from '@/application/ydoc/apply';
|
||||
import { getCollabDBName, openCollabDB } from './db';
|
||||
import { Fetcher, StrategyType } from './types';
|
||||
|
||||
export function collabTypeToDBType(type: CollabType) {
|
||||
switch (type) {
|
||||
case CollabType.Folder:
|
||||
return 'folder';
|
||||
case CollabType.Document:
|
||||
return 'document';
|
||||
case CollabType.Database:
|
||||
return 'database';
|
||||
case CollabType.WorkspaceDatabase:
|
||||
return 'databases';
|
||||
case CollabType.DatabaseRow:
|
||||
return 'database_row';
|
||||
case CollabType.UserAwareness:
|
||||
return 'user_awareness';
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
const collabSharedRootKeyMap = {
|
||||
[CollabType.Folder]: YjsEditorKey.folder,
|
||||
[CollabType.Document]: YjsEditorKey.document,
|
||||
[CollabType.Database]: YjsEditorKey.database,
|
||||
[CollabType.WorkspaceDatabase]: YjsEditorKey.workspace_database,
|
||||
[CollabType.DatabaseRow]: YjsEditorKey.database_row,
|
||||
[CollabType.UserAwareness]: YjsEditorKey.user_awareness,
|
||||
[CollabType.Empty]: YjsEditorKey.empty,
|
||||
};
|
||||
|
||||
export function hasCache(doc: YDoc, type: CollabType) {
|
||||
const data = doc.getMap(YjsEditorKey.data_section) as YSharedRoot;
|
||||
|
||||
return data.has(collabSharedRootKeyMap[type] as string);
|
||||
}
|
||||
|
||||
export async function getCollab(
|
||||
fetcher: Fetcher<{
|
||||
state: Uint8Array;
|
||||
}>,
|
||||
{
|
||||
collabId,
|
||||
collabType,
|
||||
uuid,
|
||||
}: {
|
||||
uuid?: string;
|
||||
collabId: string;
|
||||
collabType: CollabType;
|
||||
},
|
||||
strategy: StrategyType = StrategyType.CACHE_AND_NETWORK
|
||||
) {
|
||||
const name = getCollabDBName(collabId, collabTypeToDBType(collabType), uuid);
|
||||
const collab = await openCollabDB(name);
|
||||
const exist = hasCache(collab, collabType);
|
||||
|
||||
switch (strategy) {
|
||||
case StrategyType.CACHE_ONLY: {
|
||||
if (!exist) {
|
||||
throw new Error('No cache found');
|
||||
}
|
||||
|
||||
return collab;
|
||||
}
|
||||
|
||||
case StrategyType.CACHE_FIRST: {
|
||||
if (!exist) {
|
||||
await revalidateCollab(fetcher, collab);
|
||||
}
|
||||
|
||||
return collab;
|
||||
}
|
||||
|
||||
case StrategyType.CACHE_AND_NETWORK: {
|
||||
if (!exist) {
|
||||
await revalidateCollab(fetcher, collab);
|
||||
} else {
|
||||
void revalidateCollab(fetcher, collab);
|
||||
}
|
||||
|
||||
return collab;
|
||||
}
|
||||
|
||||
default: {
|
||||
await revalidateCollab(fetcher, collab);
|
||||
|
||||
return collab;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function revalidateCollab(
|
||||
fetcher: Fetcher<{
|
||||
state: Uint8Array;
|
||||
}>,
|
||||
collab: YDoc
|
||||
) {
|
||||
const { state } = await fetcher();
|
||||
|
||||
applyYDoc(collab, state);
|
||||
}
|
||||
|
||||
export async function batchCollab(
|
||||
batchFetcher: Fetcher<Record<string, number[]>>,
|
||||
collabs: {
|
||||
collabId: string;
|
||||
collabType: CollabType;
|
||||
uuid?: string;
|
||||
}[],
|
||||
strategy: StrategyType = StrategyType.CACHE_AND_NETWORK,
|
||||
itemCallback?: (id: string, doc: YDoc) => void
|
||||
) {
|
||||
const collabMap = new Map<string, YDoc>();
|
||||
|
||||
for (const { collabId, collabType, uuid } of collabs) {
|
||||
const name = getCollabDBName(collabId, collabTypeToDBType(collabType), uuid);
|
||||
const collab = await openCollabDB(name);
|
||||
const exist = hasCache(collab, collabType);
|
||||
|
||||
collabMap.set(collabId, collab);
|
||||
if (exist) {
|
||||
itemCallback?.(collabId, collab);
|
||||
}
|
||||
}
|
||||
|
||||
const notCacheIds = collabs.filter(({ collabId, collabType }) => {
|
||||
const id = collabMap.get(collabId);
|
||||
|
||||
if (!id) return false;
|
||||
|
||||
return !hasCache(id, collabType);
|
||||
});
|
||||
|
||||
if (strategy === StrategyType.CACHE_ONLY) {
|
||||
if (notCacheIds.length > 0) {
|
||||
throw new Error('No cache found');
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (strategy === StrategyType.CACHE_FIRST && notCacheIds.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const states = await batchFetcher();
|
||||
|
||||
for (const [collabId, data] of Object.entries(states)) {
|
||||
const info = collabs.find((item) => item.collabId === collabId);
|
||||
const collab = collabMap.get(collabId);
|
||||
|
||||
if (!info || !collab) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const state = new Uint8Array(data);
|
||||
|
||||
applyYDoc(collab, state);
|
||||
|
||||
itemCallback?.(collabId, collab);
|
||||
}
|
||||
}
|
12
frontend/appflowy_web_app/src/application/services/js-services/cache/types.ts
vendored
Normal file
12
frontend/appflowy_web_app/src/application/services/js-services/cache/types.ts
vendored
Normal file
@ -0,0 +1,12 @@
|
||||
export enum StrategyType {
|
||||
// Cache only: return the cache if it exists, otherwise throw an error
|
||||
CACHE_ONLY = 'CACHE_ONLY',
|
||||
// Cache first: return the cache if it exists, otherwise fetch from the network
|
||||
CACHE_FIRST = 'CACHE_FIRST',
|
||||
// Cache and network: return the cache if it exists, otherwise fetch from the network and update the cache
|
||||
CACHE_AND_NETWORK = 'CACHE_AND_NETWORK',
|
||||
// Network only: fetch from the network and update the cache
|
||||
NETWORK_ONLY = 'NETWORK_ONLY',
|
||||
}
|
||||
|
||||
export type Fetcher<T> = () => Promise<T>;
|
@ -1,16 +1,16 @@
|
||||
import { CollabType, YDatabase, YDoc, YjsDatabaseKey, YjsEditorKey } from '@/application/collab.type';
|
||||
import {
|
||||
batchCollabs,
|
||||
getCollabStorage,
|
||||
getCollabStorageWithAPICall,
|
||||
getCurrentWorkspace,
|
||||
} from '@/application/services/js-services/storage';
|
||||
import { batchCollab, getCollab } from '@/application/services/js-services/cache';
|
||||
import { StrategyType } from '@/application/services/js-services/cache/types';
|
||||
import { batchFetchCollab, fetchCollab } from '@/application/services/js-services/fetch';
|
||||
import { getCurrentWorkspace } from 'src/application/services/js-services/session';
|
||||
import { DatabaseService } from '@/application/services/services.type';
|
||||
import * as Y from 'yjs';
|
||||
|
||||
export class JSDatabaseService implements DatabaseService {
|
||||
private loadedDatabaseId: Set<string> = new Set();
|
||||
|
||||
private loadedWorkspaceId: Set<string> = new Set();
|
||||
|
||||
private cacheDatabaseRowDocMap: Map<string, Y.Doc> = new Map();
|
||||
|
||||
constructor() {
|
||||
@ -28,22 +28,30 @@ export class JSDatabaseService implements DatabaseService {
|
||||
throw new Error('Workspace database not found');
|
||||
}
|
||||
|
||||
const workspaceDatabase = await getCollabStorageWithAPICall(
|
||||
workspace.id,
|
||||
workspace.workspaceDatabaseId,
|
||||
CollabType.WorkspaceDatabase
|
||||
const isLoaded = this.loadedWorkspaceId.has(workspace.id);
|
||||
|
||||
const workspaceDatabase = await getCollab(
|
||||
() => {
|
||||
return fetchCollab(workspace.id, workspace.workspaceDatabaseId, CollabType.WorkspaceDatabase);
|
||||
},
|
||||
{
|
||||
collabId: workspace.workspaceDatabaseId,
|
||||
collabType: CollabType.WorkspaceDatabase,
|
||||
},
|
||||
isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK
|
||||
);
|
||||
|
||||
if (!isLoaded) {
|
||||
this.loadedWorkspaceId.add(workspace.id);
|
||||
}
|
||||
|
||||
return workspaceDatabase.getMap(YjsEditorKey.data_section).get(YjsEditorKey.workspace_database).toJSON() as {
|
||||
views: string[];
|
||||
database_id: string;
|
||||
}[];
|
||||
}
|
||||
|
||||
async openDatabase(
|
||||
databaseId: string,
|
||||
rowIds?: string[]
|
||||
): Promise<{
|
||||
async openDatabase(databaseId: string): Promise<{
|
||||
databaseDoc: YDoc;
|
||||
rows: Y.Map<YDoc>;
|
||||
}> {
|
||||
@ -68,13 +76,18 @@ export class JSDatabaseService implements DatabaseService {
|
||||
|
||||
const rowsFolder: Y.Map<YDoc> = rootRowsDoc.getMap();
|
||||
|
||||
let databaseDoc: YDoc | undefined = undefined;
|
||||
const databaseDoc = await getCollab(
|
||||
() => {
|
||||
return fetchCollab(workspaceId, databaseId, CollabType.Database);
|
||||
},
|
||||
{
|
||||
collabId: databaseId,
|
||||
collabType: CollabType.Database,
|
||||
},
|
||||
isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK
|
||||
);
|
||||
|
||||
if (isLoaded) {
|
||||
databaseDoc = (await getCollabStorage(databaseId, CollabType.Database)).doc;
|
||||
} else {
|
||||
databaseDoc = await getCollabStorageWithAPICall(workspaceId, databaseId, CollabType.Database);
|
||||
}
|
||||
if (!isLoaded) this.loadedDatabaseId.add(databaseId);
|
||||
|
||||
const database = databaseDoc.getMap(YjsEditorKey.data_section)?.get(YjsEditorKey.database) as YDatabase;
|
||||
const viewId = database.get(YjsDatabaseKey.metas)?.get(YjsDatabaseKey.iid)?.toString();
|
||||
@ -87,47 +100,50 @@ export class JSDatabaseService implements DatabaseService {
|
||||
throw new Error('Database rows not found');
|
||||
}
|
||||
|
||||
const ids = rowIds ? rowIds : rowOrdersIds.map((item) => item.id);
|
||||
|
||||
if (isLoaded) {
|
||||
for (const id of ids) {
|
||||
const { doc } = await getCollabStorage(id, CollabType.DatabaseRow);
|
||||
const rowsParams = rowOrdersIds.map((item) => ({
|
||||
collabId: item.id,
|
||||
collabType: CollabType.DatabaseRow,
|
||||
}));
|
||||
|
||||
void batchCollab(
|
||||
() => {
|
||||
return batchFetchCollab(workspaceId, rowsParams);
|
||||
},
|
||||
rowsParams,
|
||||
isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK,
|
||||
(id: string, doc: YDoc) => {
|
||||
if (!rowsFolder.has(id)) {
|
||||
rowsFolder.set(id, doc);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
void this.loadDatabaseRows(workspaceId, ids, (id, row) => {
|
||||
if (!rowsFolder.has(id)) {
|
||||
rowsFolder.set(id, row);
|
||||
}
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
this.loadedDatabaseId.add(databaseId);
|
||||
// Update rows if there are new rows added after the database has been loaded
|
||||
rowOrders?.observe((event) => {
|
||||
if (event.changes.added.size > 0) {
|
||||
const rowIds = rowOrders.toJSON() as {
|
||||
id: string;
|
||||
}[];
|
||||
|
||||
if (!rowIds) {
|
||||
// Update rows if new rows are added
|
||||
rowOrders?.observe((event) => {
|
||||
if (event.changes.added.size > 0) {
|
||||
const rowIds = rowOrders.toJSON() as {
|
||||
id: string;
|
||||
}[];
|
||||
const params = rowIds.map((item) => ({
|
||||
collabId: item.id,
|
||||
collabType: CollabType.DatabaseRow,
|
||||
}));
|
||||
|
||||
console.log('Update rows', rowIds);
|
||||
void this.loadDatabaseRows(
|
||||
workspaceId,
|
||||
rowIds.map((item) => item.id),
|
||||
(rowId: string, rowDoc) => {
|
||||
if (!rowsFolder.has(rowId)) {
|
||||
rowsFolder.set(rowId, rowDoc);
|
||||
}
|
||||
void batchCollab(
|
||||
() => {
|
||||
return batchFetchCollab(workspaceId, params);
|
||||
},
|
||||
params,
|
||||
StrategyType.CACHE_AND_NETWORK,
|
||||
(id: string, doc: YDoc) => {
|
||||
if (!rowsFolder.has(id)) {
|
||||
rowsFolder.set(id, doc);
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
databaseDoc,
|
||||
@ -135,21 +151,6 @@ export class JSDatabaseService implements DatabaseService {
|
||||
};
|
||||
}
|
||||
|
||||
async loadDatabaseRows(workspaceId: string, rowIds: string[], rowCallback: (rowId: string, rowDoc: YDoc) => void) {
|
||||
try {
|
||||
await batchCollabs(
|
||||
workspaceId,
|
||||
rowIds.map((id) => ({
|
||||
object_id: id,
|
||||
collab_type: CollabType.DatabaseRow,
|
||||
})),
|
||||
rowCallback
|
||||
);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
async closeDatabase(databaseId: string) {
|
||||
this.cacheDatabaseRowDocMap.delete(databaseId);
|
||||
}
|
||||
|
@ -1,8 +1,13 @@
|
||||
import { CollabOrigin, CollabType, YDoc } from '@/application/collab.type';
|
||||
import { getCollabStorageWithAPICall, getCurrentWorkspace } from '@/application/services/js-services/storage';
|
||||
import { getCollab } from '@/application/services/js-services/cache';
|
||||
import { StrategyType } from '@/application/services/js-services/cache/types';
|
||||
import { fetchCollab } from '@/application/services/js-services/fetch';
|
||||
import { getCurrentWorkspace } from 'src/application/services/js-services/session';
|
||||
import { DocumentService } from '@/application/services/services.type';
|
||||
|
||||
export class JSDocumentService implements DocumentService {
|
||||
private loaded: Set<string> = new Set();
|
||||
|
||||
constructor() {
|
||||
//
|
||||
}
|
||||
@ -14,8 +19,20 @@ export class JSDocumentService implements DocumentService {
|
||||
throw new Error('Workspace database not found');
|
||||
}
|
||||
|
||||
const doc = await getCollabStorageWithAPICall(workspace.id, docId, CollabType.Document);
|
||||
const isLoaded = this.loaded.has(docId);
|
||||
|
||||
const doc = await getCollab(
|
||||
() => {
|
||||
return fetchCollab(workspace.id, docId, CollabType.Document);
|
||||
},
|
||||
{
|
||||
collabId: docId,
|
||||
collabType: CollabType.Document,
|
||||
},
|
||||
isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK
|
||||
);
|
||||
|
||||
if (!isLoaded) this.loaded.add(docId);
|
||||
const handleUpdate = (update: Uint8Array, origin: CollabOrigin) => {
|
||||
if (origin === CollabOrigin.LocalSync) {
|
||||
// Send the update to the server
|
||||
|
@ -0,0 +1,66 @@
|
||||
import { CollabType } from '@/application/collab.type';
|
||||
import { APIService } from '@/application/services/js-services/wasm';
|
||||
|
||||
const pendingRequests = new Map();
|
||||
|
||||
function generateRequestKey<T>(url: string, params: T) {
|
||||
if (!params) return url;
|
||||
|
||||
try {
|
||||
return `${url}_${JSON.stringify(params)}`;
|
||||
} catch (_e) {
|
||||
return `${url}_${params}`;
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplication fetch requests
|
||||
// When multiple requests are made to the same URL with the same params, only one request is made
|
||||
// and the result is shared with all the requests
|
||||
function fetchWithDeduplication<Req, Res>(url: string, params: Req, fetchFunction: () => Promise<Res>): Promise<Res> {
|
||||
const requestKey = generateRequestKey<Req>(url, params);
|
||||
|
||||
if (pendingRequests.has(requestKey)) {
|
||||
return pendingRequests.get(requestKey);
|
||||
}
|
||||
|
||||
const fetchPromise = fetchFunction().finally(() => {
|
||||
pendingRequests.delete(requestKey);
|
||||
});
|
||||
|
||||
pendingRequests.set(requestKey, fetchPromise);
|
||||
return fetchPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch collab
|
||||
* @param workspaceId
|
||||
* @param id
|
||||
* @param type [CollabType]
|
||||
*/
|
||||
export function fetchCollab(workspaceId: string, id: string, type: CollabType) {
|
||||
const fetchFunction = () => APIService.getCollab(workspaceId, id, type);
|
||||
|
||||
return fetchWithDeduplication(`fetchCollab_${workspaceId}`, { id, type }, fetchFunction);
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch fetch collab
|
||||
* Usage:
|
||||
* // load database rows
|
||||
* const rows = await batchFetchCollab(workspaceId, databaseRows.map((row) => ({ collabId: row.id, collabType: CollabType.DatabaseRow })));
|
||||
*
|
||||
* @param workspaceId
|
||||
* @param params [{ collabId: string; collabType: CollabType }]
|
||||
*/
|
||||
export function batchFetchCollab(workspaceId: string, params: { collabId: string; collabType: CollabType }[]) {
|
||||
const fetchFunction = () =>
|
||||
APIService.batchGetCollab(
|
||||
workspaceId,
|
||||
params.map(({ collabId, collabType }) => ({
|
||||
object_id: collabId,
|
||||
collab_type: collabType,
|
||||
}))
|
||||
);
|
||||
|
||||
return fetchWithDeduplication(`batchFetchCollab_${workspaceId}`, params, fetchFunction);
|
||||
}
|
@ -1,14 +1,30 @@
|
||||
import { CollabOrigin, CollabType, YDoc } from '@/application/collab.type';
|
||||
import { getCollabStorageWithAPICall } from '@/application/services/js-services/storage';
|
||||
import { getCollab } from '@/application/services/js-services/cache';
|
||||
import { StrategyType } from '@/application/services/js-services/cache/types';
|
||||
import { fetchCollab } from '@/application/services/js-services/fetch';
|
||||
import { FolderService } from '@/application/services/services.type';
|
||||
|
||||
export class JSFolderService implements FolderService {
|
||||
private loaded: Set<string> = new Set();
|
||||
|
||||
constructor() {
|
||||
//
|
||||
}
|
||||
|
||||
async openWorkspace(workspaceId: string): Promise<YDoc> {
|
||||
const doc = await getCollabStorageWithAPICall(workspaceId, workspaceId, CollabType.Folder);
|
||||
const isLoaded = this.loaded.has(workspaceId);
|
||||
const doc = await getCollab(
|
||||
() => {
|
||||
return fetchCollab(workspaceId, workspaceId, CollabType.Folder);
|
||||
},
|
||||
{
|
||||
collabId: workspaceId,
|
||||
collabType: CollabType.Folder,
|
||||
},
|
||||
isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK
|
||||
);
|
||||
|
||||
if (!isLoaded) this.loaded.add(workspaceId);
|
||||
const handleUpdate = (update: Uint8Array, origin: CollabOrigin) => {
|
||||
if (origin === CollabOrigin.LocalSync) {
|
||||
// Send the update to the server
|
||||
|
@ -1,4 +1,3 @@
|
||||
export * from './token';
|
||||
export * from './user';
|
||||
export * from './collab';
|
||||
export * from './auth';
|
@ -1,118 +0,0 @@
|
||||
import { CollabType, YDoc, YjsEditorKey, YSharedRoot } from '@/application/collab.type';
|
||||
import { getDBName, openCollabDB } from '@/application/services/js-services/db';
|
||||
import { APIService } from '@/application/services/js-services/wasm';
|
||||
import { applyYDoc } from '@/application/ydoc/apply';
|
||||
|
||||
export function fetchCollab(workspaceId: string, id: string, type: CollabType) {
|
||||
return APIService.getCollab(workspaceId, id, type);
|
||||
}
|
||||
|
||||
export function batchFetchCollab(workspaceId: string, params: { object_id: string; collab_type: CollabType }[]) {
|
||||
return APIService.batchGetCollab(workspaceId, params);
|
||||
}
|
||||
|
||||
function collabTypeToDBType(type: CollabType) {
|
||||
switch (type) {
|
||||
case CollabType.Folder:
|
||||
return 'folder';
|
||||
case CollabType.Document:
|
||||
return 'document';
|
||||
case CollabType.Database:
|
||||
return 'database';
|
||||
case CollabType.WorkspaceDatabase:
|
||||
return 'databases';
|
||||
case CollabType.DatabaseRow:
|
||||
return 'database_row';
|
||||
case CollabType.UserAwareness:
|
||||
return 'user_awareness';
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
const collabSharedRootKeyMap = {
|
||||
[CollabType.Folder]: YjsEditorKey.folder,
|
||||
[CollabType.Document]: YjsEditorKey.document,
|
||||
[CollabType.Database]: YjsEditorKey.database,
|
||||
[CollabType.WorkspaceDatabase]: YjsEditorKey.workspace_database,
|
||||
[CollabType.DatabaseRow]: YjsEditorKey.database_row,
|
||||
[CollabType.UserAwareness]: YjsEditorKey.user_awareness,
|
||||
[CollabType.Empty]: YjsEditorKey.empty,
|
||||
};
|
||||
|
||||
export async function getCollabStorage(id: string, type: CollabType) {
|
||||
const name = getDBName(id, collabTypeToDBType(type));
|
||||
|
||||
const doc = await openCollabDB(name);
|
||||
let localExist = false;
|
||||
const existData = doc.share.has(YjsEditorKey.data_section);
|
||||
|
||||
if (existData) {
|
||||
const data = doc.getMap(YjsEditorKey.data_section) as YSharedRoot;
|
||||
|
||||
localExist = data.has(collabSharedRootKeyMap[type] as string);
|
||||
}
|
||||
|
||||
return {
|
||||
doc,
|
||||
localExist,
|
||||
};
|
||||
}
|
||||
|
||||
export async function getCollabStorageWithAPICall(workspaceId: string, id: string, type: CollabType) {
|
||||
const { doc, localExist } = await getCollabStorage(id, type);
|
||||
const asyncApply = async () => {
|
||||
const res = await fetchCollab(workspaceId, id, type);
|
||||
|
||||
applyYDoc(doc, res.state);
|
||||
};
|
||||
|
||||
// If the document exists locally, apply the state asynchronously,
|
||||
// otherwise, apply the state synchronously
|
||||
if (localExist) {
|
||||
void asyncApply();
|
||||
} else {
|
||||
await asyncApply();
|
||||
}
|
||||
|
||||
return doc;
|
||||
}
|
||||
|
||||
export async function batchCollabs(
|
||||
workspaceId: string,
|
||||
params: {
|
||||
object_id: string;
|
||||
collab_type: CollabType;
|
||||
}[],
|
||||
rowCallback?: (id: string, doc: YDoc) => void
|
||||
) {
|
||||
console.log('Fetching collab data:', params);
|
||||
// Create or get Y.Doc from local storage
|
||||
for (const item of params) {
|
||||
const { object_id, collab_type } = item;
|
||||
|
||||
const { doc, localExist } = await getCollabStorage(object_id, collab_type);
|
||||
|
||||
if (rowCallback && localExist) {
|
||||
rowCallback(object_id, doc);
|
||||
}
|
||||
}
|
||||
|
||||
const res = await batchFetchCollab(workspaceId, params);
|
||||
|
||||
console.log('Fetched collab data:', res);
|
||||
for (const id of Object.keys(res)) {
|
||||
const type = params.find((param) => param.object_id === id)?.collab_type;
|
||||
const data = res[id];
|
||||
|
||||
if (type === undefined || !data) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const { doc } = await getCollabStorage(id, type);
|
||||
|
||||
applyYDoc(doc, new Uint8Array(data));
|
||||
|
||||
rowCallback?.(id, doc);
|
||||
}
|
||||
}
|
@ -8,7 +8,7 @@ import {
|
||||
invalidToken,
|
||||
setSignInUser,
|
||||
setUserWorkspace,
|
||||
} from '@/application/services/js-services/storage';
|
||||
} from 'src/application/services/js-services/session';
|
||||
import { asyncDataDecorator } from '@/application/services/js-services/decorator';
|
||||
|
||||
async function getUser() {
|
||||
|
@ -2,7 +2,7 @@ import { CollabType } from '@/application/collab.type';
|
||||
import { ClientAPI } from '@appflowyinc/client-api-wasm';
|
||||
import { UserProfile, UserWorkspace } from '@/application/user.type';
|
||||
import { AFCloudConfig } from '@/application/services/services.type';
|
||||
import { invalidToken, readTokenStr, writeToken } from '@/application/services/js-services/storage';
|
||||
import { invalidToken, readTokenStr, writeToken } from 'src/application/services/js-services/session';
|
||||
|
||||
let client: ClientAPI;
|
||||
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { getCurrentWorkspace } from '@/application/services/js-services/storage';
|
||||
import { getCurrentWorkspace } from 'src/application/services/js-services/session';
|
||||
import { Button, Dialog, DialogActions, DialogContent, DialogContentText, DialogTitle } from '@mui/material';
|
||||
import React from 'react';
|
||||
import { useNavigate } from 'react-router-dom';
|
||||
|
@ -12,7 +12,6 @@ export const Database = memo(
|
||||
viewId: string;
|
||||
onNavigateToView: (viewId: string) => void;
|
||||
}) => {
|
||||
console.log('Database', viewId, iidIndex);
|
||||
return (
|
||||
<div className='appflowy-database relative flex w-full flex-1 select-text flex-col overflow-y-hidden'>
|
||||
<DatabaseViews iidIndex={iidIndex} onChangeView={onNavigateToView} viewId={viewId} />
|
||||
|
@ -1,6 +1,6 @@
|
||||
import { ReactComponent as ExpandMoreIcon } from '$icons/16x/full_view.svg';
|
||||
import { useNavigateToView } from '@/application/folder-yjs';
|
||||
import { getCurrentWorkspace } from '@/application/services/js-services/storage';
|
||||
import { getCurrentWorkspace } from 'src/application/services/js-services/session';
|
||||
import { IdProvider } from '@/components/_shared/context-provider/IdProvider';
|
||||
import { Database } from '@/components/database';
|
||||
import { useGetDatabaseId, useLoadDatabase } from '@/components/database/Database.hooks';
|
||||
|
@ -9,19 +9,19 @@ import { ReactComponent as CalendarSvg } from '$icons/16x/date.svg';
|
||||
|
||||
const renderCrumbIcon = (icon: string) => {
|
||||
if (Number(icon) === ViewLayout.Grid) {
|
||||
return <GridSvg />;
|
||||
return <GridSvg className={'h-4 w-4'} />;
|
||||
}
|
||||
|
||||
if (Number(icon) === ViewLayout.Board) {
|
||||
return <BoardSvg />;
|
||||
return <BoardSvg className={'h-4 w-4'} />;
|
||||
}
|
||||
|
||||
if (Number(icon) === ViewLayout.Calendar) {
|
||||
return <CalendarSvg />;
|
||||
return <CalendarSvg className={'h-4 w-4'} />;
|
||||
}
|
||||
|
||||
if (Number(icon) === ViewLayout.Document) {
|
||||
return <DocumentSvg />;
|
||||
return <DocumentSvg className={'h-4 w-4'} />;
|
||||
}
|
||||
|
||||
return icon;
|
||||
@ -41,7 +41,7 @@ function Item({ crumb, disableClick = false }: { crumb: Crumb; disableClick?: bo
|
||||
onNavigateToView?.(viewId);
|
||||
}}
|
||||
>
|
||||
<span className={'h-4 w-4'}>{renderCrumbIcon(icon)}</span>
|
||||
{renderCrumbIcon(icon)}
|
||||
<span
|
||||
className={!disableClick ? 'max-w-[250px] truncate hover:text-fill-default hover:underline' : 'flex-1 truncate'}
|
||||
>
|
||||
|
Loading…
Reference in New Issue
Block a user