Major 3.0 (#118)

Co-authored-by: Maison da Silva <maisonmdsgreen@hotmail.com>

features/collectionStorage/utils/collectionStorage.ts

@@ -0,0 +1,13 @@
+import { CollectionItem, GraphicsStorage } from "@/models/CollectionModels";
+
+export const collectionStorage =
+{
+	chunkCount: storage.defineItem<number>("sync:chunkCount", { fallback: 0 }),
+	syncLastUpdated: storage.defineItem<number>("sync:lastUpdated", { fallback: 0 }),
+	localLastUpdated: storage.defineItem<number>("local:lastUpdated", { fallback: 0 }),
+	localCollections: storage.defineItem<CollectionItem[]>("local:collections", { fallback: [] }),
+	count: storage.defineItem<number>("local:count", { fallback: 0 }),
+	graphics: storage.defineItem<GraphicsStorage>("local:graphics", { fallback: {} }),
+	disableCloud: storage.defineItem<boolean>("sync:disableCloud", { fallback: false }),
+	maxChunkCount: 12
+};

features/collectionStorage/utils/getChunkKeys.ts

@@ -0,0 +1,6 @@
+import { collectionStorage } from "./collectionStorage";
+
+export default function getChunkKeys(start: number = 0, end: number = collectionStorage.maxChunkCount): string[]
+{
+	return Array.from({ length: end - start }, (_, i) => "c" + (i + start));
+}

features/collectionStorage/utils/getCollections.ts

@@ -0,0 +1,41 @@
+import { trackError } from "@/features/analytics";
+import { CollectionItem } from "@/models/CollectionModels";
+import getLogger from "@/utils/getLogger";
+import { collectionStorage } from "./collectionStorage";
+import getCollectionsFromCloud from "./getCollectionsFromCloud";
+import getCollectionsFromLocal from "./getCollectionsFromLocal";
+import saveCollectionsToLocal from "./saveCollectionsToLocal";
+
+const logger = getLogger("getCollections");
+
+export default async function getCollections(): Promise<[CollectionItem[], CloudStorageIssueType | null]>
+{
+	if (await collectionStorage.disableCloud.getValue() === true)
+		return [await getCollectionsFromLocal(), null];
+
+	const lastUpdatedLocal: number = await collectionStorage.localLastUpdated.getValue();
+	const lastUpdatedSync: number = await collectionStorage.syncLastUpdated.getValue();
+
+	if (lastUpdatedLocal === lastUpdatedSync)
+		return [await getCollectionsFromLocal(), null];
+
+	if (lastUpdatedLocal > lastUpdatedSync)
+		return [await getCollectionsFromLocal(), "merge_conflict"];
+
+	try
+	{
+		const collections: CollectionItem[] = await getCollectionsFromCloud();
+		await saveCollectionsToLocal(collections, lastUpdatedSync);
+
+		return [collections, null];
+	}
+	catch (ex)
+	{
+		logger("Failed to get cloud storage");
+		console.error(ex);
+		trackError("cloud_get_error", ex as Error);
+		return [await getCollectionsFromLocal(), "parse_error"];
+	}
+}
+
+export type CloudStorageIssueType = "parse_error" | "merge_conflict";

features/collectionStorage/utils/getCollectionsFromCloud.ts

@@ -0,0 +1,20 @@
+import { CollectionItem } from "@/models/CollectionModels";
+import { decompress } from "lzutf8";
+import { collectionStorage } from "./collectionStorage";
+import getChunkKeys from "./getChunkKeys";
+import parseCollections from "./parseCollections";
+
+export default async function getCollectionsFromCloud(): Promise<CollectionItem[]>
+{
+	const chunkCount: number = await collectionStorage.chunkCount.getValue();
+
+	if (chunkCount < 1)
+		return [];
+
+	const chunks: Record<string, string> =
+		await browser.storage.sync.get(getChunkKeys(0, chunkCount)) as Record<string, string>;
+
+	const data: string = decompress(Object.values(chunks).join(), { inputEncoding: "Base64" });
+
+	return parseCollections(data);
+}

features/collectionStorage/utils/getCollectionsFromLocal.ts

@@ -0,0 +1,7 @@
+import { CollectionItem } from "@/models/CollectionModels";
+import { collectionStorage } from "./collectionStorage";
+
+export default async function getCollectionsFromLocal(): Promise<CollectionItem[]>
+{
+	return await collectionStorage.localCollections.getValue();
+}

features/collectionStorage/utils/parseCollections.ts

@@ -0,0 +1,80 @@
+import { CollectionItem, GroupItem, TabItem } from "@/models/CollectionModels";
+
+export default function parseCollections(data: string): CollectionItem[]
+{
+	if (!data)
+		return [];
+
+	const collections: CollectionItem[] = [];
+	const lines: string[] = data.split("\n");
+
+	for (const line of lines)
+	{
+		if (line.startsWith("c"))
+		{
+			const collection: CollectionItem = parseCollection(line);
+			collections.push(collection);
+		}
+		else if (line.startsWith("\tg"))
+		{
+			const group: GroupItem = parseGroup(line);
+			collections[collections.length - 1].items.push(group);
+		}
+		else if (line.startsWith("\t\tt"))
+		{
+			const tab: TabItem = parseTab(line);
+
+			const collectionIndex: number = collections.length - 1;
+			const groupIndex: number = collections[collectionIndex].items.length - 1;
+
+			(collections[collectionIndex].items[groupIndex] as GroupItem).items.push(tab);
+		}
+		else if (line.startsWith("\tt"))
+		{
+			const tab: TabItem = parseTab(line);
+			collections[collections.length - 1].items.push(tab);
+		}
+	}
+
+	return collections;
+}
+
+function parseCollection(data: string): CollectionItem
+{
+	return {
+		type: "collection",
+		timestamp: parseInt(data.match(/(?<=^c)\d+/)!.toString()),
+		color: data.match(/(?<=^c\d+\/)[a-z]+/)?.toString() as chrome.tabGroups.ColorEnum,
+		title: data.match(/(?<=^c[\da-z/]*\|).*/)?.toString(),
+		items: []
+	};
+}
+
+function parseGroup(data: string): GroupItem
+{
+	const isPinned: boolean = data.match(/^\tg\/p$/) !== null;
+
+	if (isPinned)
+		return {
+			type: "group",
+			pinned: true,
+			items: []
+		};
+
+	return {
+		type: "group",
+		pinned: false,
+		color: data.match(/(?<=^\tg\/)[a-z]+/)?.toString() as chrome.tabGroups.ColorEnum,
+		title: data.match(/(?<=^\tg\/[a-z]+\|).*$/)?.toString(),
+		items: []
+	};
+}
+
+function parseTab(data: string): TabItem
+{
+	return {
+		type: "tab",
+		url: data.match(/(?<=^(\t){1,2}t\|).*(?=\|)/)!.toString(),
+		title: data.match(/(?<=^(\t){1,2}t\|.*\|).*$/)?.toString()
+	};
+}

features/collectionStorage/utils/resolveConflict.ts

@@ -0,0 +1,43 @@
+import { trackError } from "@/features/analytics";
+import { CollectionItem } from "@/models/CollectionModels";
+import getLogger from "@/utils/getLogger";
+import { collectionStorage } from "./collectionStorage";
+import getCollectionsFromCloud from "./getCollectionsFromCloud";
+import getCollectionsFromLocal from "./getCollectionsFromLocal";
+import saveCollectionsToCloud from "./saveCollectionsToCloud";
+import saveCollectionsToLocal from "./saveCollectionsToLocal";
+
+const logger = getLogger("resolveConflict");
+
+export default function resolveConflict(acceptSource: "local" | "sync"): Promise<void>
+{
+	if (acceptSource === "local")
+		return replaceCloudWithLocal();
+
+	return replaceLocalWithCloud();
+}
+
+async function replaceCloudWithLocal(): Promise<void>
+{
+	const collections: CollectionItem[] = await getCollectionsFromLocal();
+	const lastUpdated: number = await collectionStorage.localLastUpdated.getValue();
+
+	await saveCollectionsToCloud(collections, lastUpdated);
+}
+
+async function replaceLocalWithCloud(): Promise<void>
+{
+	try
+	{
+		const collections: CollectionItem[] = await getCollectionsFromCloud();
+		const lastUpdated: number = await collectionStorage.syncLastUpdated.getValue();
+
+		await saveCollectionsToLocal(collections, lastUpdated);
+	}
+	catch (ex)
+	{
+		logger("Failed to get cloud storage");
+		console.error(ex);
+		trackError("conflict_resolve_with_cloud_error", ex as Error);
+	}
+}

features/collectionStorage/utils/saveCollections.ts

@@ -0,0 +1,47 @@
+import { trackError } from "@/features/analytics";
+import { CollectionItem, GraphicsStorage } from "@/models/CollectionModels";
+import getLogger from "@/utils/getLogger";
+import sendNotification from "@/utils/sendNotification";
+import { collectionStorage } from "./collectionStorage";
+import saveCollectionsToCloud from "./saveCollectionsToCloud";
+import saveCollectionsToLocal from "./saveCollectionsToLocal";
+import updateGraphics from "./updateGraphics";
+
+const logger = getLogger("saveCollections");
+
+export default async function saveCollections(
+	collections: CollectionItem[],
+	updateCloud: boolean = true,
+	graphicsCache?: GraphicsStorage
+): Promise<void>
+{
+	const timestamp: number = Date.now();
+	await saveCollectionsToLocal(collections, timestamp);
+
+	if (updateCloud && await collectionStorage.disableCloud.getValue() !== true)
+		try
+		{
+			await saveCollectionsToCloud(collections, timestamp);
+		}
+		catch (ex)
+		{
+			logger("Failed to save cloud storage");
+			console.error(ex);
+			trackError("cloud_save_error", ex as Error);
+
+			if ((ex as Error).message.includes("MAX_WRITE_OPERATIONS_PER_MINUTE"))
+				await sendNotification({
+					title: i18n.t("notifications.error_quota_exceeded.title"),
+					message: i18n.t("notifications.error_quota_exceeded.message"),
+					icon: "/notification_icons/cloud_error.png"
+				});
+			else
+				await sendNotification({
+					title: i18n.t("notifications.error_storage_full.title"),
+					message: i18n.t("notifications.error_storage_full.message"),
+					icon: "/notification_icons/cloud_error.png"
+				});
+		}
+
+	await updateGraphics(collections, graphicsCache);
+};

features/collectionStorage/utils/saveCollectionsToCloud.ts

@@ -0,0 +1,55 @@
+import { CollectionItem } from "@/models/CollectionModels";
+import { compress } from "lzutf8";
+import { WxtStorageItem } from "wxt/storage";
+import { collectionStorage } from "./collectionStorage";
+import getChunkKeys from "./getChunkKeys";
+import serializeCollections from "./serializeCollections";
+
+export default async function saveCollectionsToCloud(collections: CollectionItem[], timestamp: number): Promise<void>
+{
+	if (!collections || collections.length < 1)
+	{
+		await collectionStorage.chunkCount.setValue(0);
+		await browser.storage.sync.remove(getChunkKeys());
+		return;
+	}
+
+	const data: string = compress(serializeCollections(collections), { outputEncoding: "Base64" });
+	const chunks: string[] = splitIntoChunks(data);
+
+	if (chunks.length > collectionStorage.maxChunkCount)
+		throw new Error("Data is too large to be stored in sync storage.");
+
+	// Since there's a limit for cloud write operations, we need to write all chunks in one go.
+	const newRecords: Record<string, string | number> =
+	{
+		[getStorageKey(collectionStorage.chunkCount)]: chunks.length,
+		[getStorageKey(collectionStorage.syncLastUpdated)]: timestamp
+	};
+
+	for (let i = 0; i < chunks.length; i++)
+		newRecords[`c${i}`] = chunks[i];
+
+	await browser.storage.sync.set(newRecords);
+
+	if (chunks.length < collectionStorage.maxChunkCount)
+		await browser.storage.sync.remove(getChunkKeys(chunks.length));
+}
+
+function splitIntoChunks(data: string): string[]
+{
+	// QUOTA_BYTES_PER_ITEM includes length of key name, length of content and 2 more bytes (for unknown reason).
+	const chunkKey: string = getChunkKeys(collectionStorage.maxChunkCount - 1)[0];
+	const chunkSize = (chrome.storage.sync.QUOTA_BYTES_PER_ITEM ?? 8192) - chunkKey.length - 2;
+	const chunks: string[] = [];
+
+	for (let i = 0; i < data.length; i += chunkSize)
+		chunks.push(data.slice(i, i + chunkSize));
+
+	return chunks;
+}
+
+function getStorageKey(storageItem: WxtStorageItem<any, any>): string
+{
+	return storageItem.key.split(":")[1];
+}

features/collectionStorage/utils/saveCollectionsToLocal.ts

@@ -0,0 +1,9 @@
+import { CollectionItem } from "@/models/CollectionModels";
+import { collectionStorage } from "./collectionStorage";
+
+export default async function saveCollectionsToLocal(collections: CollectionItem[], timestamp: number): Promise<void>
+{
+	await collectionStorage.localCollections.setValue(collections);
+	await collectionStorage.count.setValue(collections.length);
+	await collectionStorage.localLastUpdated.setValue(timestamp);
+}

features/collectionStorage/utils/serializeCollections.ts

@@ -0,0 +1,74 @@
+import { CollectionItem, GroupItem, TabItem } from "@/models/CollectionModels";
+
+export default function serializeCollections(collections: CollectionItem[]): string
+{
+	let data: string = "";
+
+	for (const collection of collections)
+	{
+		data += getCollectionString(collection);
+
+		for (const item of collection.items)
+		{
+			if (item.type === "group")
+			{
+				data += getGroupString(item);
+
+				for (const tab of item.items)
+					data += `\t${getTabString(tab)}`;
+			}
+			else if (item.type === "tab")
+				data += getTabString(item);
+		}
+	}
+
+	return data;
+}
+
+function getCollectionString(collection: CollectionItem): string
+{
+	let data: string = `c${collection.timestamp}`;
+
+	if (collection.color)
+		data += `/${collection.color}`;
+
+	if (collection.title)
+		data += `|${collection.title}`;
+
+	data += "\n";
+
+	return data;
+}
+
+function getGroupString(group: GroupItem): string
+{
+	let data: string = "\tg";
+
+	if (group.pinned === true)
+		data += "/p";
+	else
+	{
+		data += `/${group.color}`;
+
+		if (group.title)
+			data += `|${group.title}`;
+	}
+
+	data += "\n";
+
+	return data;
+}
+
+function getTabString(tab: TabItem): string
+{
+	let data: string = "\tt";
+
+	data += `|${tab.url}|`;
+
+	if (tab.title)
+		data += tab.title;
+
+	data += "\n";
+
+	return data;
+}

features/collectionStorage/utils/setCloudStorage.ts

@@ -0,0 +1,20 @@
+import { sendMessage } from "@/utils/messaging";
+import { collectionStorage } from "./collectionStorage";
+import saveCollectionsToCloud from "./saveCollectionsToCloud";
+
+export default async function setCloudStorage(enable: boolean): Promise<void>
+{
+	if (enable)
+	{
+		await collectionStorage.disableCloud.setValue(false);
+		const collections = await collectionStorage.localCollections.getValue();
+		const lastUpdated = await collectionStorage.localLastUpdated.getValue();
+		await saveCollectionsToCloud(collections, lastUpdated);
+	}
+	else
+	{
+		await collectionStorage.disableCloud.setValue(true);
+		await saveCollectionsToCloud([], 0);
+		await sendMessage("refreshCollections", undefined);
+	}
+}

features/collectionStorage/utils/updateGraphics.ts

@@ -0,0 +1,54 @@
+import { CollectionItem, GraphicsItem, GraphicsStorage } from "@/models/CollectionModels";
+import { sendMessage } from "@/utils/messaging";
+import { collectionStorage } from "./collectionStorage";
+
+export default async function updateGraphics(
+	collections: CollectionItem[],
+	graphicsCache?: GraphicsStorage
+): Promise<void>
+{
+	const localGraphics: GraphicsStorage = await collectionStorage.graphics.getValue();
+	const tempGraphics: GraphicsStorage = graphicsCache || await sendMessage("getGraphicsCache", undefined);
+
+	function getGraphics(url: string): GraphicsItem | null
+	{
+		const preview = tempGraphics[url]?.preview ?? localGraphics[url]?.preview;
+		const capture = tempGraphics[url]?.capture ?? localGraphics[url]?.capture;
+		const icon = tempGraphics[url]?.icon ?? localGraphics[url]?.icon;
+
+		const graphics: GraphicsItem = {};
+
+		if (preview)
+			graphics.preview = preview;
+		if (icon)
+			graphics.icon = icon;
+		if (capture)
+			graphics.capture = capture;
+
+		return preview || icon ? graphics : null;
+	}
+
+	const newGraphics: GraphicsStorage = {};
+
+	for (const collection of collections)
+		for (const item of collection.items)
+		{
+			if (item.type === "group")
+				for (const tab of item.items)
+				{
+					const graphics = getGraphics(tab.url);
+
+					if (graphics)
+						newGraphics[tab.url] = graphics;
+				}
+			else
+			{
+				const graphics = getGraphics(item.url);
+
+				if (graphics)
+					newGraphics[item.url] = graphics;
+			}
+		}
+
+	await collectionStorage.graphics.setValue(newGraphics);
+}