feat(#5): create groups and import feeds into them, based on opml

Author: 0x2E

api/group.go

@@ -33,11 +33,12 @@ func (f groupAPI) Create(c echo.Context) error {
 		return err
 	}
 
-	if err := f.srv.Create(c.Request().Context(), &req); err != nil {
+	resp, err := f.srv.Create(c.Request().Context(), &req)
+	if err != nil {
 		return err
 	}
 
-	return c.NoContent(http.StatusCreated)
+	return c.JSON(http.StatusCreated, resp)
 }
 
 func (f groupAPI) Update(c echo.Context) error {

frontend/src/lib/api/group.ts

@@ -7,11 +7,13 @@ export async function allGroups() {
 }
 
 export async function createGroup(name: string) {
-	return await api.post('groups', {
-		json: {
-			name: name
-		}
-	});
+	return await api
+		.post('groups', {
+			json: {
+				name: name
+			}
+		})
+		.json<{ id: number }>();
 }
 
 export async function updateGroup(id: number, name: string) {

frontend/src/lib/opml.ts

@@ -1,18 +1,55 @@
 export function parse(content: string) {
-	const feeds: { name: string; link: string }[] = [];
+	type feedT = {
+		name: string;
+		link: string;
+	};
+	type groupT = {
+		name: string;
+		feeds: feedT[];
+	};
+	const groups = new Map<string, groupT>();
+	const defaultGroup = { name: 'Default', feeds: [] };
+	groups.set('Default', defaultGroup);
+
+	function dfs(parentGroup: groupT | null, node: Element) {
+		if (node.tagName !== 'outline') {
+			return;
+		}
+		if (node.getAttribute('type')?.toLowerCase() == 'rss') {
+			if (!parentGroup) {
+				parentGroup = defaultGroup;
+			}
+			parentGroup.feeds.push({
+				name: node.getAttribute('title') || node.getAttribute('text') || '',
+				link: node.getAttribute('xmlUrl') || node.getAttribute('htmlUrl') || ''
+			});
+			return;
+		}
+		if (!node.children.length) {
+			return;
+		}
+		const nodeName = node.getAttribute('text') || node.getAttribute('title') || '';
+		const name = parentGroup ? parentGroup.name + '/' + nodeName : nodeName;
+		let curGroup = groups.get(name);
+		if (!curGroup) {
+			curGroup = { name: name, feeds: [] };
+			groups.set(name, curGroup);
+		}
+		for (const n of node.children) {
+			dfs(curGroup, n);
+		}
+	}
+
 	const xmlDoc = new DOMParser().parseFromString(content, 'text/xml');
-	const outlines = xmlDoc.getElementsByTagName('outline');
-
-	for (let i = 0; i < outlines.length; i++) {
-		const outline = outlines.item(i);
-		if (!outline) continue;
-		const link = outline.getAttribute('xmlUrl') || outline.getAttribute('htmlUrl') || '';
-		if (!link) continue;
-		const name = outline.getAttribute('title') || outline.getAttribute('text') || '';
-		feeds.push({ name, link });
+	const body = xmlDoc.getElementsByTagName('body')[0];
+	if (!body) {
+		return [];
+	}
+	for (const n of body.children) {
+		dfs(null, n);
 	}
 
-	return feeds;
+	return Array.from(groups.values());
 }
 
 export function dump(data: { name: string; feeds: { name: string; link: string }[] }[]) {

frontend/src/routes/(authed)/feeds/ActionAdd.svelte

@@ -106,6 +106,9 @@
 						}}
 						required
 					/>
+					<p class="text-sm text-muted-foreground">
+						The existing feed with the same link will be override.
+					</p>
 				</div>
 
 				<div>
@@ -121,11 +124,6 @@
 						}}
 						required
 					/>
-					{#if formData.name}
-						<p class="text-sm text-muted-foreground">
-							The existing feed with the same link will be renamed as <b>{formData.name}</b>.
-						</p>
-					{/if}
 				</div>
 
 				<div>

frontend/src/routes/(authed)/feeds/ActionOPML.svelte

@@ -1,50 +1,71 @@
 <script lang="ts">
 	import type { groupFeeds } from './+page';
 	import * as Sheet from '$lib/components/ui/sheet';
-	import * as Select from '$lib/components/ui/select';
 	import * as Tabs from '$lib/components/ui/tabs';
 	import { Button } from '$lib/components/ui/button';
 	import { Label } from '$lib/components/ui/label';
 	import { createFeed } from '$lib/api/feed';
 	import { toast } from 'svelte-sonner';
 	import { invalidateAll } from '$app/navigation';
 	import { dump, parse } from '$lib/opml';
+	import { FolderIcon } from 'lucide-svelte';
+	import { createGroup } from '$lib/api/group';
 
 	export let groups: groupFeeds[];
 	export let open: boolean;
 
 	let uploadedOpmls: FileList;
 	$: parseOPML(uploadedOpmls);
-	let opmlGroup = { id: groups[0].id, name: groups[0].name };
-	let parsedOpmlFeeds: { name: string; link: string }[] = [];
+	let parsedGroupFeeds: { name: string; feeds: { name: string; link: string }[] }[] = [];
+	let importing = false;
+
 	$: {
 		if (!open) {
-			parsedOpmlFeeds = [];
+			parsedGroupFeeds = [];
 		}
 	}
 
 	function parseOPML(opmls: FileList) {
 		if (!opmls) return;
+
 		const reader = new FileReader();
 		reader.onload = (f) => {
 			const content = f.target?.result?.toString();
 			if (!content) {
 				toast.error('Failed to load file content');
 				return;
 			}
-			parsedOpmlFeeds = parse(content);
-			console.log(parsedOpmlFeeds);
+			parsedGroupFeeds = parse(content).filter((v) => v.feeds.length > 0);
+			console.log(parsedGroupFeeds);
 		};
 		reader.readAsText(opmls[0]);
 	}
 
 	async function handleImportFeeds() {
-		try {
-			await createFeed({ group_id: opmlGroup.id, feeds: parsedOpmlFeeds });
-			toast.success('Feeds have been imported. Refreshing is running in the background');
-		} catch (e) {
-			toast.error((e as Error).message);
+		importing = true;
+		let success = 0;
+		const existingGroups = groups.map((v) => {
+			return { id: v.id, name: v.name };
+		});
+		for (const g of parsedGroupFeeds) {
+			try {
+				let groupID = existingGroups.find((v) => v.name === g.name)?.id;
+				if (groupID === undefined) {
+					groupID = (await createGroup(g.name)).id;
+					toast.success(`Created group ${g.name}`);
+				}
+				await createFeed({ group_id: groupID, feeds: g.feeds });
+				toast.success(`Imported into group ${g.name}`);
+				success++;
+			} catch (e) {
+				toast.error(`Failed to import group ${g.name}, error: ${(e as Error).message}`);
+				break;
+			}
+		}
+		if (success === parsedGroupFeeds.length) {
+			toast.success('All feeds have been imported. Refreshing is running in the background');
 		}
+		importing = false;
 		invalidateAll();
 	}
 
@@ -69,7 +90,7 @@
 </script>
 
 <Sheet.Root bind:open>
-	<Sheet.Content class="w-full md:w-auto">
+	<Sheet.Content class="w-full md:max-w-[700px] overflow-y-auto">
 		<Sheet.Header>
 			<Sheet.Title>Import or Export Feeds</Sheet.Title>
 			<Sheet.Description>
@@ -87,25 +108,6 @@
 			</Tabs.List>
 			<Tabs.Content value="import">
 				<form class="space-y-2" on:submit|preventDefault={handleImportFeeds}>
-					<div>
-						<Label for="group">Group</Label>
-						<Select.Root
-							disabled={groups.length < 2}
-							items={groups.map((v) => {
-								return { value: v.id, label: v.name };
-							})}
-							onSelectedChange={(v) => v && (opmlGroup.id = v.value)}
-						>
-							<Select.Trigger>
-								<Select.Value placeholder={opmlGroup.name} />
-							</Select.Trigger>
-							<Select.Content>
-								{#each groups as g}
-									<Select.Item value={g.id}>{g.name}</Select.Item>
-								{/each}
-							</Select.Content>
-						</Select.Root>
-					</div>
 					<div>
 						<Label for="feed_file">File</Label>
 						<input
@@ -117,21 +119,34 @@
 							class="flex h-10 w-full rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50"
 						/>
 					</div>
-					{#if parsedOpmlFeeds.length > 0}
+					{#if parsedGroupFeeds.length > 0}
 						<div>
-							<p class="text-sm text-muted-foreground">Parsed out {parsedOpmlFeeds.length} feeds</p>
+							<p class="text-sm text-green-700">Parsed successfully.</p>
 							<div
-								class="max-h-[200px] overflow-scroll p-2 rounded-md border bg-muted text-muted-foreground text-nowrap"
+								class="p-2 rounded-md border bg-muted/40 text-muted-foreground text-nowrap overflow-x-auto"
 							>
-								<ul>
-									{#each parsedOpmlFeeds as feed, index}
-										<li>{index + 1}. <b>{feed.name}</b> {feed.link}</li>
-									{/each}
-								</ul>
+								{#each parsedGroupFeeds as group}
+									<div class="flex flex-row items-center gap-1">
+										<FolderIcon size={14} />{group.name}
+									</div>
+									<ul class="list-inside list-decimal ml-[2ch] [&:not(:last-child)]:mb-2">
+										{#each group.feeds as feed}
+											<li>{feed.name}, {feed.link}</li>
+										{/each}
+									</ul>
+								{/each}
 							</div>
 						</div>
 					{/if}
-					<Button type="submit">Import</Button>
+					<div class="text-sm text-secondary-foreground">
+						<p>Note:</p>
+						<p>
+							1. Feeds will be imported into the corresponding group, which will be created
+							automatically if it does not exist.
+						</p>
+						<p>2. The existing feed with the same link will be override.</p>
+					</div>
+					<Button type="submit" disabled={importing}>Import</Button>
 				</form>
 			</Tabs.Content>
 			<Tabs.Content value="export">

frontend/src/routes/+layout.svelte

@@ -11,7 +11,7 @@
 </svelte:head>
 
 <ModeWatcher defaultMode="system" />
-<Toaster position="top-right" richColors closeButton />
+<Toaster position="top-right" richColors closeButton visibleToasts={10} />
 
 <!-- h-screen does not work properly on mobile. Use calc(100dvh) instead. 
      https://stackoverflow.com/a/76120728/12812480 -->

model/feed.go

@@ -16,10 +16,10 @@ type Feed struct {
 	ID        uint `gorm:"primarykey"`
 	CreatedAt time.Time
 	UpdatedAt time.Time
-	DeletedAt soft_delete.DeletedAt
+	DeletedAt soft_delete.DeletedAt `gorm:"uniqueIndex:idx_link"`
 
 	Name *string `gorm:"name;not null"`
-	Link *string `gorm:"link;not null"` // FIX: unique index?
+	Link *string `gorm:"link;not null;uniqueIndex:idx_link"`
 	// LastBuild is the last time the content of the feed changed
 	LastBuild *time.Time `gorm:"last_build"`
 	// Failure is the reason of failure. If it is not null or empty, the fetch processor

repo/feed.go

@@ -6,6 +6,7 @@ import (
 	"github.com/0x2e/fusion/model"
 
 	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
 )
 
 func NewFeed(db *gorm.DB) *Feed {
@@ -47,7 +48,10 @@ func (f Feed) Get(id uint) (*model.Feed, error) {
 }
 
 func (f Feed) Create(data []*model.Feed) error {
-	return f.db.Create(data).Error
+	return f.db.Clauses(clause.OnConflict{
+		Columns:   []clause.Column{{Name: "link"}, {Name: "deleted_at"}},
+		DoUpdates: clause.AssignmentColumns([]string{"name", "link"}),
+	}).Create(data).Error
 }
 
 func (f Feed) Update(id uint, feed *model.Feed) error {

repo/repo.go

@@ -2,6 +2,7 @@ package repo
 
 import (
 	"errors"
+	"log"
 
 	"github.com/0x2e/fusion/conf"
 	"github.com/0x2e/fusion/model"
@@ -27,6 +28,48 @@ func Init() {
 }
 
 func migrage() {
+	// The verison after v0.8.7 will add a unique index to Feed.Link.
+	// We must delete any duplicate feeds before AutoMigrate applies the
+	// new unique constraint.
+	err := DB.Transaction(func(tx *gorm.DB) error {
+		// query duplicate feeds
+		dupFeeds := make([]model.Feed, 0)
+		err := tx.Model(&model.Feed{}).Where(
+			"link IN (?)",
+			tx.Model(&model.Feed{}).Select("link").Group("link").
+				Having("count(link) > 1"),
+		).Order("link, id").Find(&dupFeeds).Error
+		if err != nil {
+			return err
+		}
+
+		// filter out feeds that will be deleted.
+		// we've queried with order, so the first one is the one we should keep.
+		distinct := map[string]uint{}
+		deleteIDs := make([]uint, 0, len(dupFeeds))
+		for _, f := range dupFeeds {
+			if _, ok := distinct[*f.Link]; !ok {
+				distinct[*f.Link] = f.ID
+				continue
+			}
+			deleteIDs = append(deleteIDs, f.ID)
+			log.Println("delete duplicate feed: ", f.ID, *f.Name, *f.Link)
+		}
+
+		if len(deleteIDs) > 0 {
+			// **hard** delete duplicate feeds and their items
+			err = tx.Where("id IN ?", deleteIDs).Unscoped().Delete(&model.Feed{}).Error
+			if err != nil {
+				return err
+			}
+			return tx.Where("feed_id IN ?", deleteIDs).Unscoped().Delete(&model.Item{}).Error
+		}
+		return nil
+	})
+	if err != nil {
+		panic(err)
+	}
+
 	// FIX: gorm not auto drop index and change 'not null'
 	if err := DB.AutoMigrate(&model.Feed{}, &model.Group{}, &model.Item{}); err != nil {
 		panic(err)

server/feed.go

@@ -93,9 +93,6 @@ func (f Feed) Create(ctx context.Context, req *ReqFeedCreate) error {
 	}
 
 	if err := f.repo.Create(feeds); err != nil {
-		if errors.Is(err, repo.ErrDuplicatedKey) {
-			err = NewBizError(err, http.StatusBadRequest, "link is not allowed to be the same as other feeds")
-		}
 		return err
 	}