-
+
{{range .ForumChannels}}
diff --git a/resources/templates/header.gohtml b/resources/templates/header.gohtml
index 9cddc6b..43ae1c9 100644
--- a/resources/templates/header.gohtml
+++ b/resources/templates/header.gohtml
@@ -2,9 +2,7 @@
+
-
-
-
diff --git a/resources/templates/index.gohtml b/resources/templates/index.gohtml
index 96e99b1..5b80336 100644
--- a/resources/templates/index.gohtml
+++ b/resources/templates/index.gohtml
@@ -1,7 +1,11 @@
{{ template "header.gohtml" }}
dforum
-
this website is connected to a discord bot that you can invite to any of your servers, and it will display the forums in a fashion that Google and other search engines can index and show (though, we need somebody good at SEO to help with that). you can't interact with this forum but hopefully this will at least let these forum channels show up to google
+
+
+
+
+
this website will display the forums in any server you invite the corresponding bot to in a such a way that Google and other search engines can index and show them. we hope to host as many servers as we can so that we can usher in a new era of online forums, without locking this information behind a service that you have to sign up for (which also has a less then adequate search feature).
the site is also (hopefully) simple enough that you can print the contents of it to another site, and thus if you want your own url for this stuff you can simply include the site from your own site, via whatever language you plan on using.
diff --git a/resources/templates/nav.gohtml b/resources/templates/nav.gohtml
new file mode 100644
index 0000000..e5b792e
--- /dev/null
+++ b/resources/templates/nav.gohtml
@@ -0,0 +1,18 @@
+{{define "navbuttons"}}
+
+{{if .ShowPrev}}
+ {{template "prev_post" .PrevPage}}
+{{end}}
+{{if .ShowNext}}
+ {{template "next_post" .NextPage}}
+{{end}}
+
+{{end}}
+
+{{define "prev_post"}}
+ Previous
+{{end}}
+
+{{define "next_post"}}
+ Next
+{{end}}
\ No newline at end of file
diff --git a/resources/templates/post.gohtml b/resources/templates/post.gohtml
index 196ea80..c5bfcdf 100644
--- a/resources/templates/post.gohtml
+++ b/resources/templates/post.gohtml
@@ -5,7 +5,9 @@
{{$title := print .Post.Name " - " .Guild.Name}}
+dforum
- Posted {{$firstMsg.ID.Time.Format "January 2, 2006 3:04 PM"}}
+ Posted {{$firstMsg.ID.Time.Format "January 2, 2006 3:04 PM"}} - {{.ID}}
{{range .Messages}}
{{.RenderedContent}}
{{range .MediaPreviews}}
@@ -88,4 +99,12 @@
{{end}}
-{{ template "footer.gohtml" .}}
+
+{{if .PrevID }}
+Previous
+{{end}}
+{{if .NextID }}
+Next
+{{end}}
+
+{{ template "footer.gohtml" .}}
\ No newline at end of file
diff --git a/server.go b/server.go
index 1d5b36d..fc4660c 100644
--- a/server.go
+++ b/server.go
@@ -2,13 +2,13 @@ package main
import (
"bytes"
- "context"
"errors"
"fmt"
"hash/crc32"
"io/fs"
"net/http"
"sort"
+ "strconv"
"sync"
"time"
@@ -20,6 +20,9 @@ import (
"github.com/go-chi/chi/v5/middleware"
)
+// limit for items that show up in post and message listing.
+const paginationLimit = 25
+
type server struct {
r *chi.Mux
@@ -66,6 +69,8 @@ func newServer(st *state.State, fsys fs.FS, config config) (*server, error) {
r := chi.NewRouter()
srv.r = r
r.Use(middleware.Logger)
+ r.Mount("/debug", middleware.Profiler())
+
getHead(r, `/sitemap.xml`, srv.getSitemap)
getHead(r, "/", srv.getIndex)
r.Route("/{guildID:\\d+}", func(r chi.Router) {
@@ -80,6 +85,7 @@ func newServer(st *state.State, fsys fs.FS, config config) (*server, error) {
getHead(r, "/privacy", srv.PrivacyPage)
getHead(r, "/tos", srv.TOSPage)
getHead(r, "/static/*", http.FileServer(http.FS(fsys)).ServeHTTP)
+
r.NotFound(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
displayErr(w, http.StatusNotFound, nil)
}))
@@ -164,6 +170,14 @@ type ForumChannel struct {
}
func (s *server) getGuild(w http.ResponseWriter, r *http.Request) {
+ ch := make(chan int)
+ go func() {
+ s.getGuildSync(w, r)
+ ch <- 1
+ }()
+ <-ch
+}
+func (s *server) getGuildSync(w http.ResponseWriter, r *http.Request) {
guild, ok := s.guildFromReq(w, r)
if !ok {
return
@@ -251,6 +265,14 @@ func (p Post) IsPinned() bool {
}
func (s *server) getForum(w http.ResponseWriter, r *http.Request) {
+ ch := make(chan int)
+ go func() {
+ s.getForumSync(w, r)
+ ch <- 1
+ }()
+ <-ch
+}
+func (s *server) getForumSync(w http.ResponseWriter, r *http.Request) {
guild, ok := s.guildFromReq(w, r)
if !ok {
return
@@ -266,23 +288,65 @@ func (s *server) getForum(w http.ResponseWriter, r *http.Request) {
return
}
+ // query parameters
+ tagFilterString := r.URL.Query().Get("tag-filter")
+ afterString := r.URL.Query().Get("after")
+ var tagFilter int
+ var after discord.ChannelID
+ if tagFilterString != "" {
+ tagFilter, err = strconv.Atoi(tagFilterString)
+ if err != nil {
+ displayErr(w, http.StatusInternalServerError,
+ fmt.Errorf("parsing tag filter: %s", err))
+ return
+ }
+ } else {
+ tagFilter = -1
+ }
+ if afterString != "" {
+ after_, err := discord.ParseSnowflake(afterString)
+ if err != nil {
+ displayErr(w, http.StatusInternalServerError,
+ fmt.Errorf("parsing after number: %s", err))
+ return
+ }
+ after = discord.ChannelID(after_)
+ } else {
+ after = 0
+ }
+
ctx := struct {
- Guild *discord.Guild
- Forum *discord.Channel
- Posts []Post
- URL string
- }{guild, forum, nil, s.URL}
+ Guild *discord.Guild
+ Forum *discord.Channel
+ Posts []Post
+ URL string
+ TagFilter int
+ ShowPrev bool
+ PrevPage int
+ ShowNext bool
+ NextPage int
+ }{Guild: guild, Forum: forum, Posts: nil, URL: s.URL,
+ TagFilter: tagFilter}
channels, err := s.discord.Cabinet.Channels(guild.ID)
if err != nil {
displayErr(w, http.StatusInternalServerError,
fmt.Errorf("fetching guild threads: %w", err))
return
}
+ // due to the way discord handles threads in forums (they're basically more
+ // channels), we need to go through the entire server and filter by
+ // parent channel.
+ // trying to limit how many results within this for loop leads to
+ // unstable results.
for _, thread := range channels {
if thread.ParentID != forum.ID ||
thread.Type != discord.GuildPublicThread {
continue
}
+ show := true
+ if tagFilter != -1 {
+ show = false
+ }
post := Post{Channel: thread}
for _, tag := range thread.AppliedTags {
for _, availtag := range forum.AvailableTags {
@@ -290,19 +354,72 @@ func (s *server) getForum(w http.ResponseWriter, r *http.Request) {
post.Tags = append(post.Tags, availtag)
}
}
+ if tag == discord.TagID(tagFilter) {
+ show = true
+ }
+ }
+ if show {
+ ctx.Posts = append(ctx.Posts, post)
}
- ctx.Posts = append(ctx.Posts, post)
}
+
sort.SliceStable(ctx.Posts, func(i, j int) bool {
if ctx.Posts[i].Flags^ctx.Posts[j].Flags&discord.PinnedThread != 0 {
return ctx.Posts[i].Flags&discord.PinnedThread != 0
}
return ctx.Posts[i].LastMessageID.Time().After(ctx.Posts[j].LastMessageID.Time())
})
+
+ // so limit the posts right here instead.
+ postsFiltered := make([]Post, 0)
+ postNum := 0
+ show := false
+ lastIgnoredPost := 0
+ if after == 0 {
+ show = true
+ }
+ for i, post := range ctx.Posts {
+ if post.ID == after {
+ show = true
+ }
+ if show {
+ if postNum >= paginationLimit {
+ ctx.NextPage = int(post.ID)
+ ctx.ShowNext = true
+ break
+ }
+ postsFiltered = append(postsFiltered, post)
+ postNum++
+ } else {
+ lastIgnoredPost = i + 1
+ continue
+ }
+ }
+ // if there's a post behind us, have a "previous" button for that
+ if lastIgnoredPost-paginationLimit >= 0 {
+ post := ctx.Posts[lastIgnoredPost-paginationLimit]
+ ctx.ShowPrev = true
+ ctx.PrevPage = int(post.ID)
+ }
+ ctx.Posts = postsFiltered
+
s.executeTemplate(w, r, "forum.gohtml", ctx)
}
+// we want to limit the number of threads that can be dedicated to the post page.
+var postPageSemaphore = NewSemaphore(100)
+
func (s *server) getPost(w http.ResponseWriter, r *http.Request) {
+ postPageSemaphore.AcquireRead()
+ defer postPageSemaphore.Release()
+ ch := make(chan int)
+ go func() {
+ s.getPostSync(w, r)
+ ch <- 1
+ }()
+ <-ch
+}
+func (s *server) getPostSync(w http.ResponseWriter, r *http.Request) {
guild, ok := s.guildFromReq(w, r)
if !ok {
return
@@ -320,18 +437,36 @@ func (s *server) getPost(w http.ResponseWriter, r *http.Request) {
Forum *discord.Channel
Post *discord.Channel
MessageGroups []MessageGroup
+ NextID discord.MessageID
+ PrevID discord.MessageID
URL string
- }{guild, forum, post, nil, s.URL}
+ }{Guild: guild, Forum: forum, Post: post, URL: s.URL}
+
+ cur := 0
+ asc := true
+ curstr := r.URL.Query().Get("cur")
+ if len(curstr) >= 2 {
+ asc = curstr[0] == 'a'
+ cur, _ = strconv.Atoi(curstr[1:])
+ } else {
+ asc = true
+ cur = 0
+ }
- msgs, err := s.messageCache.Messages(post.ID)
+ var msgs []discord.Message
+ var err error
+ if asc {
+ msgs, err, ctx.PrevID, ctx.NextID = s.messageCache.MessagesAfter(post.ID, uint(cur), paginationLimit)
+ } else {
+ msgs, err, ctx.PrevID, ctx.NextID = s.messageCache.MessagesBefore(post.ID, uint(cur), paginationLimit)
+ }
if err != nil {
displayErr(w, http.StatusInternalServerError,
fmt.Errorf("fetching post's messages: %w", err))
return
}
- timeout, cancel := context.WithTimeout(r.Context(), 5*time.Second)
- err = s.ensureMembers(timeout, *post, msgs)
- cancel()
+
+ err = s.ensureMembers(r.Context(), *post, msgs)
if err != nil {
displayErr(w, http.StatusInternalServerError,
fmt.Errorf("fetching post's members: %w", err))
diff --git a/sitemap.go b/sitemap.go
index 6a5dcdb..f1e8b43 100644
--- a/sitemap.go
+++ b/sitemap.go
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"net/http"
+ "sync"
"time"
"github.com/diamondburned/arikawa/v3/discord"
@@ -59,6 +60,23 @@ var XMLURLSetStart = xml.StartElement{
var XMLURLSetEnd = XMLURLSetStart.End()
+type URLMap struct {
+ urls map[int]URL
+ sync.RWMutex
+}
+
+func NewURLMap() URLMap {
+ return URLMap{
+ urls: make(map[int]URL),
+ }
+}
+
+func (u *URLMap) Push(url URL) {
+ u.Lock()
+ defer u.Unlock()
+ u.urls[len(u.urls)] = url
+}
+
func (s *server) writeSitemap(w io.Writer) error {
if _, err := io.WriteString(w, xml.Header); err != nil {
return err
@@ -69,81 +87,151 @@ func (s *server) writeSitemap(w io.Writer) error {
return err
}
guilds, _ := s.discord.Cabinet.Guilds()
- for _, guild := range guilds {
- if err = enc.Encode(URL{
- Location: fmt.Sprintf("%s/%s", s.URL, guild.ID),
- }); err != nil {
- return err
- }
+ forumURLs, chanURLs, postURLs := NewURLMap(), NewURLMap(), NewURLMap()
+ var wg1 sync.WaitGroup
+ wg1.Add(len(guilds))
+ errCh1 := make(chan error)
+ waitCh1 := make(chan bool)
+ // level 1 wrapper
+ go func() {
+ for _, guild := range guilds {
+ // level 1
+ go func(guild discord.Guild) {
+ defer wg1.Done()
+ forumURLs.Push(URL{
+ Location: fmt.Sprintf("%s/%s", s.URL, guild.ID),
+ })
- channels, err := s.discord.Cabinet.Channels(guild.ID)
- if err != nil {
- return err
+ channels, err := s.discord.Cabinet.Channels(guild.ID)
+ if err != nil {
+ errCh1 <- err
+ }
+
+ var wg2 sync.WaitGroup
+ wg2.Add(len(channels))
+ // we first need to go through these channels and ensure their messages
+ // are cached
+ errCh2 := make(chan error)
+ waitCh2 := make(chan bool)
+ // level 2 wrapper
+ go func() {
+ // level 2
+ for _, channel := range channels {
+ go func(channel discord.Channel) {
+ defer wg2.Done()
+ if channel.Type != discord.GuildForum {
+ return
+ }
+ me, _ := s.discord.Cabinet.Me()
+ perms, err := s.discord.Permissions(channel.ID, me.ID)
+ if err != nil {
+ errCh2 <- fmt.Errorf("fetching channel permissions for %s: %s", channel.Name, err)
+ }
+ if !perms.Has(0 |
+ discord.PermissionReadMessageHistory |
+ discord.PermissionViewChannel) {
+ return
+ }
+ err = s.ensureArchivedThreads(channel.ID)
+ if err != nil {
+ errCh2 <- fmt.Errorf("fetching archived threads for %s: %s", channel.Name, err)
+ }
+ }(channel)
+ }
+ wg2.Wait()
+ waitCh2 <- true
+ }()
+ select {
+ case e := <-errCh2:
+ {
+ errCh1 <- e
+ }
+ case <-waitCh2:
+ {
+ }
+ }
+
+ // and then go through it again.
+ channels, _ = s.discord.Cabinet.Channels(guild.ID)
+ var forums []discord.Channel
+ for _, channel := range channels {
+ if channel.Type != discord.GuildForum {
+ continue
+ }
+ forums = append(forums, channel)
+ }
+ for _, forum := range forums {
+ chanURLs.Push(URL{
+ Location: fmt.Sprintf("%s/%s/%s", s.URL, guild.ID, forum.ID),
+ })
+ var posts []discord.Channel
+ for _, t := range channels {
+ if t.ParentID == forum.ID &&
+ t.Type == discord.GuildPublicThread {
+ posts = append(posts, t)
+ }
+ }
+ if len(posts) < 1 {
+ continue
+ }
+ var wg3 sync.WaitGroup
+ wg3.Add(len(posts))
+ errCh3 := make(chan error)
+ waitCh3 := make(chan bool)
+ // level 3 wrapper
+ go func() {
+ for _, post := range posts {
+ // level 3
+ go func(post discord.Channel) {
+ defer wg3.Done()
+ postURLs.Push(URL{
+ Location: fmt.Sprintf("%s/%s/%s/%s", s.URL, guild.ID, forum.ID, post.ID),
+ })
+ }(post)
+ }
+ wg3.Wait()
+ waitCh3 <- true
+ }()
+
+ select {
+ case e := <-errCh3:
+ {
+ errCh2 <- e
+ }
+ case <-waitCh3:
+ {
+ }
+ }
+
+ }
+
+ }(guild)
}
- // we first need to go through these channels and ensure their messages
- // are cached
- for _, channel := range channels {
- if channel.Type != discord.GuildForum {
- continue
- }
- me, _ := s.discord.Cabinet.Me()
- perms, err := s.discord.Permissions(channel.ID, me.ID)
- if err != nil {
- return fmt.Errorf("fetching channel permissions for %s: %s", channel.Name, err)
- }
- if !perms.Has(0 |
- discord.PermissionReadMessageHistory |
- discord.PermissionViewChannel) {
- continue
- }
- err = s.ensureArchivedThreads(channel.ID)
- if err != nil {
- return fmt.Errorf("fetching archived threads for %s: %s", channel.Name, err)
- }
+ wg1.Wait()
+ waitCh1 <- true
+ }()
+ select {
+ case e := <-errCh1:
+ {
+ return e
}
- // and then go through it again.
- channels, _ = s.discord.Cabinet.Channels(guild.ID)
- var forums []discord.Channel
- for _, channel := range channels {
- if channel.Type != discord.GuildForum {
- continue
- }
- me, _ := s.discord.Cabinet.Me()
- perms, err := s.discord.Permissions(channel.ID, me.ID)
- if err != nil {
- return fmt.Errorf("fetching channel permissions for %s: %s", channel.Name, err)
- }
- if !perms.Has(0 |
- discord.PermissionReadMessageHistory |
- discord.PermissionViewChannel) {
- continue
- }
- err = s.ensureArchivedThreads(channel.ID)
- if err != nil {
- return fmt.Errorf("fetching archived threads for %s: %s", channel.Name, err)
- }
- forums = append(forums, channel)
+ case <-waitCh1:
+ {
}
- for _, forum := range forums {
- if err = enc.Encode(URL{
- Location: fmt.Sprintf("%s/%s/%s", s.URL, guild.ID, forum.ID),
- }); err != nil {
- return err
- }
- var posts []discord.Channel
- for _, t := range channels {
- if t.ParentID == forum.ID &&
- t.Type == discord.GuildPublicThread {
- posts = append(posts, t)
- }
- }
- for _, post := range posts {
- if err = enc.Encode(URL{
- Location: fmt.Sprintf("%s/%s/%s/%s", s.URL, guild.ID, forum.ID, post.ID),
- }); err != nil {
- return err
- }
- }
+ }
+ for _, url := range forumURLs.urls {
+ if err = enc.Encode(url); err != nil {
+ return err
+ }
+ }
+ for _, url := range chanURLs.urls {
+ if err = enc.Encode(url); err != nil {
+ return err
+ }
+ }
+ for _, url := range postURLs.urls {
+ if err = enc.Encode(url); err != nil {
+ return err
}
}
if err = enc.EncodeToken(XMLURLSetEnd); err != nil {