Several visual changes and pagnation/QOL features (#76)

* dark mode

* more visual changes

* border-radius on nav in dark mode only, by acident

* tag sorting

* pagenation

* pagnation works properly now

* favicon

* make prev/next buttons consistent with rest of site

* guild image. dropdown looks good on mobile now

* revised the index page a bit

* show it -> show them

* fixed reaction displays finally holy

* made reactions look better in general

* value based pagination

* Accidentally renamed some functions when i shouldn't have

* int -> snowflake ig

* buttons to links

* dark mode ubtton

* nicer padding on mobile

* rem set to 0.5

* more css tweaks

* pagination on post page

* account for pagination in sitemap

* account for pagination in sitemap

* oops the pagination on the sitemap was messed up

* accidentally broke message grouping

* no more timeout

* we don't need mutexes

* accidentally left sitemap-timings.txt in

* typo

* only get all the messages in a channel if it goes beyond the pagination limit, in the sitemap

* multithreaded sitemap

* use bool channels instead of struct channels

* don't break in empty switch case

* i misplaced a .Wait call

* tried to make the get functions async. no difference

* pprof debugging

* introduced semaphores.

* raised semaphore limit to 1024

* 'before' and 'after'

* better system for pagination. scrapped paginated pages in the sitemap. sorted sitemap by fastest to slowest

discord.go

@@ -3,12 +3,40 @@ package main
 import (
 	"context"
 	"sync"
+	"time"
 
 	"github.com/diamondburned/arikawa/v3/api"
 	"github.com/diamondburned/arikawa/v3/discord"
 	"github.com/diamondburned/arikawa/v3/gateway"
 )
 
+const SemaphoreLimit = 128
+
+type Semaphore struct {
+	semaCh chan struct{}
+}
+
+func NewSemaphore(maxReq int) *Semaphore {
+	return &Semaphore{
+		semaCh: make(chan struct{}, maxReq),
+	}
+}
+
+func (s *Semaphore) AcquireRead() {
+	s.semaCh <- struct{}{}
+}
+
+func (s *Semaphore) AcquireWrite() {
+	for len(s.semaCh) > 0 {
+		<-s.semaCh
+	}
+	s.semaCh <- struct{}{}
+}
+
+func (s *Semaphore) Release() {
+	<-s.semaCh
+}
+
 // ensureArchivedThreads ensures that all archived threads in the channel are in
 // the cache.
 func (s *server) ensureArchivedThreads(cid discord.ChannelID) error {
@@ -84,9 +112,11 @@ type messageCache struct {
 	*api.Client
 	channels sync.Map // discord.ChannelID -> *channel
 }
+
 type channel struct {
-	mut  sync.Mutex
-	msgs []discord.Message
+	msgs   []discord.Message
+	sem    *Semaphore
+	hasAll bool
 }
 
 func newMessageCache(c *api.Client) *messageCache {
@@ -96,25 +126,30 @@ func newMessageCache(c *api.Client) *messageCache {
 }
 
 func (c *messageCache) Set(m discord.Message, update bool) {
-	v, ok := c.channels.Load(m.ChannelID)
-	if !ok {
-		return
-	}
-	ch := v.(*channel)
-	ch.mut.Lock()
-	defer ch.mut.Unlock()
-	if ch.msgs == nil {
-		return
-	}
-	if update {
-		for i := len(ch.msgs) - 1; i >= 0; i-- {
-			if ch.msgs[i].ID == m.ID {
-				ch.msgs[i] = m
-				return
+	for i := 0; i < 0; i++ {
+		v, ok := c.channels.Load(m.ChannelID)
+		if !ok {
+			return
+		}
+		ch := v.(*channel)
+		if ch.sem == nil {
+			ch.sem = NewSemaphore(SemaphoreLimit)
+		}
+		ch.sem.AcquireWrite()
+		defer ch.sem.Release()
+		if ch.msgs == nil {
+			return
+		}
+		if update {
+			for i := len(ch.msgs) - 1; i >= 0; i-- {
+				if ch.msgs[i].ID == m.ID {
+					ch.msgs[i] = m
+					return
+				}
 			}
 		}
+		ch.msgs = append(ch.msgs, m)
 	}
-	ch.msgs = append(ch.msgs, m)
 }
 
 func (c *messageCache) Remove(chid discord.ChannelID, id discord.MessageID) {
@@ -123,8 +158,11 @@ func (c *messageCache) Remove(chid discord.ChannelID, id discord.MessageID) {
 		return
 	}
 	ch := v.(*channel)
-	ch.mut.Lock()
-	defer ch.mut.Unlock()
+	if ch.sem == nil {
+		ch.sem = NewSemaphore(SemaphoreLimit)
+	}
+	ch.sem.AcquireWrite()
+	defer ch.sem.Release()
 	for i, msg := range ch.msgs {
 		if msg.ID == id {
 			ch.msgs = append(ch.msgs[:i], ch.msgs[i+1:]...)
@@ -133,23 +171,91 @@ func (c *messageCache) Remove(chid discord.ChannelID, id discord.MessageID) {
 	}
 }
 
-func (c *messageCache) Messages(id discord.ChannelID) ([]discord.Message, error) {
+func (c *messageCache) MessagesBetween(id discord.ChannelID, before, after, limit uint) ([]discord.Message, error, discord.MessageID, discord.MessageID) {
 	v, _ := c.channels.LoadOrStore(id, &channel{})
 	ch := v.(*channel)
-	ch.mut.Lock()
-	defer ch.mut.Unlock()
+	if ch.sem == nil {
+		ch.sem = NewSemaphore(SemaphoreLimit)
+	}
+	ch.sem.AcquireRead()
+	defer ch.sem.Release()
 	if ch.msgs == nil {
-		msgs, err := c.Client.Messages(id, 0)
+		msgs, err := c.Client.MessagesAfter(id, discord.MessageID(after), paginationLimit+1)
 		if err != nil {
-
-			return nil, err
+			return nil, err, 0, 0
 		}
 		for i, j := 0, len(msgs)-1; i < j; i, j = i+1, j-1 {
 			msgs[i], msgs[j] = msgs[j], msgs[i]
 		}
 		ch.msgs = msgs
+		// start a little goroutine in the back to cache the rest of the messages
+		go func() {
+			time.Sleep(1 * time.Second)
+			msgs, _ := c.Client.Messages(id, 0)
+			for i, j := 0, len(msgs)-1; i < j; i, j = i+1, j-1 {
+				msgs[i], msgs[j] = msgs[j], msgs[i]
+			}
+			ch.msgs = msgs
+			ch.hasAll = true
+		}()
+	}
+	// if either before or after are set we need to wait for all the messages to be cached
+	//fmt.Println(before, ", ", after)
+	if before != 0 && after != 0 {
+		for !ch.hasAll {
+			time.Sleep(250 * time.Millisecond)
+		}
+	}
+
+	beforeInt := 0
+	afterInt := 0
+	var prevID, nextID discord.MessageID
+
+	for i, msg := range ch.msgs {
+		if uint(msg.ID) == before {
+			beforeInt = i
+		}
+		if uint(msg.ID) == after {
+			afterInt = i
+		}
+
 	}
-	msgs := make([]discord.Message, len(ch.msgs))
-	copy(msgs, ch.msgs)
-	return msgs, nil
+
+	// if there's nothing before us or before is unset
+	if beforeInt == 0 {
+		// set the before int to the max
+		beforeInt = int(limit + uint(afterInt))
+		if beforeInt > len(ch.msgs) {
+			beforeInt = len(ch.msgs)
+		}
+	}
+	if beforeInt+1 < len(ch.msgs) {
+		nextID = ch.msgs[beforeInt+1].ID
+	}
+	if afterInt-1 > 0 {
+		afterInt -= 1
+		prevID = ch.msgs[afterInt].ID
+	}
+
+	// if after is unset
+	if afterInt == 0 {
+		afterInt = beforeInt - int(limit)
+	}
+	if afterInt < 0 {
+		afterInt = 0
+	}
+
+	msgs := make([]discord.Message, len(ch.msgs[afterInt:beforeInt]))
+	copy(msgs, ch.msgs[afterInt:beforeInt])
+	return msgs, nil, prevID, nextID
+}
+
+func (c *messageCache) Messages(id discord.ChannelID, limit uint) ([]discord.Message, error, discord.MessageID, discord.MessageID) {
+	return c.MessagesBetween(id, 0, 0, limit)
+}
+func (c *messageCache) MessagesBefore(id discord.ChannelID, before, limit uint) ([]discord.Message, error, discord.MessageID, discord.MessageID) {
+	return c.MessagesBetween(id, before, 0, limit)
+}
+func (c *messageCache) MessagesAfter(id discord.ChannelID, after, limit uint) ([]discord.Message, error, discord.MessageID, discord.MessageID) {
+	return c.MessagesBetween(id, 0, after, limit)
 }

main.go

@@ -13,6 +13,8 @@ import (
 	"os/signal"
 	"time"
 
+	_ "net/http/pprof"
+
 	"github.com/diamondburned/arikawa/v3/gateway"
 	"github.com/diamondburned/arikawa/v3/state"
 	"github.com/naoina/toml"
@@ -85,8 +87,8 @@ func main() {
 	httpserver := &http.Server{
 		Addr:           config.ListenAddr,
 		Handler:        server,
-		ReadTimeout:    10 * time.Second,
-		WriteTimeout:   10 * time.Second,
+		ReadTimeout:    60 * time.Second,
+		WriteTimeout:   60 * time.Second,
 		MaxHeaderBytes: 1 << 20,
 	}
 	httperr := make(chan error, 1)