From 37d310f981f3e26bc643aeabac134b591c84455a Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Tue, 29 Mar 2022 11:54:56 +0200 Subject: [PATCH] [feature] Dereference remote mentions when the account is not already known (#442) * remove mention util function from db * add ParseMentionFunc to gtsmodel * add parseMentionFunc to processor * refactor search to simplify it a bit * add parseMentionFunc to account * add parseMentionFunc to status * some renaming for clarity * test dereference of unknown mentioned account --- internal/api/client/status/status_test.go | 98 +++++++++++- .../api/client/status/statuscreate_test.go | 134 ++++++++++------ internal/db/bundb/bundb.go | 80 ---------- internal/db/db.go | 9 -- internal/gtsmodel/mention.go | 17 +- internal/processing/account/account.go | 4 +- internal/processing/account/account_test.go | 3 +- internal/processing/account/update.go | 12 +- internal/processing/processor.go | 6 +- internal/processing/search.go | 11 +- internal/processing/status/status.go | 4 +- internal/processing/status/status_test.go | 21 ++- internal/processing/status/util.go | 33 ++-- internal/processing/util.go | 151 ++++++++++++++++++ internal/util/statustools.go | 12 +- internal/util/statustools_test.go | 8 +- 16 files changed, 420 insertions(+), 183 deletions(-) create mode 100644 internal/processing/util.go diff --git a/internal/api/client/status/status_test.go b/internal/api/client/status/status_test.go index c6e5b354fe..fd44a6a7c3 100644 --- a/internal/api/client/status/status_test.go +++ b/internal/api/client/status/status_test.go @@ -19,8 +19,18 @@ package status_test import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/activity/pub" + "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/gotosocial/internal/api/client/status" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -73,7 +83,7 @@ func (suite *StatusStandardTestSuite) SetupTest() { suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(suite.testHttpClient(), suite.db), suite.storage, suite.mediaManager) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) suite.statusModule = status.New(suite.processor).(*status.Module) @@ -85,3 +95,89 @@ func (suite *StatusStandardTestSuite) TearDownTest() { testrig.StandardDBTeardown(suite.db) testrig.StandardStorageTeardown(suite.storage) } + +func (suite *StatusStandardTestSuite) testHttpClient() pub.HttpClient { + remoteAccount := suite.testAccounts["remote_account_1"] + remoteAccountNamestring := fmt.Sprintf("acct:%s@%s", remoteAccount.Username, remoteAccount.Domain) + remoteAccountWebfingerURI := fmt.Sprintf("https://%s/.well-known/webfinger?resource=%s", remoteAccount.Domain, remoteAccountNamestring) + + fmt.Println(remoteAccountWebfingerURI) + + httpClient := testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { + + // respond correctly to a webfinger lookup + if req.URL.String() == remoteAccountWebfingerURI { + responseJson := fmt.Sprintf(` + { + "subject": "%s", + "aliases": [ + "%s", + "%s" + ], + "links": [ + { + "rel": "http://webfinger.net/rel/profile-page", + "type": "text/html", + "href": "%s" + }, + { + "rel": "self", + "type": "application/activity+json", + "href": "%s" + } + ] + }`, remoteAccountNamestring, remoteAccount.URI, remoteAccount.URL, remoteAccount.URL, remoteAccount.URI) + responseType := "application/json" + + reader := bytes.NewReader([]byte(responseJson)) + readCloser := io.NopCloser(reader) + response := &http.Response{ + StatusCode: 200, + Body: readCloser, + ContentLength: int64(len(responseJson)), + Header: http.Header{ + "content-type": {responseType}, + }, + } + return response, nil + } + + // respond correctly to an account dereference + if req.URL.String() == remoteAccount.URI { + satanAS, err := suite.tc.AccountToAS(context.Background(), remoteAccount) + if err != nil { + panic(err) + } + + satanI, err := streams.Serialize(satanAS) + if err != nil { + panic(err) + } + satanJson, err := json.Marshal(satanI) + if err != nil { + panic(err) + } + responseType := "application/activity+json" + + reader := bytes.NewReader(satanJson) + readCloser := io.NopCloser(reader) + response := &http.Response{ + StatusCode: 200, + Body: readCloser, + ContentLength: int64(len(satanJson)), + Header: http.Header{ + "content-type": {responseType}, + }, + } + return response, nil + } + + r := ioutil.NopCloser(bytes.NewReader([]byte{})) + return &http.Response{ + StatusCode: 200, + Body: r, + }, nil + }) + + return httpClient +} diff --git a/internal/api/client/status/statuscreate_test.go b/internal/api/client/status/statuscreate_test.go index cfacc2a93f..a1df255473 100644 --- a/internal/api/client/status/statuscreate_test.go +++ b/internal/api/client/status/statuscreate_test.go @@ -29,7 +29,6 @@ import ( "testing" "github.com/gin-gonic/gin" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/status" "github.com/superseriousbusiness/gotosocial/internal/api/model" @@ -85,26 +84,69 @@ func (suite *StatusCreateTestSuite) TestPostNewStatus() { result := recorder.Result() defer result.Body.Close() b, err := ioutil.ReadAll(result.Body) - assert.NoError(suite.T(), err) + suite.NoError(err) statusReply := &model.Status{} err = json.Unmarshal(b, statusReply) - assert.NoError(suite.T(), err) - - assert.Equal(suite.T(), "hello hello", statusReply.SpoilerText) - assert.Equal(suite.T(), "

this is a brand new status! #helloworld

", statusReply.Content) - assert.True(suite.T(), statusReply.Sensitive) - assert.Equal(suite.T(), model.VisibilityPrivate, statusReply.Visibility) // even though we set this status to mutuals only, it should serialize to private, because the mastodon api has no idea about mutuals_only - assert.Len(suite.T(), statusReply.Tags, 1) - assert.Equal(suite.T(), model.Tag{ + suite.NoError(err) + + suite.Equal("hello hello", statusReply.SpoilerText) + suite.Equal("

this is a brand new status! #helloworld

", statusReply.Content) + suite.True(statusReply.Sensitive) + suite.Equal(model.VisibilityPrivate, statusReply.Visibility) // even though we set this status to mutuals only, it should serialize to private, because the mastodon api has no idea about mutuals_only + suite.Len(statusReply.Tags, 1) + suite.Equal(model.Tag{ Name: "helloworld", URL: "http://localhost:8080/tags/helloworld", }, statusReply.Tags[0]) gtsTag := >smodel.Tag{} err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "name", Value: "helloworld"}}, gtsTag) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), statusReply.Account.ID, gtsTag.FirstSeenFromAccountID) + suite.NoError(err) + suite.Equal(statusReply.Account.ID, gtsTag.FirstSeenFromAccountID) +} + +// mention an account that is not yet known to the instance -- it should be looked up and put in the db +func (suite *StatusCreateTestSuite) TestMentionUnknownAccount() { + + // first remove remote account 1 from the database so it gets looked up again + remoteAccount := suite.testAccounts["remote_account_1"] + if err := suite.db.DeleteByID(context.Background(), remoteAccount.ID, >smodel.Account{}); err != nil { + panic(err) + } + + t := suite.testTokens["local_account_1"] + oauthToken := oauth.DBTokenToToken(t) + + // setup + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"]) + ctx.Set(oauth.SessionAuthorizedToken, oauthToken) + ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"]) + ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"]) + ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080/%s", status.BasePath), nil) // the endpoint we're hitting + ctx.Request.Header.Set("accept", "application/json") + ctx.Request.Form = url.Values{ + "status": {"hello @foss_satan@fossbros-anonymous.io"}, + "visibility": {string(model.VisibilityPublic)}, + } + suite.statusModule.StatusCreatePOSTHandler(ctx) + + suite.EqualValues(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + b, err := ioutil.ReadAll(result.Body) + suite.NoError(err) + + statusReply := &model.Status{} + err = json.Unmarshal(b, statusReply) + suite.NoError(err) + + // if the status is properly formatted, that means the account has been put in the db + suite.Equal("

hello @foss_satan

", statusReply.Content) + suite.Equal(model.VisibilityPublic, statusReply.Visibility) } func (suite *StatusCreateTestSuite) TestPostAnotherNewStatus() { @@ -134,13 +176,13 @@ func (suite *StatusCreateTestSuite) TestPostAnotherNewStatus() { result := recorder.Result() defer result.Body.Close() b, err := ioutil.ReadAll(result.Body) - assert.NoError(suite.T(), err) + suite.NoError(err) statusReply := &model.Status{} err = json.Unmarshal(b, statusReply) - assert.NoError(suite.T(), err) + suite.NoError(err) - assert.Equal(suite.T(), "

#test alright, should be able to post #links with fragments in them now, let's see........

docs.gotosocial.org/en/latest/user_guide/posts/#links

#gotosocial

(tobi remember to pull the docker image challenge)

", statusReply.Content) + suite.Equal("

#test alright, should be able to post #links with fragments in them now, let's see........

docs.gotosocial.org/en/latest/user_guide/posts/#links

#gotosocial

(tobi remember to pull the docker image challenge)

", statusReply.Content) } func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() { @@ -167,22 +209,22 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() { result := recorder.Result() defer result.Body.Close() b, err := ioutil.ReadAll(result.Body) - assert.NoError(suite.T(), err) + suite.NoError(err) statusReply := &model.Status{} err = json.Unmarshal(b, statusReply) - assert.NoError(suite.T(), err) + suite.NoError(err) - assert.Equal(suite.T(), "", statusReply.SpoilerText) - assert.Equal(suite.T(), "

here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:
here's an emoji that isn't in the db: :test_emoji:

", statusReply.Content) + suite.Equal("", statusReply.SpoilerText) + suite.Equal("

here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:
here's an emoji that isn't in the db: :test_emoji:

", statusReply.Content) - assert.Len(suite.T(), statusReply.Emojis, 1) + suite.Len(statusReply.Emojis, 1) apiEmoji := statusReply.Emojis[0] gtsEmoji := testrig.NewTestEmojis()["rainbow"] - assert.Equal(suite.T(), gtsEmoji.Shortcode, apiEmoji.Shortcode) - assert.Equal(suite.T(), gtsEmoji.ImageURL, apiEmoji.URL) - assert.Equal(suite.T(), gtsEmoji.ImageStaticURL, apiEmoji.StaticURL) + suite.Equal(gtsEmoji.Shortcode, apiEmoji.Shortcode) + suite.Equal(gtsEmoji.ImageURL, apiEmoji.URL) + suite.Equal(gtsEmoji.ImageStaticURL, apiEmoji.StaticURL) } // Try to reply to a status that doesn't exist @@ -213,8 +255,8 @@ func (suite *StatusCreateTestSuite) TestReplyToNonexistentStatus() { result := recorder.Result() defer result.Body.Close() b, err := ioutil.ReadAll(result.Body) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), `{"error":"bad request"}`, string(b)) + suite.NoError(err) + suite.Equal(`{"error":"bad request"}`, string(b)) } // Post a reply to the status of a local user that allows replies. @@ -243,19 +285,19 @@ func (suite *StatusCreateTestSuite) TestReplyToLocalStatus() { result := recorder.Result() defer result.Body.Close() b, err := ioutil.ReadAll(result.Body) - assert.NoError(suite.T(), err) + suite.NoError(err) statusReply := &model.Status{} err = json.Unmarshal(b, statusReply) - assert.NoError(suite.T(), err) - - assert.Equal(suite.T(), "", statusReply.SpoilerText) - assert.Equal(suite.T(), fmt.Sprintf("

hello @%s this reply should work!

", testrig.NewTestAccounts()["local_account_2"].Username, testrig.NewTestAccounts()["local_account_2"].Username), statusReply.Content) - assert.False(suite.T(), statusReply.Sensitive) - assert.Equal(suite.T(), model.VisibilityPublic, statusReply.Visibility) - assert.Equal(suite.T(), testrig.NewTestStatuses()["local_account_2_status_1"].ID, statusReply.InReplyToID) - assert.Equal(suite.T(), testrig.NewTestAccounts()["local_account_2"].ID, statusReply.InReplyToAccountID) - assert.Len(suite.T(), statusReply.Mentions, 1) + suite.NoError(err) + + suite.Equal("", statusReply.SpoilerText) + suite.Equal(fmt.Sprintf("

hello @%s this reply should work!

", testrig.NewTestAccounts()["local_account_2"].Username, testrig.NewTestAccounts()["local_account_2"].Username), statusReply.Content) + suite.False(statusReply.Sensitive) + suite.Equal(model.VisibilityPublic, statusReply.Visibility) + suite.Equal(testrig.NewTestStatuses()["local_account_2_status_1"].ID, statusReply.InReplyToID) + suite.Equal(testrig.NewTestAccounts()["local_account_2"].ID, statusReply.InReplyToAccountID) + suite.Len(statusReply.Mentions, 1) } // Take a media file which is currently not associated with a status, and attach it to a new status. @@ -286,33 +328,33 @@ func (suite *StatusCreateTestSuite) TestAttachNewMediaSuccess() { result := recorder.Result() defer result.Body.Close() b, err := ioutil.ReadAll(result.Body) - assert.NoError(suite.T(), err) + suite.NoError(err) statusResponse := &model.Status{} err = json.Unmarshal(b, statusResponse) - assert.NoError(suite.T(), err) + suite.NoError(err) - assert.Equal(suite.T(), "", statusResponse.SpoilerText) - assert.Equal(suite.T(), "

here's an image attachment

", statusResponse.Content) - assert.False(suite.T(), statusResponse.Sensitive) - assert.Equal(suite.T(), model.VisibilityPublic, statusResponse.Visibility) + suite.Equal("", statusResponse.SpoilerText) + suite.Equal("

here's an image attachment

", statusResponse.Content) + suite.False(statusResponse.Sensitive) + suite.Equal(model.VisibilityPublic, statusResponse.Visibility) // there should be one media attachment - assert.Len(suite.T(), statusResponse.MediaAttachments, 1) + suite.Len(statusResponse.MediaAttachments, 1) // get the updated media attachment from the database gtsAttachment, err := suite.db.GetAttachmentByID(context.Background(), statusResponse.MediaAttachments[0].ID) - assert.NoError(suite.T(), err) + suite.NoError(err) // convert it to a api attachment gtsAttachmentAsapi, err := suite.tc.AttachmentToAPIAttachment(context.Background(), gtsAttachment) - assert.NoError(suite.T(), err) + suite.NoError(err) // compare it with what we have now - assert.EqualValues(suite.T(), statusResponse.MediaAttachments[0], gtsAttachmentAsapi) + suite.EqualValues(statusResponse.MediaAttachments[0], gtsAttachmentAsapi) // the status id of the attachment should now be set to the id of the status we just created - assert.Equal(suite.T(), statusResponse.ID, gtsAttachment.StatusID) + suite.Equal(statusResponse.ID, gtsAttachment.StatusID) } func TestStatusCreateTestSuite(t *testing.T) { diff --git a/internal/db/bundb/bundb.go b/internal/db/bundb/bundb.go index 5ccca06295..f190ce5eaf 100644 --- a/internal/db/bundb/bundb.go +++ b/internal/db/bundb/bundb.go @@ -382,86 +382,6 @@ func tweakConnectionValues(sqldb *sql.DB) { CONVERSION FUNCTIONS */ -// TODO: move these to the type converter, it's bananas that they're here and not there - -func (ps *bunDBService) MentionStringsToMentions(ctx context.Context, targetAccounts []string, originAccountID string, statusID string) ([]*gtsmodel.Mention, error) { - ogAccount := >smodel.Account{} - if err := ps.conn.NewSelect().Model(ogAccount).Where("id = ?", originAccountID).Scan(ctx); err != nil { - return nil, err - } - - menchies := []*gtsmodel.Mention{} - for _, a := range targetAccounts { - // A mentioned account looks like "@test@example.org" or just "@test" for a local account - // -- we can guarantee this from the regex that targetAccounts should have been derived from. - // But we still need to do a bit of fiddling to get what we need here -- the username and domain (if given). - - // 1. trim off the first @ - t := strings.TrimPrefix(a, "@") - - // 2. split the username and domain - s := strings.Split(t, "@") - - // 3. if it's length 1 it's a local account, length 2 means remote, anything else means something is wrong - var local bool - switch len(s) { - case 1: - local = true - case 2: - local = false - default: - return nil, fmt.Errorf("mentioned account format '%s' was not valid", a) - } - - var username, domain string - username = s[0] - if !local { - domain = s[1] - } - - // 4. check we now have a proper username and domain - if username == "" || (!local && domain == "") { - return nil, fmt.Errorf("username or domain for '%s' was nil", a) - } - - // okay we're good now, we can start pulling accounts out of the database - mentionedAccount := >smodel.Account{} - var err error - - // match username + account, case insensitive - if local { - // local user -- should have a null domain - err = ps.conn.NewSelect().Model(mentionedAccount).Where("LOWER(?) = LOWER(?)", bun.Ident("username"), username).Where("? IS NULL", bun.Ident("domain")).Scan(ctx) - } else { - // remote user -- should have domain defined - err = ps.conn.NewSelect().Model(mentionedAccount).Where("LOWER(?) = LOWER(?)", bun.Ident("username"), username).Where("LOWER(?) = LOWER(?)", bun.Ident("domain"), domain).Scan(ctx) - } - - if err != nil { - if err == sql.ErrNoRows { - // no result found for this username/domain so just don't include it as a mencho and carry on about our business - logrus.Debugf("no account found with username '%s' and domain '%s', skipping it", username, domain) - continue - } - // a serious error has happened so bail - return nil, fmt.Errorf("error getting account with username '%s' and domain '%s': %s", username, domain, err) - } - - // id, createdAt and updatedAt will be populated by the db, so we have everything we need! - menchies = append(menchies, >smodel.Mention{ - StatusID: statusID, - OriginAccountID: ogAccount.ID, - OriginAccountURI: ogAccount.URI, - TargetAccountID: mentionedAccount.ID, - NameString: a, - TargetAccountURI: mentionedAccount.URI, - TargetAccountURL: mentionedAccount.URL, - OriginAccount: mentionedAccount, - }) - } - return menchies, nil -} - func (ps *bunDBService) TagStringsToTags(ctx context.Context, tags []string, originAccountID string) ([]*gtsmodel.Tag, error) { protocol := viper.GetString(config.Keys.Protocol) host := viper.GetString(config.Keys.Host) diff --git a/internal/db/db.go b/internal/db/db.go index 151cc1f975..24205542c2 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -48,15 +48,6 @@ type DB interface { USEFUL CONVERSION FUNCTIONS */ - // MentionStringsToMentions takes a slice of deduplicated, lowercase account names in the form "@test@whatever.example.org" for a remote account, - // or @test for a local account, which have been mentioned in a status. - // It takes the id of the account that wrote the status, and the id of the status itself, and then - // checks in the database for the mentioned accounts, and returns a slice of mentions generated based on the given parameters. - // - // Note: this func doesn't/shouldn't do any manipulation of the accounts in the DB, it's just for checking - // if they exist in the db and conveniently returning them if they do. - MentionStringsToMentions(ctx context.Context, targetAccounts []string, originAccountID string, statusID string) ([]*gtsmodel.Mention, error) - // TagStringsToTags takes a slice of deduplicated, lowercase tags in the form "somehashtag", which have been // used in a status. It takes the id of the account that wrote the status, and the id of the status itself, and then // returns a slice of *model.Tag corresponding to the given tags. If the tag already exists in database, that tag diff --git a/internal/gtsmodel/mention.go b/internal/gtsmodel/mention.go index e00df89110..fd1d45622a 100644 --- a/internal/gtsmodel/mention.go +++ b/internal/gtsmodel/mention.go @@ -18,7 +18,10 @@ package gtsmodel -import "time" +import ( + "context" + "time" +) // Mention refers to the 'tagging' or 'mention' of a user within a status. type Mention struct { @@ -57,3 +60,15 @@ type Mention struct { TargetAccountURL string `validate:"-" bun:"-"` // A pointer to the gtsmodel account of the mentioned account. } + +// ParseMentionFunc describes a function that takes a lowercase account name +// in the form "@test@whatever.example.org" for a remote account, or "@test" +// for a local account, and returns a fully populated mention for that account, +// with the given origin status ID and origin account ID. +// +// If the account is remote and not yet found in the database, then ParseMentionFunc +// will try to webfinger the remote account and put it in the database before returning. +// +// Mentions generated by this function are not put in the database, that's still up to +// the caller to do. +type ParseMentionFunc func(ctx context.Context, targetAccount string, originAccountID string, statusID string) (*Mention, error) diff --git a/internal/processing/account/account.go b/internal/processing/account/account.go index c670ee24a0..1ef92cf85f 100644 --- a/internal/processing/account/account.go +++ b/internal/processing/account/account.go @@ -87,10 +87,11 @@ type processor struct { formatter text.Formatter db db.DB federator federation.Federator + parseMention gtsmodel.ParseMentionFunc } // New returns a new account processor. -func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauthServer oauth.Server, fromClientAPI chan messages.FromClientAPI, federator federation.Federator) Processor { +func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauthServer oauth.Server, fromClientAPI chan messages.FromClientAPI, federator federation.Federator, parseMention gtsmodel.ParseMentionFunc) Processor { return &processor{ tc: tc, mediaManager: mediaManager, @@ -100,5 +101,6 @@ func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauth formatter: text.NewFormatter(db), db: db, federator: federator, + parseMention: parseMention, } } diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go index 5a9382ed63..aff4f02aa9 100644 --- a/internal/processing/account/account_test.go +++ b/internal/processing/account/account_test.go @@ -29,6 +29,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" + "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/processing/account" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" @@ -88,7 +89,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails) - suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, suite.fromClientAPIChan, suite.federator) + suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, suite.fromClientAPIChan, suite.federator, processing.GetParseMentionFunc(suite.db, suite.federator)) testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../testrig/media") } diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index a96b44befa..a0056563bc 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -199,10 +199,14 @@ func (p *processor) processNote(ctx context.Context, note string, accountID stri return "", err } - mentionStrings := util.DeriveMentionsFromText(note) - mentions, err := p.db.MentionStringsToMentions(ctx, mentionStrings, accountID, "") - if err != nil { - return "", err + mentionStrings := util.DeriveMentionNamesFromText(note) + mentions := []*gtsmodel.Mention{} + for _, mentionString := range mentionStrings { + mention, err := p.parseMention(ctx, mentionString, accountID, "") + if err != nil { + continue + } + mentions = append(mentions, mention) } // TODO: support emojis in account notes diff --git a/internal/processing/processor.go b/internal/processing/processor.go index 50f4af492c..f5d9eab289 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -267,12 +267,14 @@ func NewProcessor( storage *kv.KVStore, db db.DB, emailSender email.Sender) Processor { + fromClientAPI := make(chan messages.FromClientAPI, 1000) fromFederator := make(chan messages.FromFederator, 1000) + parseMentionFunc := GetParseMentionFunc(db, federator) - statusProcessor := status.New(db, tc, fromClientAPI) + statusProcessor := status.New(db, tc, fromClientAPI, parseMentionFunc) streamingProcessor := streaming.New(db, oauthServer) - accountProcessor := account.New(db, tc, mediaManager, oauthServer, fromClientAPI, federator) + accountProcessor := account.New(db, tc, mediaManager, oauthServer, fromClientAPI, federator, parseMentionFunc) adminProcessor := admin.New(db, tc, mediaManager, fromClientAPI) mediaProcessor := mediaProcessor.New(db, tc, mediaManager, federator.TransportController(), storage) userProcessor := user.New(db, emailSender) diff --git a/internal/processing/search.go b/internal/processing/search.go index c8c302857b..6c3d7da39a 100644 --- a/internal/processing/search.go +++ b/internal/processing/search.go @@ -194,20 +194,15 @@ func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Au // we got a db.ErrNoEntries, so we just don't have the account locally stored -- check if we can dereference it if resolve { // we're allowed to resolve it so let's try - // first we need to webfinger the remote account to convert the username and domain into the activitypub URI for the account acctURI, err := p.federator.FingerRemoteAccount(ctx, authed.Account.Username, username, domain) if err != nil { // something went wrong doing the webfinger lookup so we can't process the request - return nil, fmt.Errorf("searchAccountByMention: error fingering remote account with username %s and domain %s: %s", username, domain, err) + return nil, fmt.Errorf("error fingering remote account with username %s and domain %s: %s", username, domain, err) } - // we don't have it locally so try and dereference it - account, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, acctURI, true, true) - if err != nil { - return nil, fmt.Errorf("searchAccountByMention: error dereferencing account with uri %s: %s", acctURI.String(), err) - } - return account, nil + // return the attempt to get the remove account + return p.federator.GetRemoteAccount(ctx, authed.Account.Username, acctURI, true, true) } return nil, nil diff --git a/internal/processing/status/status.go b/internal/processing/status/status.go index 3118b23ada..421ab5bbee 100644 --- a/internal/processing/status/status.go +++ b/internal/processing/status/status.go @@ -74,15 +74,17 @@ type processor struct { filter visibility.Filter formatter text.Formatter fromClientAPI chan messages.FromClientAPI + parseMention gtsmodel.ParseMentionFunc } // New returns a new status processor. -func New(db db.DB, tc typeutils.TypeConverter, fromClientAPI chan messages.FromClientAPI) Processor { +func New(db db.DB, tc typeutils.TypeConverter, fromClientAPI chan messages.FromClientAPI, parseMention gtsmodel.ParseMentionFunc) Processor { return &processor{ tc: tc, db: db, filter: visibility.NewFilter(db), formatter: text.NewFormatter(db), fromClientAPI: fromClientAPI, + parseMention: parseMention, } } diff --git a/internal/processing/status/status_test.go b/internal/processing/status/status_test.go index 803e7a0a55..25794070e6 100644 --- a/internal/processing/status/status_test.go +++ b/internal/processing/status/status_test.go @@ -19,11 +19,16 @@ package status_test import ( + "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/messages" + "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/processing/status" + "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -32,6 +37,10 @@ type StatusStandardTestSuite struct { suite.Suite db db.DB typeConverter typeutils.TypeConverter + tc transport.Controller + storage *kv.KVStore + mediaManager media.Manager + federator federation.Federator fromClientAPIChan chan messages.FromClientAPI // standard suite models @@ -62,17 +71,23 @@ func (suite *StatusStandardTestSuite) SetupSuite() { } func (suite *StatusStandardTestSuite) SetupTest() { - testrig.InitTestLog() testrig.InitTestConfig() + testrig.InitTestLog() suite.db = testrig.NewTestDB() suite.typeConverter = testrig.NewTestTypeConverter(suite.db) suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100) - suite.status = status.New(suite.db, suite.typeConverter, suite.fromClientAPIChan) + suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) + suite.storage = testrig.NewTestStorage() + suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, suite.tc, suite.storage, suite.mediaManager) + suite.status = status.New(suite.db, suite.typeConverter, suite.fromClientAPIChan, processing.GetParseMentionFunc(suite.db, suite.federator)) - testrig.StandardDBSetup(suite.db, nil) + testrig.StandardDBSetup(suite.db, suite.testAccounts) + testrig.StandardStorageSetup(suite.storage, "../../../testrig/media") } func (suite *StatusStandardTestSuite) TearDownTest() { testrig.StandardDBTeardown(suite.db) + testrig.StandardStorageTeardown(suite.storage) } diff --git a/internal/processing/status/util.go b/internal/processing/status/util.go index f2640929d1..5de66af8a3 100644 --- a/internal/processing/status/util.go +++ b/internal/processing/status/util.go @@ -23,10 +23,10 @@ import ( "errors" "fmt" + "github.com/sirupsen/logrus" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -192,27 +192,30 @@ func (p *processor) ProcessLanguage(ctx context.Context, form *apimodel.Advanced } func (p *processor) ProcessMentions(ctx context.Context, form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error { - menchies := []string{} - gtsMenchies, err := p.db.MentionStringsToMentions(ctx, util.DeriveMentionsFromText(form.Status), accountID, status.ID) - if err != nil { - return fmt.Errorf("error generating mentions from status: %s", err) - } - for _, menchie := range gtsMenchies { - menchieID, err := id.NewRandomULID() + mentionedAccountNames := util.DeriveMentionNamesFromText(form.Status) + mentions := []*gtsmodel.Mention{} + mentionIDs := []string{} + + for _, mentionedAccountName := range mentionedAccountNames { + gtsMention, err := p.parseMention(ctx, mentionedAccountName, accountID, status.ID) if err != nil { - return err + logrus.Errorf("ProcessMentions: error parsing mention %s from status: %s", mentionedAccountName, err) + continue } - menchie.ID = menchieID - if err := p.db.Put(ctx, menchie); err != nil { - return fmt.Errorf("error putting mentions in db: %s", err) + if err := p.db.Put(ctx, gtsMention); err != nil { + logrus.Errorf("ProcessMentions: error putting mention in db: %s", err) } - menchies = append(menchies, menchie.ID) + + mentions = append(mentions, gtsMention) + mentionIDs = append(mentionIDs, gtsMention.ID) } + // add full populated gts menchies to the status for passing them around conveniently - status.Mentions = gtsMenchies + status.Mentions = mentions // add just the ids of the mentioned accounts to the status for putting in the db - status.MentionIDs = menchies + status.MentionIDs = mentionIDs + return nil } diff --git a/internal/processing/util.go b/internal/processing/util.go new file mode 100644 index 0000000000..7ae7b8604a --- /dev/null +++ b/internal/processing/util.go @@ -0,0 +1,151 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package processing + +import ( + "context" + "fmt" + "strings" + + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" +) + +func GetParseMentionFunc(dbConn db.DB, federator federation.Federator) gtsmodel.ParseMentionFunc { + return func(ctx context.Context, targetAccount string, originAccountID string, statusID string) (*gtsmodel.Mention, error) { + // get the origin account first since we'll need it to create the mention + originAccount, err := dbConn.GetAccountByID(ctx, originAccountID) + if err != nil { + return nil, fmt.Errorf("couldn't get mention origin account with id %s", originAccountID) + } + + // A mentioned account looks like "@test@example.org" or just "@test" for a local account + // -- we can guarantee this from the regex that targetAccounts should have been derived from. + // But we still need to do a bit of fiddling to get what we need here -- the username and domain (if given). + + // 1. trim off the first @ + trimmed := strings.TrimPrefix(targetAccount, "@") + + // 2. split the username and domain + split := strings.Split(trimmed, "@") + + // 3. if it's length 1 it's a local account, length 2 means remote, anything else means something is wrong + + var local bool + switch len(split) { + case 1: + local = true + case 2: + local = false + default: + return nil, fmt.Errorf("mentioned account format '%s' was not valid", targetAccount) + } + + var username, domain string + username = split[0] + if !local { + domain = split[1] + } + + // 4. check we now have a proper username and domain + if username == "" || (!local && domain == "") { + return nil, fmt.Errorf("username or domain for '%s' was nil", targetAccount) + } + + var mentionedAccount *gtsmodel.Account + + if local { + localAccount, err := dbConn.GetLocalAccountByUsername(ctx, username) + if err != nil { + return nil, err + } + mentionedAccount = localAccount + } else { + remoteAccount := >smodel.Account{} + + where := []db.Where{ + { + Key: "username", + Value: username, + CaseInsensitive: true, + }, + { + Key: "domain", + Value: domain, + CaseInsensitive: true, + }, + } + + err := dbConn.GetWhere(ctx, where, remoteAccount) + if err == nil { + // the account was already in the database + mentionedAccount = remoteAccount + } else { + // we couldn't get it from the database + if err != db.ErrNoEntries { + // a serious error has happened so bail + return nil, fmt.Errorf("error getting account with username '%s' and domain '%s': %s", username, domain, err) + } + + // We just don't have the account, so try webfingering it. + // + // If the mention originates from our instance we should use the username of the origin account to do the dereferencing, + // otherwise we should just use our instance account (that is, provide an empty string), since obviously we can't use + // a remote account to do remote dereferencing! + var fingeringUsername string + if originAccount.Domain == "" { + fingeringUsername = originAccount.Username + } + + acctURI, err := federator.FingerRemoteAccount(ctx, fingeringUsername, username, domain) + if err != nil { + // something went wrong doing the webfinger lookup so we can't process the request + return nil, fmt.Errorf("error fingering remote account with username %s and domain %s: %s", username, domain, err) + } + + resolvedAccount, err := federator.GetRemoteAccount(ctx, fingeringUsername, acctURI, true, true) + if err != nil { + return nil, fmt.Errorf("error dereferencing account with uri %s: %s", acctURI.String(), err) + } + + // we were able to resolve it! + mentionedAccount = resolvedAccount + } + } + + mentionID, err := id.NewRandomULID() + if err != nil { + return nil, err + } + + return >smodel.Mention{ + ID: mentionID, + StatusID: statusID, + OriginAccountID: originAccount.ID, + OriginAccountURI: originAccount.URI, + TargetAccountID: mentionedAccount.ID, + NameString: targetAccount, + TargetAccountURI: mentionedAccount.URI, + TargetAccountURL: mentionedAccount.URL, + OriginAccount: mentionedAccount, + }, nil + } +} diff --git a/internal/util/statustools.go b/internal/util/statustools.go index bb87c23fab..08032a8f4e 100644 --- a/internal/util/statustools.go +++ b/internal/util/statustools.go @@ -25,13 +25,11 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/regexes" ) -// DeriveMentionsFromText takes a plaintext (ie., not html-formatted) text, -// and applies a regex to it to return a deduplicated list of accounts -// mentioned in that text. -// -// It will look for fully-qualified account names in the form "@user@example.org". -// or the form "@username" for local users. -func DeriveMentionsFromText(text string) []string { +// DeriveMentionNamesFromText takes a plaintext (ie., not html-formatted) text, +// and applies a regex to it to return a deduplicated list of account names +// mentioned in that text, in the format "@user@example.org" or "@username" for +// local users. +func DeriveMentionNamesFromText(text string) []string { mentionedAccounts := []string{} for _, m := range regexes.MentionFinder.FindAllStringSubmatch(text, -1) { mentionedAccounts = append(mentionedAccounts, m[1]) diff --git a/internal/util/statustools_test.go b/internal/util/statustools_test.go index bcd9a48831..bea89158a5 100644 --- a/internal/util/statustools_test.go +++ b/internal/util/statustools_test.go @@ -37,7 +37,7 @@ https://localhost:8080/@the_mighty_zork/statuses/01FGVP55XMF2K6316MQRX6PFG1 that link shouldn't come out formatted as a mention!` - menchies := util.DeriveMentionsFromText(statusText) + menchies := util.DeriveMentionNamesFromText(statusText) suite.Empty(menchies) } @@ -56,7 +56,7 @@ func (suite *StatusTestSuite) TestDeriveMentionsOK() { ` - menchies := util.DeriveMentionsFromText(statusText) + menchies := util.DeriveMentionNamesFromText(statusText) assert.Len(suite.T(), menchies, 6) assert.Equal(suite.T(), "@dumpsterqueer@example.org", menchies[0]) assert.Equal(suite.T(), "@someone_else@testing.best-horse.com", menchies[1]) @@ -68,7 +68,7 @@ func (suite *StatusTestSuite) TestDeriveMentionsOK() { func (suite *StatusTestSuite) TestDeriveMentionsEmpty() { statusText := `` - menchies := util.DeriveMentionsFromText(statusText) + menchies := util.DeriveMentionNamesFromText(statusText) assert.Len(suite.T(), menchies, 0) } @@ -126,7 +126,7 @@ func (suite *StatusTestSuite) TestDeriveMultiple() { Text` - ms := util.DeriveMentionsFromText(statusText) + ms := util.DeriveMentionNamesFromText(statusText) hs := util.DeriveHashtagsFromText(statusText) es := util.DeriveEmojisFromText(statusText)