diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go
index d17e4875b13a7..6dc2e813e9b82 100644
--- a/routers/api/packages/api.go
+++ b/routers/api/packages/api.go
@@ -115,6 +115,7 @@ func CommonRoutes() *web.Router {
 
 	verifyAuth(r, []auth.Method{
 		&auth.OAuth2{},
+		&auth.AccessToken{},
 		&auth.Basic{},
 		&nuget.Auth{},
 		&conan.Auth{},
@@ -671,6 +672,7 @@ func ContainerRoutes() *web.Router {
 	r.Use(context.PackageContexter())
 
 	verifyAuth(r, []auth.Method{
+		&auth.AccessToken{},
 		&auth.Basic{},
 		&container.Auth{},
 	})
diff --git a/routers/api/packages/chef/auth.go b/routers/api/packages/chef/auth.go
index a790e9a3631b6..8ed4f03993b24 100644
--- a/routers/api/packages/chef/auth.go
+++ b/routers/api/packages/chef/auth.go
@@ -52,6 +52,10 @@ func (a *Auth) Name() string {
 	return "chef"
 }
 
+func (a *Auth) Match(req *http.Request) bool {
+	return true
+}
+
 // Verify extracts the user from the signed request
 // If the request is signed with the user private key the user is verified.
 func (a *Auth) Verify(req *http.Request, w http.ResponseWriter, store auth.DataStore, sess auth.SessionStore) (*user_model.User, error) {
diff --git a/routers/api/packages/conan/auth.go b/routers/api/packages/conan/auth.go
index 9c03d01391fc2..7a7703c5d8a52 100644
--- a/routers/api/packages/conan/auth.go
+++ b/routers/api/packages/conan/auth.go
@@ -20,6 +20,10 @@ func (a *Auth) Name() string {
 	return "conan"
 }
 
+func (a *Auth) Match(req *http.Request) bool {
+	return true
+}
+
 // Verify extracts the user from the Bearer token
 func (a *Auth) Verify(req *http.Request, w http.ResponseWriter, store auth.DataStore, sess auth.SessionStore) (*user_model.User, error) {
 	packageMeta, err := packages.ParseAuthorizationRequest(req)
diff --git a/routers/api/packages/container/auth.go b/routers/api/packages/container/auth.go
index 1d8ae6af7d7a9..01f273ddf6fcf 100644
--- a/routers/api/packages/container/auth.go
+++ b/routers/api/packages/container/auth.go
@@ -20,6 +20,10 @@ func (a *Auth) Name() string {
 	return "container"
 }
 
+func (a *Auth) Match(req *http.Request) bool {
+	return true
+}
+
 // Verify extracts the user from the Bearer token
 // If it's an anonymous session a ghost user is returned
 func (a *Auth) Verify(req *http.Request, w http.ResponseWriter, store auth.DataStore, sess auth.SessionStore) (*user_model.User, error) {
diff --git a/routers/api/packages/nuget/auth.go b/routers/api/packages/nuget/auth.go
index e81ad01b2b39d..20dff9c46c30c 100644
--- a/routers/api/packages/nuget/auth.go
+++ b/routers/api/packages/nuget/auth.go
@@ -21,6 +21,10 @@ func (a *Auth) Name() string {
 	return "nuget"
 }
 
+func (a *Auth) Match(req *http.Request) bool {
+	return true
+}
+
 // https://docs.microsoft.com/en-us/nuget/api/package-publish-resource#request-parameters
 func (a *Auth) Verify(req *http.Request, w http.ResponseWriter, store auth.DataStore, sess auth.SessionStore) (*user_model.User, error) {
 	token, err := auth_model.GetAccessTokenBySHA(req.Context(), req.Header.Get("X-NuGet-ApiKey"))
diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go
index 23f466873bad9..e623f1b08d92c 100644
--- a/routers/api/v1/api.go
+++ b/routers/api/v1/api.go
@@ -739,6 +739,7 @@ func buildAuthGroup() *auth.Group {
 	group := auth.NewGroup(
 		&auth.OAuth2{},
 		&auth.HTTPSign{},
+		&auth.AccessToken{},
 		&auth.Basic{}, // FIXME: this should be removed once we don't allow basic auth in API
 	)
 	if setting.Service.EnableReverseProxyAuthAPI {
diff --git a/routers/web/web.go b/routers/web/web.go
index 137c67730652d..87036f08f2b79 100644
--- a/routers/web/web.go
+++ b/routers/web/web.go
@@ -99,7 +99,8 @@ func optionsCorsHandler() func(next http.Handler) http.Handler {
 func buildAuthGroup() *auth_service.Group {
 	group := auth_service.NewGroup()
 	group.Add(&auth_service.OAuth2{}) // FIXME: this should be removed and only applied in download and oauth related routers
-	group.Add(&auth_service.Basic{})  // FIXME: this should be removed and only applied in download and git/lfs routers
+	group.Add(&auth_service.AccessToken{})
+	group.Add(&auth_service.Basic{}) // FIXME: this should be removed and only applied in download and git/lfs routers
 
 	if setting.Service.EnableReverseProxyAuth {
 		group.Add(&auth_service.ReverseProxy{}) // reverseproxy should before Session, otherwise the header will be ignored if user has login
diff --git a/services/auth/access_token.go b/services/auth/access_token.go
new file mode 100644
index 0000000000000..53195cfd72031
--- /dev/null
+++ b/services/auth/access_token.go
@@ -0,0 +1,113 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package auth
+
+import (
+	"net/http"
+	"strings"
+
+	auth_model "code.gitea.io/gitea/models/auth"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/timeutil"
+	"code.gitea.io/gitea/modules/web/middleware"
+)
+
+// Ensure the struct implements the interface.
+var (
+	_ Method = &AccessToken{}
+)
+
+// BasicMethodName is the constant name of the basic authentication method
+const (
+	AccessTokenMethodName = "access_token"
+)
+
+// AccessToken implements the Auth interface and authenticates requests (API requests
+// only) by looking for access token
+type AccessToken struct{}
+
+// Name represents the name of auth method
+func (b *AccessToken) Name() string {
+	return AccessTokenMethodName
+}
+
+// Match returns true if the request matched AccessToken requirements
+// TODO: remove path check once AccessToken will not be a global middleware but only
+// for specific routes
+func (b *AccessToken) Match(req *http.Request) bool {
+	if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawOrAttachOrLFSPath(req) {
+		return false
+	}
+	baHead := req.Header.Get("Authorization")
+	if baHead == "" {
+		return false
+	}
+	auths := strings.SplitN(baHead, " ", 2)
+	if len(auths) != 2 || (strings.ToLower(auths[0]) != "basic") {
+		return false
+	}
+	return true
+}
+
+// Verify extracts and validates Basic data (username and password/token) from the
+// "Authorization" header of the request and returns the corresponding user object for that
+// name/token on successful validation.
+// Returns nil if header is empty or validation fails.
+func (b *AccessToken) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) (*user_model.User, error) {
+	// Basic authentication should only fire on API, Download or on Git or LFSPaths
+	if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawOrAttachOrLFSPath(req) {
+		return nil, nil
+	}
+
+	baHead := req.Header.Get("Authorization")
+	if len(baHead) == 0 {
+		return nil, nil
+	}
+
+	auths := strings.SplitN(baHead, " ", 2)
+	if len(auths) != 2 || (strings.ToLower(auths[0]) != "basic") {
+		return nil, nil
+	}
+
+	uname, passwd, _ := base.BasicAuthDecode(auths[1])
+
+	// Check if username or password is a token
+	isUsernameToken := len(passwd) == 0 || passwd == "x-oauth-basic"
+	// Assume username is token
+	authToken := uname
+	if !isUsernameToken {
+		log.Trace("Basic Authorization: Attempting login for: %s", uname)
+		// Assume password is token
+		authToken = passwd
+	} else {
+		log.Trace("Basic Authorization: Attempting login with username as token")
+	}
+
+	// check personal access token
+	token, err := auth_model.GetAccessTokenBySHA(req.Context(), authToken)
+	if err == nil {
+		log.Trace("Basic Authorization: Valid AccessToken for user[%d]", token.UID)
+		u, err := user_model.GetUserByID(req.Context(), token.UID)
+		if err != nil {
+			log.Error("GetUserByID:  %v", err)
+			return nil, err
+		}
+
+		token.UpdatedUnix = timeutil.TimeStampNow()
+		if err = auth_model.UpdateAccessToken(req.Context(), token); err != nil {
+			log.Error("UpdateAccessToken:  %v", err)
+		}
+
+		store.GetData()["LoginMethod"] = AccessTokenMethodName
+		store.GetData()["IsApiToken"] = true
+		store.GetData()["ApiTokenScope"] = token.Scope
+		return u, nil
+	} else if !auth_model.IsErrAccessTokenNotExist(err) && !auth_model.IsErrAccessTokenEmpty(err) {
+		log.Error("GetAccessTokenBySha: %v", err)
+	}
+
+	return nil, nil
+}
diff --git a/services/auth/basic.go b/services/auth/basic.go
index 90bd64237091d..22bce55173d04 100644
--- a/services/auth/basic.go
+++ b/services/auth/basic.go
@@ -14,7 +14,6 @@ import (
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/modules/web/middleware"
 )
@@ -27,7 +26,6 @@ var (
 // BasicMethodName is the constant name of the basic authentication method
 const (
 	BasicMethodName       = "basic"
-	AccessTokenMethodName = "access_token"
 	OAuth2TokenMethodName = "oauth2_token"
 	ActionTokenMethodName = "action_token"
 )
@@ -42,6 +40,10 @@ func (b *Basic) Name() string {
 	return BasicMethodName
 }
 
+func (b *Basic) Match(req *http.Request) bool {
+	return true
+}
+
 // Verify extracts and validates Basic data (username and password/token) from the
 // "Authorization" header of the request and returns the corresponding user object for that
 // name/token on successful validation.
@@ -92,29 +94,6 @@ func (b *Basic) Verify(req *http.Request, w http.ResponseWriter, store DataStore
 		return u, nil
 	}
 
-	// check personal access token
-	token, err := auth_model.GetAccessTokenBySHA(req.Context(), authToken)
-	if err == nil {
-		log.Trace("Basic Authorization: Valid AccessToken for user[%d]", uid)
-		u, err := user_model.GetUserByID(req.Context(), token.UID)
-		if err != nil {
-			log.Error("GetUserByID:  %v", err)
-			return nil, err
-		}
-
-		token.UpdatedUnix = timeutil.TimeStampNow()
-		if err = auth_model.UpdateAccessToken(req.Context(), token); err != nil {
-			log.Error("UpdateAccessToken:  %v", err)
-		}
-
-		store.GetData()["LoginMethod"] = AccessTokenMethodName
-		store.GetData()["IsApiToken"] = true
-		store.GetData()["ApiTokenScope"] = token.Scope
-		return u, nil
-	} else if !auth_model.IsErrAccessTokenNotExist(err) && !auth_model.IsErrAccessTokenEmpty(err) {
-		log.Error("GetAccessTokenBySha: %v", err)
-	}
-
 	// check task token
 	task, err := actions_model.GetRunningTaskByToken(req.Context(), authToken)
 	if err == nil && task != nil {
diff --git a/services/auth/group.go b/services/auth/group.go
index aecf43cb24402..b9339dc80a631 100644
--- a/services/auth/group.go
+++ b/services/auth/group.go
@@ -41,10 +41,22 @@ func (b *Group) Name() string {
 	return strings.Join(names, ",")
 }
 
+func (b *Group) Match(req *http.Request) bool {
+	return true
+}
+
 func (b *Group) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) (*user_model.User, error) {
-	// Try to sign in with each of the enabled plugins
-	var retErr error
+	// find all methods that match the request
+	matchedMethods := make([]Method, 0, len(b.methods))
 	for _, m := range b.methods {
+		if m.Match(req) {
+			matchedMethods = append(matchedMethods, m)
+		}
+	}
+
+	var retErr error
+	// Try to sign in with each of the matched plugins
+	for _, m := range matchedMethods {
 		user, err := m.Verify(req, w, store, sess)
 		if err != nil {
 			if retErr == nil {
diff --git a/services/auth/httpsign.go b/services/auth/httpsign.go
index 83a36bef238c4..9a8b9d77ccc3b 100644
--- a/services/auth/httpsign.go
+++ b/services/auth/httpsign.go
@@ -36,6 +36,10 @@ func (h *HTTPSign) Name() string {
 	return "httpsign"
 }
 
+func (h *HTTPSign) Match(req *http.Request) bool {
+	return true
+}
+
 // Verify extracts and validates HTTPsign from the Signature header of the request and returns
 // the corresponding user object on successful validation.
 // Returns nil if header is empty or validation fails.
diff --git a/services/auth/interface.go b/services/auth/interface.go
index ece28af12d1be..f2e6fceeb3b56 100644
--- a/services/auth/interface.go
+++ b/services/auth/interface.go
@@ -24,10 +24,12 @@ type Method interface {
 	// If verification is successful returns either an existing user object (with id > 0)
 	// or a new user object (with id = 0) populated with the information that was found
 	// in the authentication data (username or email).
-	// Second argument returns err if verification fails, otherwise
+	// Third argument returns err if verification fails, otherwise
+	// Second return argument returns true
 	// First return argument returns nil if no matched verification condition
 	Verify(http *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) (*user_model.User, error)
-
+	// Match returns true if the request is a match for this method
+	Match(*http.Request) bool
 	Name() string
 }
 
diff --git a/services/auth/oauth2.go b/services/auth/oauth2.go
index d0aec085b107d..07d8bcb6023c0 100644
--- a/services/auth/oauth2.go
+++ b/services/auth/oauth2.go
@@ -131,6 +131,19 @@ func (o *OAuth2) userIDFromToken(ctx context.Context, tokenSHA string, store Dat
 	return t.UID
 }
 
+// Match returns true if the request matched OAuth2 requirements
+// TODO: remove path check once AccessToken will not be a global middleware but only
+// for specific routes
+func (o *OAuth2) Match(req *http.Request) bool {
+	if !middleware.IsAPIPath(req) && !isAttachmentDownload(req) && !isAuthenticatedTokenRequest(req) &&
+		!isGitRawOrAttachPath(req) && !isArchivePath(req) {
+		return false
+	}
+
+	_, ok := parseToken(req)
+	return ok
+}
+
 // Verify extracts the user ID from the OAuth token in the query parameters
 // or the "Authorization" header and returns the corresponding user object for that ID.
 // If verification is successful returns an existing user object.
diff --git a/services/auth/reverseproxy.go b/services/auth/reverseproxy.go
index 36b4ef68f42f4..4479fa60f18e4 100644
--- a/services/auth/reverseproxy.go
+++ b/services/auth/reverseproxy.go
@@ -100,6 +100,10 @@ func (r *ReverseProxy) getUserFromAuthEmail(req *http.Request) *user_model.User
 	return user
 }
 
+func (r *ReverseProxy) Match(req *http.Request) bool {
+	return true
+}
+
 // Verify attempts to load a user object based on headers sent by the reverse proxy.
 // First it will attempt to load it based on the username (see docs for getUserFromAuthUser),
 // and failing that it will attempt to load it based on the email (see docs for getUserFromAuthEmail).
diff --git a/services/auth/session.go b/services/auth/session.go
index 35d97e42da149..98d8d11ca8487 100644
--- a/services/auth/session.go
+++ b/services/auth/session.go
@@ -24,6 +24,10 @@ func (s *Session) Name() string {
 	return "session"
 }
 
+func (s *Session) Match(req *http.Request) bool {
+	return true
+}
+
 // Verify checks if there is a user uid stored in the session and returns the user
 // object for that uid.
 // Returns nil if there is no user uid stored in the session.
diff --git a/services/auth/sspi.go b/services/auth/sspi.go
index 7f8a03a4c67da..648fbc617bb80 100644
--- a/services/auth/sspi.go
+++ b/services/auth/sspi.go
@@ -54,6 +54,10 @@ func (s *SSPI) Name() string {
 	return "sspi"
 }
 
+func (s *SSPI) Match(req *http.Request) bool {
+	return true
+}
+
 // Verify uses SSPI (Windows implementation of SPNEGO) to authenticate the request.
 // If authentication is successful, returns the corresponding user object.
 // If negotiation should continue or authentication fails, immediately returns a 401 HTTP