option for SearchAll, remove big games from game searches, also chang…

…ed behavior of GameArtOverride

libStreamDetective/config.py

@@ -36,6 +36,7 @@
         "GameName": {"type": "string"},
         "UserName": {"type": "string"},
         "SearchTags": {"type": "array"},
+        "SearchAll": {"type": "boolean"},
         "filters": {"type": "array", "items":filters_schema},
         "Notifications": {"type": ["array", "object"]},
         "CustomDiscordMessage": {"type": "string"},
@@ -82,8 +83,9 @@ def validateConfig(conf):
     assert conf.get('NotificationServices'), 'config has NotificationServices'
 
     for search in conf.get('Searches',[]):
-        # Must have one but not both
-        assert ("GameName" in search) ^ ("UserName" in search) ^ ("SearchTags" in search), 'testing config for search: ' + repr(search)
+        # Must have only one type
+        type = set(('GameName','UserName','SearchTags','SearchAll')).intersection(search)
+        assert len(type)==1, 'testing config for search: ' + repr(search)
 
     for service in conf.get('NotificationServices',[]):
         assert service.get("ProfileName"), 'testing notification service for: ' + repr(service)

libStreamDetective/filters.py

@@ -2,19 +2,24 @@
 from libStreamDetective.util import *
 
 def CheckStream(entry, streamer, title, tags, gameName):
-    trace("")
-    trace("Name: ", streamer, title, tags, gameName, entry)
+    if 'SearchAll' in entry:
+        ttrace=trace
+    else:
+        ttrace = lambda *a: None # do-nothing function
+
+    ttrace("")
+    ttrace("Name: ", streamer, title, tags, gameName, entry)
 
     if gameName != entry.get('GameName', gameName):
-        trace("did not match GameName")
+        ttrace("did not match GameName")
         return False
     if streamer != entry.get('UserName', streamer):
-        trace("did not match UserName")
+        ttrace("did not match UserName")
         return False
 
     if not entry.get('filters'):
         # return True if the filters array is empty, or the key is missing
-        trace("no filters, accepting stream")
+        ttrace("no filters, accepting stream")
         return True
 
     if tags:
@@ -24,8 +29,8 @@ def CheckStream(entry, streamer, title, tags, gameName):
         if CheckStreamFilter(filter, streamer, title, tags, gameName):
             debug(streamer, title, tags, gameName, "accepted by filter", filter)
             return True
-        trace(streamer, "not accepted by filter", filter)
-    debug(streamer, "not accepted by any filters")
+        ttrace(streamer, "not accepted by filter", filter)
+    ttrace(streamer, "not accepted by any filters")
     return False
 
 

libStreamDetective/libStreamDetective.py

@@ -62,10 +62,10 @@ def FetchAllStreams(self):
                 searchProviders.AddGame(search['GameName'])
             elif 'UserName' in search:
                 searchProviders.AddUser(search['UserName'])
-            elif 'SearchTags' in search:
-                searchProviders.AddTags(search['SearchTags'])
+            elif search.get('SearchAll') or search.get('SearchTags'): # this one is a boolean
+                searchProviders.SearchAll()
 
-        (self.fetchedGames, self.fetchedTags, self.fetchedStreamers) = searchProviders.FetchAllStreams()
+        (self.fetchedGames, self.fetchedAll, self.fetchedStreamers) = searchProviders.FetchAllStreams()
 
 
     def CheckSingleUser(self, user):# for CLI
@@ -82,7 +82,7 @@ def CheckSingleUser(self, user):# for CLI
 
         searchProviders = AllProviders(self.config)
         searchProviders.AddUser(user)
-        (fetchedGames, fetchedTags, fetchedStreamers) = searchProviders.FetchAllStreams()
+        (fetchedGames, fetchedAll, fetchedStreamers) = searchProviders.FetchAllStreams()
 
         if user in fetchedStreamers:
             print('found', user, s)
@@ -153,7 +153,10 @@ def HandleSearches(self):
                 streams = self.GetAllStreamerStreams(search['UserName'])
             elif "SearchTags" in search:
                 debug('Handling', search['SearchTags'])
-                streams = self.GetAllTagsStreams(search['SearchTags'])
+                streams = self.GetAllStreams()
+            elif search.get('SearchAll'): # this one is a boolean
+                debug('Handling search all', search.get('filters'))
+                streams = self.GetAllStreams()
             else:
                 print('unknown search type', search)
                 continue
@@ -166,9 +169,8 @@ def HandleSearches(self):
     def GetAllGameStreams(self,gameName) -> list:
         return self.fetchedGames.get(gameName.lower(),[])
 
-    def GetAllTagsStreams(self,tags) -> list:
-        tags.sort()
-        return self.fetchedTags.get(' '.join(tags).lower(),[])
+    def GetAllStreams(self) -> list:
+        return self.fetchedAll
 
     def GetAllStreamerStreams(self,streamer) -> list:
         streamer = streamer.lower()

libStreamDetective/notifiers.py

@@ -39,8 +39,11 @@ def handleSingleNotificationService(self, notifierData, entry, newStreams):
 
         if notifierData and notifierData.get('chance', 100) < 100:
             if notifierData['chance'] < random.randint(1, 100):
+                print(self.ProfileName, 'lost the lottery:', newStreams)
                 return
-            newStreams = random.sample(newStreams, 1) # the chance property can be misleading, because even 99% means a maximum of 1 per group
+            winners = random.sample(newStreams, 1) # the chance property can be misleading, because even 99% means a maximum of 1 per group
+            print(self.ProfileName, winners, 'won out of', newStreams)
+            newStreams = winners
         else:
             random.shuffle(newStreams)
 
@@ -158,7 +161,7 @@ def buildDiscordMsgs(self, discordProfile, toSend, atUserId, titleOverride, game
 
             gameArtUrl = ''
             try:
-                gameArtName = gameName
+                gameArtName = stream["game_name"]
                 if gameArtOverride:
                     gameArtName = gameArtOverride
                 gameArtUrl = TwitchApi.GetGameArt(gameArtName)

libStreamDetective/searchProviders.py

@@ -7,6 +7,7 @@ def __init__(self, config):
         self.users = set()
         self.searches = set()
         self.tagsets = dict()
+        self.searchAll = False
 
     def AddGame(self, game:str):
         self.games.add(game)
@@ -16,25 +17,29 @@ def AddUser(self, user:str):
 
     def AddSearch(self, query:str):
         self.searches.add(query)
+        self.searchAll = True
 
     def AddTags(self, tags:list):
         tags.sort()
         self.tagsets[' '.join(tags).lower()] = tags
+        self.searchAll = True
+
+    def SearchAll(self):
+        self.searchAll = True
 
     def FetchAllStreams(self):
         allGames = {}
-        allTags = {}
+        all = []
         allStreamers = {}
         for p in self.providers:
-            (games, tags, streamers) = p.FetchAllStreams(self.games, self.tagsets, self.users)
+            (games, all_temp, streamers) = p.FetchAllStreams(self.games, self.searchAll, self.users)
 
             for (k,v) in games.items():
                 allGames[k] = v
 
-            for (k,v) in tags.items():
-                allTags[k] = v
+            all += all_temp
 
             for (k,v) in streamers.items():
                 allStreamers[k] = v
 
-        return (allGames, allTags, allStreamers)
+        return (allGames, all, allStreamers)

libStreamDetective/searches.py

@@ -6,18 +6,22 @@
 def HandleFilters(self, search, allStreams):
     newStreams = []
     now = datetime.now()
+    searchAll = 'SearchAll' in search
+    searchTags = search.get('SearchTags')
 
     for stream in allStreams:
         streamer = stream['user_login']
         title = stream['title']
         tags = stream['tags']
         stream['last_seen'] = now.isoformat()
+        if searchTags and not MatchAnyTag(searchTags, tags): # for now, this makes it easier to require different combos of tags, as seen in DosSpeedruns.json
+            continue
         matched = self.CheckStream(search, streamer, title, tags, stream["game_name"])
         if matched:
             debug("matched "+streamer)
             stream['last_matched'] = now.isoformat()
             newStreams.append(stream)
-        else:
+        elif not searchAll:
             trace('didn\'t match', streamer)
 
     # All stream info now retrieved
@@ -27,3 +31,28 @@ def HandleFilters(self, search, allStreams):
 
     debug("\n\n")
     return newStreams
+
+
+def MatchAllTags(desiredTags, actualTags):
+    try:
+        for desiredTag in desiredTags:
+            matched = False
+            for actualTag in actualTags:
+                if desiredTag.lower() == actualTag.lower():
+                    matched = True
+                    break
+            if not matched:
+                return False
+        return True
+    except:
+        return False
+
+def MatchAnyTag(desiredTags, actualTags):
+    try:
+        for desiredTag in desiredTags:
+            for actualTag in actualTags:
+                if desiredTag.lower() == actualTag.lower():
+                    return True
+        return False
+    except:
+        return False

libStreamDetective/twitch.py

@@ -26,87 +26,87 @@ def __init__(self, config):
         accessToken = config['accessToken']
 
 
-    def FetchAllStreams(self, gameNames:set, tagsets:dict, streamers:set):
-        #print("All Games: "+str(gameNames))
-        #print("All Streamers: "+str(streamers))
+    def FetchAllStreams(self, gameNames:set, searchAll:bool, streamers:set):
         fetchedGames = {}
-        fetchedTags = {}
+        fetchedAll = []
         fetchedStreamers = {}
+
+        bigGames = set(('Retro', 'StarCraft II'))
+        gameNames = gameNames.difference(bigGames)
+
+        if gameNames:
+            fetchedGames = self.FetchAllGames(gameNames)
+        if searchAll:
+            fetchedAll = self.FetchAll()
+        if streamers:
+            fetchedStreamers = self.FetchAllStreamers(streamers)
 
+        lowerBigGames = set()
+        for GAME in bigGames:
+            g = GAME.lower()
+            lowerBigGames.add(g)
+            if g not in fetchedGames:
+                fetchedGames[g] = []
+        for stream in fetchedAll: # grab large games from the fetchAll
+            game = stream["game_name"].lower()
+            if game not in lowerBigGames:
+                continue
+            fetchedGames[game].append(stream)
+
+        self.end() # print some text
+        return (fetchedGames, fetchedAll, fetchedStreamers)
+
+
+    def FetchAllGames(self, names:set):
+        fetched:dict[str,list] = {}
+        allGamesUrl = TwitchApi.streamsUrl
         # TODO: This should be extended to handle more than 100 unique games
-        if gameNames:
-            allGamesUrl = TwitchApi.streamsUrl
-            for game in gameNames:
-                gameId = TwitchApi.GetGameId(game)
-                allGamesUrl += "game_id="+gameId+"&"
-            #print("All games: "+allGamesUrl)
-            fetched = self.GetAllStreams(allGamesUrl)
-            for stream in fetched:
-                gameName = stream["game_name"].lower()
-                if gameName not in fetchedGames:
-                    fetchedGames[gameName] = []
-                if stream not in fetchedGames[gameName]:
-                    fetchedGames[gameName].append(stream)
-                user = stream["user_login"].lower()
-                fetchedStreamers[user] = stream
-
-        fetched = None # just in case
-        if tagsets:
-            url = TwitchApi.streamsUrl
-            fetched = self.GetAllStreams(url)
+        for game in names:
+            gameId = TwitchApi.GetGameId(game)
+            allGamesUrl += "game_id="+gameId+"&"
+        #print("All games: "+allGamesUrl)
+        res = self.GetAllPages(allGamesUrl)
+        for stream in res:
+            game = stream["game_name"].lower()
+            if game not in fetched:
+                fetched[game] = []
+            if stream not in fetched[game]:
+                fetched[game].append(stream)
+        return fetched
+
+    def FetchAll(self):
+        url = TwitchApi.streamsUrl
+        res = self.GetAllPages(url)
+        return res
+
+    def FetchAllTags(self, tagsets:dict):
+        fetched:dict[str,list] = {}
+        url = TwitchApi.streamsUrl
+        res = self.GetAllPages(url)
         for (k,v) in tagsets.items():
-            if k not in fetchedTags:
-                fetchedTags[k] = []
-            for stream in fetched:
-                user = stream["user_login"].lower()
-                fetchedStreamers[user] = stream
-                if stream in fetchedTags[k]:
+            if k not in fetched:
+                fetched[k] = []
+            for stream in res:
+                if stream in fetched[k]:
                     continue
                 if self.MatchAnyTag(v, stream['tags']):
-                    fetchedTags[k].append(stream)
-
-        # TODO: This should be extended to handle more than 100 unique streamers
-        if streamers:
-            allStreamersUrl = TwitchApi.streamsUrl
-            for streamer in streamers:
-                if streamer.lower() not in fetchedStreamers: # don't need to fetch them if we found them above
-                    allStreamersUrl += "user_login="+streamer+"&"
-            fetched = self.GetAllStreams(allStreamersUrl)
-            for stream in fetched:
-                user = stream["user_login"].lower()
-                fetchedStreamers[user] = stream
-
-        self.end()
-        return (fetchedGames, fetchedTags, fetchedStreamers)
+                    fetched[k].append(stream)
+        return fetched
 
+    def FetchAllStreamers(self, streamers:set):
+        fetched = {}
+        # TODO: This should be extended to handle more than 100 unique streamers
+        allStreamersUrl = TwitchApi.streamsUrl
+        for streamer in streamers:
+            allStreamersUrl += "user_login="+streamer+"&"
+        res = self.GetAllPages(allStreamersUrl)
+        for stream in res:
+            user = stream["user_login"].lower()
+            fetched[user] = stream
+        return fetched
 
-    @staticmethod
-    def MatchAllTags(desiredTags, actualTags):
-        try:
-            for desiredTag in desiredTags:
-                matched = False
-                for actualTag in actualTags:
-                    if desiredTag.lower() == actualTag.lower():
-                        matched = True
-                        break
-                if not matched:
-                    return False
-            return True
-        except:
-            return False
-
-    @staticmethod
-    def MatchAnyTag(desiredTags, actualTags):
-        try:
-            for desiredTag in desiredTags:
-                for actualTag in actualTags:
-                    if desiredTag.lower() == actualTag.lower():
-                        return True
-            return False
-        except:
-            return False
 
-    def GetAllStreams(self, lookupUrl, maxPages=100):
+    def GetAllPages(self, lookupUrl, maxPages=100):
         allStreams = []
         cursor = ""
         resume_page = 0