From 6bc9463999d2b83ba25977dc2c5e21469aaa462c Mon Sep 17 00:00:00 2001 From: Rodrigo Rodrigues Date: Tue, 24 Apr 2018 19:30:52 -0300 Subject: [PATCH 1/3] Added isReply and replyTo in tweets --- got/manager/TweetManager.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/got/manager/TweetManager.py b/got/manager/TweetManager.py index 63d29dd1..529ce58c 100644 --- a/got/manager/TweetManager.py +++ b/got/manager/TweetManager.py @@ -61,6 +61,12 @@ def getTweets(tweetCriteria, receiveBuffer=None, bufferLength=100, proxy=None): tweet.mentions = " ".join(re.compile('(@\\w*)').findall(tweet.text)) tweet.hashtags = " ".join(re.compile('(#\\w*)').findall(tweet.text)) tweet.geo = geo + + tweet.isReply = tweetPQ("div.ReplyingToContextBelowAuthor").is_("div") + if tweet.isReply: + tweet.replyTo = tweetPQ("div.ReplyingToContextBelowAuthor span.username b").contents()[0] + else: + tweet.replyTo = '' results.append(tweet) resultsAux.append(tweet) From 589ea1c4cfa6da169307521dce2c6f22e2de8c1f Mon Sep 17 00:00:00 2001 From: Rodrigo Rodrigues Date: Tue, 24 Apr 2018 19:54:05 -0300 Subject: [PATCH 2/3] Implemented NestLakerJasonLIN fix for HashTags and Mentions --- got/manager/TweetManager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/got/manager/TweetManager.py b/got/manager/TweetManager.py index 529ce58c..3915a492 100644 --- a/got/manager/TweetManager.py +++ b/got/manager/TweetManager.py @@ -58,8 +58,8 @@ def getTweets(tweetCriteria, receiveBuffer=None, bufferLength=100, proxy=None): tweet.date = datetime.datetime.fromtimestamp(dateSec) tweet.retweets = retweets tweet.favorites = favorites - tweet.mentions = " ".join(re.compile('(@\\w*)').findall(tweet.text)) - tweet.hashtags = " ".join(re.compile('(#\\w*)').findall(tweet.text)) + tweet.mentions = " ".join(re.compile(r'(@\s\w*)').findall(tweet.text)) + tweet.hashtags = " ".join(re.compile(r'(#\s\w*)').findall(tweet.text)) tweet.geo = geo tweet.isReply = tweetPQ("div.ReplyingToContextBelowAuthor").is_("div") From e61d444a4c3f18082066ed727289b0dcef96870a Mon Sep 17 00:00:00 2001 From: Rodrigo Rodrigues Date: Tue, 24 Apr 2018 20:07:21 -0300 Subject: [PATCH 3/3] Fixed Mentions and Hashtags in Text and in Tweet Fields --- got/manager/TweetManager.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/got/manager/TweetManager.py b/got/manager/TweetManager.py index 3915a492..82564dbc 100644 --- a/got/manager/TweetManager.py +++ b/got/manager/TweetManager.py @@ -39,7 +39,9 @@ def getTweets(tweetCriteria, receiveBuffer=None, bufferLength=100, proxy=None): tweet = models.Tweet() usernameTweet = tweetPQ("span:first.username.u-dir b").text() - txt = re.sub(r"\s+", " ", tweetPQ("p.js-tweet-text").text().replace('# ', '#').replace('@ ', '@')) + txt = re.sub(r"\s+", " ", tweetPQ("p.js-tweet-text").text()) + txt = re.sub(r'#\s*', '#',txt) + txt = re.sub(r'@\s*', '@',txt) retweets = int(tweetPQ("span.ProfileTweet-action--retweet span.ProfileTweet-actionCount").attr("data-tweet-stat-count").replace(",", "")) favorites = int(tweetPQ("span.ProfileTweet-action--favorite span.ProfileTweet-actionCount").attr("data-tweet-stat-count").replace(",", "")) dateSec = int(tweetPQ("small.time span.js-short-timestamp").attr("data-time")) @@ -58,8 +60,8 @@ def getTweets(tweetCriteria, receiveBuffer=None, bufferLength=100, proxy=None): tweet.date = datetime.datetime.fromtimestamp(dateSec) tweet.retweets = retweets tweet.favorites = favorites - tweet.mentions = " ".join(re.compile(r'(@\s\w*)').findall(tweet.text)) - tweet.hashtags = " ".join(re.compile(r'(#\s\w*)').findall(tweet.text)) + tweet.mentions = " ".join(re.compile('(@\\w*)').findall(tweet.text)) + tweet.hashtags = " ".join(re.compile('(#\\w*)').findall(tweet.text)) tweet.geo = geo tweet.isReply = tweetPQ("div.ReplyingToContextBelowAuthor").is_("div")