Skip to content

Commit

Permalink
New threading architecture (#8)
Browse files Browse the repository at this point in the history 
* fixed formatting

* initial refactor

* proper closing, docstrings, and some clean up

* last bit of clean up

* updated README

* updated some comments

* explicit continue after pong
  • Loading branch information
@tnibert
tnibert authored Mar 21, 2020
1 parent dd33a10 commit 38d55f3
Show file tree
Hide file tree
Showing 2 changed files with 204 additions and 132 deletions.
7 changes: 5 additions & 2 deletions README
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Abunai version 2.1
Abunai version 2.2
Author: Timothy Nibert
Iron Lotus Studios

Expand All @@ -10,7 +10,7 @@ Dependencies:
Python, TextBlob

About:
Abunai is language translation liaison bot written in Python.
Abunai is an IRC bot for real time language translation written in Python.

Usage:

Expand All @@ -22,3 +22,6 @@ languages must be given in two letter codes, e.g. en, es, etc
to use:
All messages from the CHANNEL will be sent to USERNICK in a private message
USERNICK can respond to the CHANNEL by private messaging the bot

to quit:
At the console, type q and then <enter>
329 changes: 199 additions & 130 deletions abunai.py
View file
Original file line number Diff line number Diff line change
@@ -1,64 +1,84 @@
#! /usr/bin/env python
# Abunai v2.1
# an irc translation bot script
# Abunai v2.2
# IRC translator bot
# call with python abunai.py server nick channel USER USERLANGUAGE CHANNELLANGUAGE
# do not include the # sign in the channel arg
# this script won't play nicely with servers that need nickserv as is
# as is, this script won't play nicely with servers that need nickserv or sasl
# quit by entering q at the terminal

import sys
import socket
from threading import Thread
import traceback
from queue import Queue
from threading import Thread, Lock
from textblob import TextBlob

"""
Threading:
We have three types of threads and two queues
- main thread (main program execution) - reads from queues, cleans queues, sends all messages
- listen thread - listens to socket, launches an appropriate translation thread, or if we get a ping sets pong flag true
- translation threads - launched by listen thread, does translation, appends the translated message to the msgqueue
We have one main thread and one listen thread, but we can have many translation threads going at one time.
threads variable is the thread queue, main thread checks if each thread is finished and removes them if they are
msgqueue variable is the message queue (outgoing messages), main thread checks msgqueue, sends messages, and removes them from the msgqueue
Threading Architecture:
We have four threads and two queues
Threads:
- main thread (main program execution) - sets up everything, waits for user to input 'q' to quit
- listen thread - listens to socket, responds to PING, queues messages for translation
- translation thread - reads from queue, does translation, queues messages for sending
- send thread - reads from queue, sends message to server
Queues:
- inmsgqueue - bridges messages from listen thread to translation thread for translation
- outmsgqueue - bridges messages from translation thread to send thread for sending back to server
"""

# declare variables for where bot goes
# settings for bot from command invocation
try:
HOST=sys.argv[1]
PORT=6667
NICK=sys.argv[2]
IDENT=sys.argv[2]
REALNAME=sys.argv[2]
CHAN="#" + sys.argv[3]
USER=sys.argv[4]
userlang=sys.argv[5]
chanlang=sys.argv[6]
HOST = sys.argv[1]
PORT = 6667
NICK = sys.argv[2]
IDENT = sys.argv[2]
REALNAME = sys.argv[2]
CHAN = "#" + sys.argv[3]
USER = sys.argv[4]
userlang = sys.argv[5]
chanlang = sys.argv[6]
except:
print("invalid args")
print("Usage: python abunai.py SERVER NICK CHANNEL USERNICK USERLANGUAGE CHANNELLANGUAGE")
print("do not include the # in the channel name")
exit()
print("invalid args")
print("Usage: python abunai.py SERVER NICK CHANNEL USERNICK USERLANGUAGE CHANNELLANGUAGE")
print("do not include the # in the channel name")
exit()

# if you want to manually set the info...
#HOST=""
#PORT=6667
#NICK="Bot"
#IDENT="Bot"
#REALNAME="Bot"
#CHAN="#abu"
# HOST="irc.whatever.net"
# PORT=6667
# NICK="Bot"
# IDENT="Bot"
# REALNAME="Bot"
# CHAN="#abu"
# USER="myusername"
# userlang = "en"
# chanlang = "es"

DEBUG = False
# this marker will be sent through the queues to stop the consumer threads
END = "quit"

threads = []
msgqueue = []
# when we connect to a network, it will probably forward us to a node
# indivserver is the node address preceeded with a : for ping pong
indivserver = ""
PONG = False
inmsgqueue = Queue()
outmsgqueue = Queue()

# I don't think this lock is really necessary
# because I tested making simultaneous send() calls from
# multiple threads to netcat -l and nothing was garbled
# but always good to be safe
# We wrap our socket send() calls in this lock
socket_lock = Lock()

stopped = False


def create_conn():
# open connection to irc server
s=socket.socket( )
"""
Open connection to irc server
:return: a socket
"""
s = socket.socket()
s.connect((HOST, PORT))
s.send("NICK {}\r\n".format(NICK).encode())
s.send("USER {} {} bla :{}\r\n".format(IDENT, HOST, REALNAME).encode())
Expand All @@ -67,112 +87,161 @@ def create_conn():


class message:
def __init__(self, text, to):
def __init__(self, line, to, lang):
# line[0] is user ident
self.userident = line[0].split("!")[0]
self.recipient = to
self.text = text
self.text = extract_text_from_line(line)
self.target_lang = lang
self.sent = False

def translate(self):
blob = TextBlob(self.text)
self.text = blob.translate(to=self.target_lang)

def mesg(msg):
"""
Send message, return number of bytes sent
"""
return s.send("PRIVMSG {} :{}\r\n".format(msg.recipient, msg.text).encode())
def send_info(self):
out_text = self.text
if self.recipient == USER:
# tell user who the message came from
out_text = self.userident + ": " + str(out_text)
return self.recipient, out_text

def __str__(self):
return self.text

def trans(line, sendto, lang):

def extract_text_from_line(line):
"""
Translate message
extract translateable text from the irc message
:param line:
:return:
"""
totrans = ""
translation = ""
for x in line[3:]:
totrans = totrans + (x + " ")
totrans = totrans[1:]
blob = TextBlob(totrans)
try: # for error if same language
translation = blob.translate(to=lang)
except Exception as e:
print("THREAD An exception of type {0} occurred. Arguments:\n{1!r}".format(type(e).__name__, e.args))
return
if(sendto == USER):
#mesg(line[0], to)
translation = line[0] + ": " + str(translation)
msgqueue.append(message(str(translation), sendto))
print("THREAD Translation: {}".format(translation))
return totrans


def translate_thread():
while not stopped:
# Queue will block until item is available
cur_msg = inmsgqueue.get()
if cur_msg == END:
print("Closing out translate thread")
continue

print("Translating")
try:
cur_msg.translate()
except Exception as e:
print("Problem translating, sending untranslated: {}".format(cur_msg))
traceback.print_exc()
outmsgqueue.put(cur_msg)

print("Translate thread complete")


def send_thread():
while not stopped:
cur_msg = outmsgqueue.get()
if cur_msg == END:
print("Closing out send thread")
continue

print("Sending")
try:
with socket_lock:
s.send("PRIVMSG {} :{}\r\n".format(*cur_msg.send_info()).encode())
# todo: make catch more specific
except Exception as e:
print("Problem sending message, retrying: {}".format(cur_msg))
traceback.print_exc()
outmsgqueue.put(cur_msg)
continue

print("Send thread complete")


def listen():
readbuffer = ""
while True: # loop FOREVER (exit with ctrl c)
# all of the code between set 1 and set 2 is just putting the message received from the server into a nice format for us
# set 1
readbuffer=readbuffer+s.recv(1024).decode() # store info sent from the server into
print("LTHREAD received data")
temp=readbuffer.split("\n") # remove \n from the readbuffer and store in a temp variable
readbuffer=temp.pop( ) # restore readbuffer to empty
#totranslate = ""

for line in temp: # parse through every line read from server
# turn line into a list
line=line.rstrip()
line=line.split()

# set 2
if(line[0]=="PING"): #if irc server sends a ping, pong back at it
# this assignment (indivserver) really should only be done once
while not stopped:
# store buffer from the server
readbuffer = s.recv(1024).decode()

print("Listen thread received data")
if len(readbuffer) == 0:
continue

# split the read buffer into separate lines
temp = readbuffer.split("\n")
# the last item is always empty or ":", discard
temp.pop()

# parse through every line read from server
for line in temp:
print(line)
# turn line into a list
line = line.rstrip()
line = line.split()

if len(line) < 2:
continue

if line[0] == "PING": # if irc server sends a ping, pong back at it
# indivserver is the node address in the irc network
indivserver = line[1]
PONG = True
print("LTHREAD PONG")
elif(line[2]==CHAN): #if a message comes in from the channel
print("LTHREAD message sent from " + CHAN)
thread = Thread(target = trans, args = (line, USER, userlang))
thread.handled = False
thread.start()
threads.append(thread)
#line[0] is user ident

elif(line[0][1:len(USER)+1] == USER and line[2]==NICK): #if user privmsg us
#transmesg(line, CHAN, chanlang)
thread = Thread(target = trans, args = (line, CHAN, chanlang))
thread.handled = False
thread.start()
threads.append(thread)
with socket_lock:
s.send("PONG {}\r\n".format(indivserver).encode())
print("PONG")
continue

if len(line) < 3:
continue

# if a message comes in from the channel, private message to our user
if line[2] == CHAN:
print("Message sent from " + CHAN)
inmsgqueue.put(message(line, USER, userlang))

# if user privmsg us, send to channel
elif line[0][1:len(USER) + 1] == USER and line[2] == NICK:
inmsgqueue.put(message(line, CHAN, chanlang))

print("Listen thread complete")


if __name__ == '__main__':
s = create_conn()
print("Connection created")

listenthread = Thread(target = listen)
listenthread.start()
listenthread = Thread(target=listen)
translatethread = Thread(target=translate_thread)
sendthread = Thread(target=send_thread)

while(True):
if PONG:
s.send("PONG {}\r\n".format(indivserver).encode())
print("MAIN PONG")
PONG = False

# clean up thread pool
for t in threads:
if not t.isAlive():
t.handled = True
else:
t.handled = False

threads = [t for t in threads if not t.handled]

# lol, it's all fancy and multithreaded now but this approach is probably more inherently error prone
# whatever, it's just an experimental project anyway
for m in msgqueue:
print("in queue for")
mesg(m)
m.sent = True

# if socket sent 0 bytes we retry
msgqueue = [m for m in msgqueue if not m.sent]

if len(threads) > 0 and DEBUG:
print("MAIN Threads: {}".format(len(threads)))
#print(totranslate)

if len(msgqueue) != 0:
print("MAIN WARNING - Message queue not empty - {} items".format(len(msgqueue)))
listenthread.start()
translatethread.start()
sendthread.start()

print("Threads started")

# wait for quit signal
while not stopped:
c = sys.stdin.read(1)
if c == 'q':
print("Quit signal received")
stopped = True
with inmsgqueue.mutex:
inmsgqueue.queue.clear()
inmsgqueue.put(END)
with outmsgqueue.mutex:
outmsgqueue.queue.clear()
outmsgqueue.put(END)

s.shutdown(socket.SHUT_RDWR)

print("Joining threads")
sendthread.join()
translatethread.join()
listenthread.join()

print("Complete")

0 comments on commit 38d55f3

Please sign in to comment.