diff --git a/MarioMaker2OCR/Form1.cs b/MarioMaker2OCR/Form1.cs index ecfae10..bf21527 100644 --- a/MarioMaker2OCR/Form1.cs +++ b/MarioMaker2OCR/Form1.cs @@ -16,17 +16,13 @@ using System.Text.RegularExpressions; + namespace MarioMaker2OCR { public partial class Form1 : Form { private static readonly log4net.ILog log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); private const string LEVEL_JSON_FILE = "ocrLevel.json"; - private Size resolution720 = new Size(1280, 720); - private Size resolution480 = new Size(640, 480); - - private Mat levelDetailScreen; - private readonly Mat levelSelectScreen720 = new Image("referenceImage.jpg").Mat; // based on 1280x720 // Product version public string CurrentVersion @@ -38,61 +34,6 @@ public string CurrentVersion } } - private List templates = new List(); - - private EventTemplate[] engTemplates = new EventTemplate[] { - new EventTemplate("./templates/480/exit.png", "exit", 0.8, new Rectangle[] { - new Rectangle(new Point(400,330), new Size(230, 65)), //Pause Menu - new Rectangle(new Point(410,225), new Size(215, 160)), //Clear Screen - }), - new EventTemplate("./templates/480/quit.png", "exit", 0.9, new Rectangle[] { - new Rectangle(new Point(408,338), new Size(224, 70)) //Pause Menu - }), - new EventTemplate("./templates/480/quit_full.png", "exit", 0.8, new Rectangle[] { - new Rectangle(new Point(195,225), new Size(215, 160)), //Clear Screen - }), - new EventTemplate("./templates/480/startover.png", "restart", 0.8, new Rectangle[] { - new Rectangle(new Point(400,275), new Size(230, 65)), //Pause Menu - new Rectangle(new Point(195,225), new Size(215, 160)), //Clear Screen - }), - new EventTemplate("./templates/480/death_big.png", "death", 0.8), - new EventTemplate("./templates/480/death_small.png", "death", 0.8), - new EventTemplate("./templates/480/death_partial.png", "death", 0.9), - new EventTemplate("./templates/480/gameover.png", "gameover", 0.8, new Rectangle[] { - new Rectangle(new Point(187,195), new Size(270, 100)) - }), - new EventTemplate("./templates/480/skip.png", "skip", 0.85, new Rectangle[] { - new Rectangle(new Point(308,200), new Size(25, 37)) - }) - }; - - private EventTemplate[] langNeutralTemplates = new EventTemplate[] - { - new EventTemplate("./templates/480/lang_neutral/startover.png", "restart", 0.8, new Rectangle[] { - new Rectangle(new Point(397,269), new Size(243, 71)), // Pause Menu - //new Rectangle(new Point(195,225), new Size(230, 160)), // This is ROI is "Start Over" or "Quit" depending on gamemode, leave out for now - }), - // This works for Quit (endless) and Exit (other modes) - new EventTemplate("./templates/480/lang_neutral/quit.png", "exit", 0.96, new Rectangle[] { - new Rectangle(new Point(537,331), new Size(103, 71)) //Pause Menu - }), - // This is Next (endless) or Exit (other modes) - new EventTemplate("./templates/480/lang_neutral/exit_next.png", "exit", 0.9, new Rectangle[] { - new Rectangle(new Point(598,323), new Size(30, 60)), // Clear Screen - new Rectangle(new Point(598,223), new Size(30, 60)) // Clear Screen (w/ comments) - }) - }; - - private readonly EventTemplate[] clearTemplates = new EventTemplate[] - { - new EventTemplate("./templates/480/worldrecord.png", "worldrecord", 0.8, new Rectangle[] { - new Rectangle(new Point(445,85), new Size(115, 130)), - }), - new EventTemplate("./templates/480/firstclear.png", "firstclear", 0.8, new Rectangle[] { - new Rectangle(new Point(445,85), new Size(115, 130)), - }) - }; - public DsDevice SelectedDevice => (deviceComboBox.SelectedItem as dynamic)?.Value; public Size SelectedResolution => (resolutionsCombobox.SelectedItem as dynamic)?.Value; @@ -189,18 +130,6 @@ private void startButton_Click(object sender, EventArgs e) } try { - templates.Clear(); - templates.AddRange(engTemplates); - - // Add language neutral templates if selected. - if (langNeutralcheckBox.Checked) - { - templates.AddRange(langNeutralTemplates); - } - - // resize reference image based on current resolution - levelDetailScreen = ImageLibrary.ChangeSize(levelSelectScreen720, resolution720, SelectedResolution); - SMMServer.port = decimal.ToUInt16(numPort.Value); log.Info(string.Format("Start Web Server on http://localhost:{0}/", SMMServer.port)); SMMServer.Start(); @@ -212,11 +141,24 @@ private void startButton_Click(object sender, EventArgs e) { WarpWorld = null; } - processor = new VideoProcessor(deviceComboBox.SelectedIndex, SelectedResolution); - processor.BlackScreen += VideoProcessor_BlackScreen; - processor.ClearScreen += VideoProcessor_ClearScreen; - processor.NewFrame += VideoProcessor_NewFrame; + + processor.TemplateMatch += broadcastTemplateMatch; + + processor.TemplateMatch += previewMatch; + processor.NewFrame += previewNewFrame; + + + processor.LevelScreen += Processor_LevelScreen; + processor.ClearScreen += Processor_ClearScreen; + + processor.Exit += clearJsonOnEvent; + processor.Skip += clearJsonOnEvent; + processor.GameOver += clearJsonOnEvent; + + processor.ClearScreen += warpWorldCallback; + processor.Exit += warpWorldCallback; + processor.Start(); lockForm(); } @@ -226,9 +168,20 @@ private void startButton_Click(object sender, EventArgs e) } } - private void VideoProcessor_NewFrame(object sender, VideoProcessor.VideoEventArgs e) + private void clearJsonOnEvent(object sender, VideoProcessor.TemplateMatchEventArgs e) { - previewer.SetLiveFrame(e.currentFrame); + switch (e.template.eventType) + { + case "exit": + if (JsonSettings.ClearOnExit) clearJsonFile(); + break; + case "skip": + if (JsonSettings.ClearOnSkip) clearJsonFile(); + break; + case "gameover": + if (JsonSettings.ClearOnGameover) clearJsonFile(); + break; + } } private void stopButton_Click(object sender, EventArgs e) @@ -274,139 +227,6 @@ private void unlockForm() webServerAddressStatusLabel.Text = ""; } - /// - /// Event callback for the Clear Screen event generatead by the VideoProcessor - /// - private void VideoProcessor_ClearScreen(object sender, VideoProcessor.ClearScreenEventArgs e) - { - log.Debug("Detected Level Clear"); - - Image grayscaleFrame = e.currentFrame.Mat.ToImage().Resize(640, 480, Inter.Cubic); - //e.currentFrame.Save("clearmatch_" + DateTime.Now.ToString("yyyyMMddHHmmssffff") + ".png"); - - Dictionary events = new Dictionary - { - { "worldrecord", false }, - { "firstclear", false }, - }; - List boundaries = new List(); - foreach (EventTemplate tmpl in clearTemplates) - { - if (events[tmpl.eventType]) continue; - Point loc = tmpl.getLocation(grayscaleFrame); - if (!loc.IsEmpty) - { - events[tmpl.eventType] = true; - boundaries.Add(ImageLibrary.ChangeSize(new Rectangle(loc.X, loc.Y, tmpl.template.Width, tmpl.template.Height), grayscaleFrame.Size, e.currentFrame.Size)); - previewer.SetLastMatch(e.currentFrame, boundaries.ToArray()); - } - } - - foreach (var evt in events) - { - if (evt.Value) - { - log.Info(String.Format("Detected {0}.", evt.Key)); - SMMServer.BroadcastEvent(evt.Key); - } - } - - if (Properties.Settings.Default.WarpWorldEnabled) - { - WarpWorld?.win(); - } - - // Read time from screen - string clearTime = OCRLibrary.GetClearTimeFromFrame(e.currentFrame, e.commentsEnabled); - SMMServer.BroadcastDataEvent("clear", clearTime); - } - /// - /// Event Callback for the Black Screen event generated by the VideoProcessor - /// - private void VideoProcessor_BlackScreen(object sender, VideoProcessor.BlackScreenEventArgs e) - { - log.Debug(String.Format("Detected a black screen [{0}]", e.seconds)); - BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing black screen...")); - - double imageMatchPercent = ImageLibrary.CompareImages(e.currentFrame, levelDetailScreen); - - // Is this frame a 90% match to a level screen? - if(imageMatchPercent > 0.90) - { - log.Info(String.Format("Detected new level. [{0}]", e.seconds)); - - BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing level screen...")); - - Level level = OCRLibrary.GetLevelFromFrame(e.currentFrame); - writeLevelToFile(level); - SMMServer.BroadcastLevel(level); - - BeginInvoke((MethodInvoker)(() => ocrTextBox.Text = level.code + " | " + level.author + " | " + level.name)); - BeginInvoke((MethodInvoker)(() => processingLabel.Text = "")); - - previewer.SetLastMatch(e.currentFrame); - } - else - { - // Not a new level, see if we can detect a template. - Dictionary events = new Dictionary - { - { "death", false }, - { "restart", false }, - { "exit", false }, - { "gameover", false }, - { "skip", false } - }; - - BeginInvoke((MethodInvoker)(() => processingLabel.Text = "Processing events...")); - foreach (Image f in e.frameBuffer) - { - // Skip any empty frames in the buffer - if (f == null) - continue; - - Image grayscaleFrame = f.Mat.ToImage().Resize(640, 480, Inter.Cubic); - //grayscaleFrame.Save("frame_" + DateTime.Now.ToString("yyyyMMddHHmmssffff") + ".png"); // XXX: Useful for debugging template false-negatives, and for getting templates - - List boundaries = new List(); - foreach (EventTemplate tmpl in templates) - { - if (events[tmpl.eventType]) continue; - Point loc = tmpl.getLocation(grayscaleFrame); - if (!loc.IsEmpty) - { - events[tmpl.eventType] = true; - boundaries.Add(ImageLibrary.ChangeSize(new Rectangle(loc.X, loc.Y, tmpl.template.Width, tmpl.template.Height), grayscaleFrame.Size, f.Size)); - previewer.SetLastMatch(f, boundaries.ToArray()); - } - } - } - BeginInvoke((MethodInvoker)(() => processingLabel.Text = "")); - - foreach (var evt in events) - { - if (evt.Value) - { - log.Info(String.Format("Detected {0} [{1}].", evt.Key, e.seconds)); - SMMServer.BroadcastEvent(evt.Key); - - // Even though this will get sent on exiting after a clear, it only matters if a entry is active, and after marking it as a win it goes inactive. - if(evt.Key == "exit") - { - if (Properties.Settings.Default.WarpWorldEnabled) - WarpWorld?.lose(); - if (JsonSettings.ClearOnExit) - clearJsonFile(); - } - if (evt.Key == "skip" && JsonSettings.ClearOnSkip) - clearJsonFile(); - if (evt.Key == "gameover" && JsonSettings.ClearOnGameover) - clearJsonFile(); - } - } - } - } - private void propertiesButton_Click(object sender, EventArgs e) { if (deviceComboBox.SelectedItem == null) @@ -587,5 +407,48 @@ private void settingsToolStripMenuItem1_Click(object sender, EventArgs e) jsonSettings.ShowDialog(); jsonSettings.BringToFront(); } + + private void broadcastTemplateMatch(object sender, VideoProcessor.TemplateMatchEventArgs e) + { + SMMServer.BroadcastEvent(e.template.eventType); + } + + private void previewMatch(object sender, VideoProcessor.TemplateMatchEventArgs e) + { + var boundary = ImageLibrary.ChangeSize(new Rectangle(e.location, e.template.template.Size), processor.TEMPLATE_FRAME_SIZE, processor.frameSize); + previewer.SetLastMatch(e.frame, new Rectangle[] { boundary }); + + } + private void previewNewFrame(object sender, VideoProcessor.NewFrameEventArgs e) + { + previewer.SetLiveFrame(e.frame); + } + + private void Processor_ClearScreen(object sender, VideoProcessor.ClearScreenEventArgs e) + { + if (Properties.Settings.Default.WarpWorldEnabled) WarpWorld?.win(); + SMMServer.BroadcastDataEvent("clear", e.clearTime); + // TODO: Send WR/First Clear here also since we have that in the event + } + + private void Processor_LevelScreen(object sender, VideoProcessor.LevelScreenEventArgs e) + { + previewer.SetLastMatch(e.frame); + writeLevelToFile(e.levelInfo); + SMMServer.BroadcastLevel(e.levelInfo); + BeginInvoke((MethodInvoker)(() => ocrTextBox.Text = e.levelInfo.code + " | " + e.levelInfo.author + " | " + e.levelInfo.name)); + } + + private void warpWorldCallback(object sender, VideoProcessor.TemplateMatchEventArgs e) + { + if (!Properties.Settings.Default.WarpWorldEnabled) return; + if (e.template.eventType == "exit") WarpWorld?.lose(); + } + + private void warpWorldCallback(object sender, VideoProcessor.ClearScreenEventArgs e) + { + if (!Properties.Settings.Default.WarpWorldEnabled) return; + WarpWorld?.win(); + } } } diff --git a/MarioMaker2OCR/OCRLibrary.cs b/MarioMaker2OCR/OCRLibrary.cs index 66f2d6e..c62896a 100644 --- a/MarioMaker2OCR/OCRLibrary.cs +++ b/MarioMaker2OCR/OCRLibrary.cs @@ -94,6 +94,8 @@ internal static string GetClearTimeFromFrame(Image frame, bool commen // Segment characters List characters = segmentCharacters(ocrReadyImage); + frame.ROI = Rectangle.Empty; + // expect time to be 9 characters, quote reads as 2 chars (ex: 01'34''789) if (characters.Count == 10) { diff --git a/MarioMaker2OCR/Objects/EventTemplate.cs b/MarioMaker2OCR/Objects/EventTemplate.cs index 8c60e6a..103703c 100644 --- a/MarioMaker2OCR/Objects/EventTemplate.cs +++ b/MarioMaker2OCR/Objects/EventTemplate.cs @@ -16,6 +16,7 @@ public class EventTemplate : IDisposable public string eventType { get; } public string filename { get; } public Rectangle[] regions { get; } + public double scale { get; } bool disposed = false; public void Dispose() @@ -41,6 +42,7 @@ public EventTemplate(string fn, string type, double thresh) threshold = thresh; eventType = type; filename = fn; + scale = 1; } public EventTemplate(string fn, string type, double thresh, Rectangle[] ROIs) @@ -50,33 +52,58 @@ public EventTemplate(string fn, string type, double thresh, Rectangle[] ROIs) eventType = type; filename = fn; regions = ROIs; + scale = 1; + } + + public EventTemplate(string fn, string type, double thresh, Rectangle[] ROIs, double scale) + { + template = new Image(fn); + threshold = thresh; + eventType = type; + filename = fn; + regions = ROIs; + this.scale = scale; } public Point getLocation(Image frame) { - if(regions == null || regions.Length == 0) - { - return getLocation(frame, Rectangle.Empty); - } else + for (double i = 1; i <= scale; i = i + .05d) { - foreach(Rectangle roi in regions) + if (regions == null || regions.Length == 0) { - Point ret = getLocation(frame, roi); + Point ret = getLocation(frame, Rectangle.Empty, i); if (!ret.IsEmpty) { - ret.X += roi.X; - ret.Y += roi.Y; return ret; } } + else + { + foreach (Rectangle roi in regions) + { + Point ret = getLocation(frame, roi, i); + if (!ret.IsEmpty) + { + ret.X += roi.X; + ret.Y += roi.Y; + return ret; + } + } + } } + return Point.Empty; } - public Point getLocation(Image frame, Rectangle roi) + public Point getLocation(Image frame, Rectangle roi, double scale) { + Image templateResized; + if (scale > 1) + templateResized = template.Resize(scale, Emgu.CV.CvEnum.Inter.Cubic); + else + templateResized = template; frame.ROI = roi; - Image match = frame.MatchTemplate(template, Emgu.CV.CvEnum.TemplateMatchingType.CcoeffNormed); + Image match = frame.MatchTemplate(templateResized, Emgu.CV.CvEnum.TemplateMatchingType.CcoeffNormed); match.MinMax(out _, out double[] max, out _, out Point[] maxLoc); if (max[0] < threshold) return Point.Empty; return maxLoc[0]; diff --git a/MarioMaker2OCR/VideoProcessor.cs b/MarioMaker2OCR/VideoProcessor.cs index 1e7bca7..79148c3 100644 --- a/MarioMaker2OCR/VideoProcessor.cs +++ b/MarioMaker2OCR/VideoProcessor.cs @@ -10,6 +10,7 @@ using System.Runtime.InteropServices; using System.Drawing; using System.Runtime.CompilerServices; +using MarioMaker2OCR.Objects; namespace MarioMaker2OCR @@ -19,10 +20,11 @@ class VideoProcessor : IDisposable private static readonly log4net.ILog log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); public bool disposed = false; // Flag to indicate the object has been disposed public const int NO_DEVICE = -1; // Constant indicating that no video device was used + public readonly Size TEMPLATE_FRAME_SIZE = new Size(640, 480); + public Size frameSize; // Contains the Size() object for the frame, needed for frameBuffer_tick to create the iamge private VideoCapture cap; // EmguCV VideoCapture device object private int deviceId; // Device id of the video capture device - private Size frameSize; // Contains the Size() object for the frame, needed for frameBuffer_tick to create the iamge private Thread processorThread; // The thread performing the main video processing private System.Threading.Timer frameBufferTimer; // Timer used to fill the frame buffer private Image[] frameBuffer = new Image[16]; // Frame buffer that is passed to events @@ -30,6 +32,12 @@ class VideoProcessor : IDisposable private const int FRAME_BUFFER_INTERVAL = 250; // put frame in buffer every 250ms + private Mat levelDetailScreen; + private string lastEvent; + + private Dictionary> templates = new Dictionary>(); + + /// /// Essentially copied from https://docs.microsoft.com/en-us/dotnet/standard/garbage-collection/implementing-dispose /// @@ -59,6 +67,8 @@ public VideoProcessor(VideoCapture video) this.deviceId = NO_DEVICE; this.cap = video; this.frameSize = getCaptureInfo(video).resolution; + + initializeTemplates(); } public VideoProcessor(int device, Size resolution) @@ -66,6 +76,115 @@ public VideoProcessor(int device, Size resolution) this.deviceId = device; this.cap = createCaptureDevice(device, resolution); this.frameSize = getCaptureInfo(this.cap).resolution; + + initializeTemplates(); + } + + private void initializeTemplates() + { + var originalLevelScreen = new Image("referenceImage.jpg").Mat; + levelDetailScreen = ImageLibrary.ChangeSize(originalLevelScreen, new Size(originalLevelScreen.Width, originalLevelScreen.Height), frameSize); + + List clearDetailTemplates = new List(); + List blackScreenTemplates = new List(); + List postClearTemplates = new List(); + + + // Start Clear Detail screen templates + clearDetailTemplates.Add( + new EventTemplate("./templates/480/worldrecord.png", "worldrecord", 0.6, new Rectangle[] { + new Rectangle(new Point(445,85), new Size(115, 130)), + }, 1.31) + ); + clearDetailTemplates.Add( + new EventTemplate("./templates/480/firstclear.png", "firstclear", 0.6, new Rectangle[] { + new Rectangle(new Point(445,85), new Size(115, 130)), + }, 1.31) + ); + + //Start Templates that run on black screen immediately following a clear + postClearTemplates.Add( + new EventTemplate("./templates/480/exit.png", "exit", 0.8, new Rectangle[] { + new Rectangle(new Point(410,225), new Size(215, 160)), //Clear Screen + }) + ); + postClearTemplates.Add( + new EventTemplate("./templates/480/quit_full.png", "exit", 0.8, new Rectangle[] { + new Rectangle(new Point(195,225), new Size(215, 160)), //Clear Screen + }) + ); + postClearTemplates.Add( + new EventTemplate("./templates/480/startover.png", "restart", 0.8, new Rectangle[] { + new Rectangle(new Point(195,225), new Size(215, 160)), //Clear Screen + }) + ); + + // Start Black Screen Templates + blackScreenTemplates.Add( + new EventTemplate("./templates/480/exit.png", "exit", 0.8, new Rectangle[] { + new Rectangle(new Point(400,330), new Size(230, 65)), //Pause Menu + }) + ); + blackScreenTemplates.Add( + new EventTemplate("./templates/480/quit.png", "exit", 0.9, new Rectangle[] { + new Rectangle(new Point(408,338), new Size(224, 70)) //Pause Menu + }) + ); + blackScreenTemplates.Add( + new EventTemplate("./templates/480/startover.png", "restart", 0.8, new Rectangle[] { + new Rectangle(new Point(400,275), new Size(230, 65)), //Pause Menu + }) + ); + blackScreenTemplates.Add( + new EventTemplate("./templates/480/death_big.png", "death", 0.8) + ); + blackScreenTemplates.Add( + new EventTemplate("./templates/480/death_small.png", "death", 0.8) + ); + blackScreenTemplates.Add( + new EventTemplate("./templates/480/death_partial.png", "death", 0.9) + ); + blackScreenTemplates.Add( + new EventTemplate("./templates/480/gameover.png", "gameover", 0.8, new Rectangle[] { + new Rectangle(new Point(187,195), new Size(270, 100)) + }) + ); + blackScreenTemplates.Add( + new EventTemplate("./templates/480/skip.png", "skip", 0.85, new Rectangle[] { + new Rectangle(new Point(308,200), new Size(25, 37)) + }) + ); + + if (Properties.Settings.Default.DetectMultipleLanguages) + { + postClearTemplates.Add( + new EventTemplate("./templates/480/lang_neutral/exit_next.png", "exit", 0.9, new Rectangle[] { + new Rectangle(new Point(598,323), new Size(30, 60)), // Clear Screen + new Rectangle(new Point(598,223), new Size(30, 60)) // Clear Screen (w/ comments) + }) + ); + + blackScreenTemplates.Add( + new EventTemplate("./templates/480/lang_neutral/startover.png", "restart", 0.8, new Rectangle[] { + new Rectangle(new Point(397,269), new Size(243, 71)), // Pause Menu + //new Rectangle(new Point(195,225), new Size(230, 160)), // This is ROI is "Start Over" or "Quit" depending on gamemode, leave out for now + }) + ); + blackScreenTemplates.Add( + new EventTemplate("./templates/480/lang_neutral/quit.png", "exit", 0.96, new Rectangle[] { + new Rectangle(new Point(537,331), new Size(103, 71)) //Pause Menu + }) + ); + + } + + templates.Add("clear", clearDetailTemplates); + templates.Add("postclear", postClearTemplates); + templates.Add("black", blackScreenTemplates); + + + + } /// /// Starts the main video processing loop @@ -130,8 +249,10 @@ public void frameBuffer_tick(object sender=null) } frameBuffer[0] = frame; - VideoEventArgs args = new VideoEventArgs(); - args.currentFrame = frame.Clone(); + NewFrameEventArgs args = new NewFrameEventArgs + { + frame = frame.Clone() + }; onNewFrame(args); } catch(Exception ex) @@ -223,7 +344,9 @@ public void processingLoop() bool WaitForClearStats = false; while (true) { - if(deviceId == NO_DEVICE) + if (flags.ShouldStop) return; + + if (deviceId == NO_DEVICE) { cap.Read(currentFrame); if (currentFrame.IsEmpty) return; @@ -245,10 +368,8 @@ public void processingLoop() if(!flags.IsBlack) { BlackScreenEventArgs args = new BlackScreenEventArgs(); - args.frameBuffer = copyFrameBuffer(); - args.currentFrame = getLevelScreenImageFromBuffer(args.frameBuffer); args.seconds = DateTime.Now.Subtract(blackStart).TotalMilliseconds/1000; - onBlackScreen(args); + new Thread(new ParameterizedThreadStart(onBlackScreenEnd)).Start(args); } } else if (flags.IsClear) @@ -259,30 +380,28 @@ public void processingLoop() } else if (WaitForClearStats && isClearWithStatsScreen(hues)) { - log.Info("Have clear screen"); + log.Info("Detected level clear."); // HACK: Apart from taking up more CPU to do a comparision like the Level Select screen this is the best solution imo // Match happens during transition, so 500ms is long enough to get to the screen, but not long enough to exit and miss it. - Thread.Sleep(593); + Thread.Sleep(500); cap.Retrieve(currentFrame); ClearScreenEventArgs args = new ClearScreenEventArgs(); - args.currentFrame = currentFrame.Clone().ToImage(); + args.frame = currentFrame.Clone().ToImage(); // Check to see if this is the clear screen with comments on - things are positioned differently. // Top of screen is yellow if comments are on Size topOfScreen = new Size(frameSize.Width, frameSize.Height / 6); Dictionary topHues = getHues(data, topOfScreen, skip); - if (isMostlyYellow(topHues)) - args.commentsEnabled = true; - - - onClearScreen(args); + if (isMostlyYellow(topHues)) args.commentsEnabled = true; + new Thread(new ParameterizedThreadStart(onClearScreen)).Start(args); } else if (isBlackFrame(hues)) { flags.IsBlack = true; - WaitForClearStats = false; // XXX: If we get a black screen and this is true, something weird is going on + WaitForClearStats = false; // If we get a black screen and this is true, something weird is going on blackStart = DateTime.Now; + new Thread(new ThreadStart(onBlackScreenStart)).Start(); } else if (isClearFrame(hues)) { @@ -431,55 +550,203 @@ public static bool isMostlyYellow(Dictionary hues) /// Event that fires off whenever a black screen is detected /// public event EventHandler BlackScreen; - protected virtual void onBlackScreen(BlackScreenEventArgs e) + protected virtual void onBlackScreenEnd(object a) { if (frameBuffer[0] == null) return; - BlackScreen?.Invoke(this, e); - for (int i = 0; i < frameBuffer.Length; i++) + + BlackScreen?.Invoke(this, (BlackScreenEventArgs)a); + clearFrameBuffer(); + } + + + public class TemplateMatchEventArgs: EventArgs + { + public Image frame; + public Point location; + public EventTemplate template; + } + public class LevelScreenEventArgs : TemplateMatchEventArgs + { + public Level levelInfo; + } + public event EventHandler LevelScreen; + public event EventHandler TemplateMatch; + public event EventHandler Death; + public event EventHandler Exit; + public event EventHandler Restart; + public event EventHandler Skip; + public event EventHandler GameOver; + public event EventHandler WorldRecord; + public event EventHandler FirstClear; + + protected virtual void onClear() + { + + } + protected virtual void onBlackScreenStart() + { + var buffer = copyFrameBuffer(); + var levelFrame = getLevelScreenImageFromBuffer(buffer); + + // Do not process anything if the buffer is empty + if (levelFrame == null) return; + + double levelScreenMatch = ImageLibrary.CompareImages(levelFrame, levelDetailScreen); + if(levelScreenMatch > 0.90) { - frameBuffer[i]?.Dispose(); - frameBuffer[i] = null; + this.lastEvent = "level"; + Level level = OCRLibrary.GetLevelFromFrame(levelFrame); + log.Info(String.Format("Detected new level: {0}", level.code)); + LevelScreenEventArgs args = new LevelScreenEventArgs { + frame = levelFrame, + levelInfo = level, + }; + LevelScreen?.Invoke(this, args); + } else + { + List ts; + switch(this.lastEvent) + { + case "clear": + ts = templates["postclear"]; + break; + default: + ts = templates["black"]; + break; + } + + bool matchFound = false; + foreach (var frame in buffer) + { + if (frame == null) break; + Image grayscaleFrame = frame.Mat.ToImage().Resize(TEMPLATE_FRAME_SIZE.Width, TEMPLATE_FRAME_SIZE.Height, Inter.Cubic); + foreach (var tmpl in ts) + { + var loc = tmpl.getLocation(grayscaleFrame); + if (loc != Point.Empty) + { + this.lastEvent = tmpl.eventType; + log.Info(String.Format("Detected {0}", tmpl.eventType)); + TemplateMatchEventArgs args = new TemplateMatchEventArgs + { + frame = frame, + location = loc, + template = tmpl, + }; + TemplateMatch?.Invoke(this, args); + + switch(tmpl.eventType) + { + case "death": + Death?.Invoke(this, args); + break; + case "exit": + Exit?.Invoke(this, args); + break; + case "restart": + Restart?.Invoke(this, args); + break; + case "skip": + Skip?.Invoke(this, args); + break; + case "gameover": + GameOver?.Invoke(this, args); + break; + default: + log.Error(String.Format("No handler for event type: {0}", tmpl.eventType)); + break; + } + matchFound = true; + break; + } + } + if (matchFound) break; + } + } + + for (int i = 0; i < buffer.Length; i++) + { + buffer[i]?.Dispose(); + buffer[i] = null; } } + + /// /// Event fires off whenever a clear screen is detected /// public event EventHandler ClearScreen; - protected virtual void onClearScreen(ClearScreenEventArgs e) + protected virtual void onClearScreen(object a) { + ClearScreenEventArgs e = (ClearScreenEventArgs)a; + this.lastEvent = "clear"; if (frameBuffer[0] == null) return; - ClearScreen?.Invoke(this, e); - for (int i = 0; i < frameBuffer.Length; i++) + + e.clearTime = OCRLibrary.GetClearTimeFromFrame(e.frame, e.commentsEnabled); + + Image grayscaleFrame = e.frame.Mat.ToImage().Resize(640, 480, Inter.Cubic); + foreach (var tmpl in templates["clear"]) { - frameBuffer[i]?.Dispose(); - frameBuffer[i] = null; + var loc = tmpl.getLocation(grayscaleFrame); + if (loc != Point.Empty) + { + log.Info(String.Format("Detected {0}", tmpl.eventType)); + TemplateMatchEventArgs args = new TemplateMatchEventArgs + { + frame = e.frame, + location = loc, + template = tmpl, + }; + TemplateMatch?.Invoke(this, args); + + switch (tmpl.eventType) + { + case "firstclear": + e.firstClear = true; + FirstClear?.Invoke(this, e); + break; //or count a firstClear as a WR? + case "worldrecord": + e.worldRecord = true; + WorldRecord?.Invoke(this, e); + break; + default: + log.Error(String.Format("No handler for event type: {0}", tmpl.eventType)); + break; + } + break; + } } + ClearScreen?.Invoke(this, e); + clearFrameBuffer(); } /// /// Fires off every time the frameBuffer adds a new frame. /// - public event EventHandler NewFrame; - protected virtual void onNewFrame(VideoEventArgs e) + public event EventHandler NewFrame; + protected virtual void onNewFrame(NewFrameEventArgs e) { NewFrame?.Invoke(this, e); - e.currentFrame.Dispose(); + e.frame.Dispose(); } - public class VideoEventArgs : EventArgs + public class NewFrameEventArgs : EventArgs { - public Image[] frameBuffer; - public Image currentFrame; + public Image frame; } - public class BlackScreenEventArgs: VideoEventArgs + public class BlackScreenEventArgs: EventArgs { public double seconds; } - public class ClearScreenEventArgs : VideoEventArgs + public class ClearScreenEventArgs : EventArgs { + public Image frame; public bool commentsEnabled; + public string clearTime; + public bool firstClear; + public bool worldRecord; } /// @@ -511,5 +778,6 @@ private Image getLevelScreenImageFromBuffer(Image[] buffer return returnImage; } + } }