From 9e493b5ec82dbd65156cef90de578cc26f792df2 Mon Sep 17 00:00:00 2001 From: dram55 Date: Tue, 13 Aug 2019 18:12:12 -0400 Subject: [PATCH] Add scaling for clear templates + bugfix --- MarioMaker2OCR/Form1.cs | 4 --- MarioMaker2OCR/OCRLibrary.cs | 2 ++ MarioMaker2OCR/Objects/EventTemplate.cs | 47 +++++++++++++++++++------ MarioMaker2OCR/VideoProcessor.cs | 18 ++++++---- 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/MarioMaker2OCR/Form1.cs b/MarioMaker2OCR/Form1.cs index 1002092..bf21527 100644 --- a/MarioMaker2OCR/Form1.cs +++ b/MarioMaker2OCR/Form1.cs @@ -159,10 +159,6 @@ private void startButton_Click(object sender, EventArgs e) processor.ClearScreen += warpWorldCallback; processor.Exit += warpWorldCallback; - - - - processor.Start(); lockForm(); } diff --git a/MarioMaker2OCR/OCRLibrary.cs b/MarioMaker2OCR/OCRLibrary.cs index 66f2d6e..c62896a 100644 --- a/MarioMaker2OCR/OCRLibrary.cs +++ b/MarioMaker2OCR/OCRLibrary.cs @@ -94,6 +94,8 @@ internal static string GetClearTimeFromFrame(Image frame, bool commen // Segment characters List characters = segmentCharacters(ocrReadyImage); + frame.ROI = Rectangle.Empty; + // expect time to be 9 characters, quote reads as 2 chars (ex: 01'34''789) if (characters.Count == 10) { diff --git a/MarioMaker2OCR/Objects/EventTemplate.cs b/MarioMaker2OCR/Objects/EventTemplate.cs index 8c60e6a..103703c 100644 --- a/MarioMaker2OCR/Objects/EventTemplate.cs +++ b/MarioMaker2OCR/Objects/EventTemplate.cs @@ -16,6 +16,7 @@ public class EventTemplate : IDisposable public string eventType { get; } public string filename { get; } public Rectangle[] regions { get; } + public double scale { get; } bool disposed = false; public void Dispose() @@ -41,6 +42,7 @@ public EventTemplate(string fn, string type, double thresh) threshold = thresh; eventType = type; filename = fn; + scale = 1; } public EventTemplate(string fn, string type, double thresh, Rectangle[] ROIs) @@ -50,33 +52,58 @@ public EventTemplate(string fn, string type, double thresh, Rectangle[] ROIs) eventType = type; filename = fn; regions = ROIs; + scale = 1; + } + + public EventTemplate(string fn, string type, double thresh, Rectangle[] ROIs, double scale) + { + template = new Image(fn); + threshold = thresh; + eventType = type; + filename = fn; + regions = ROIs; + this.scale = scale; } public Point getLocation(Image frame) { - if(regions == null || regions.Length == 0) - { - return getLocation(frame, Rectangle.Empty); - } else + for (double i = 1; i <= scale; i = i + .05d) { - foreach(Rectangle roi in regions) + if (regions == null || regions.Length == 0) { - Point ret = getLocation(frame, roi); + Point ret = getLocation(frame, Rectangle.Empty, i); if (!ret.IsEmpty) { - ret.X += roi.X; - ret.Y += roi.Y; return ret; } } + else + { + foreach (Rectangle roi in regions) + { + Point ret = getLocation(frame, roi, i); + if (!ret.IsEmpty) + { + ret.X += roi.X; + ret.Y += roi.Y; + return ret; + } + } + } } + return Point.Empty; } - public Point getLocation(Image frame, Rectangle roi) + public Point getLocation(Image frame, Rectangle roi, double scale) { + Image templateResized; + if (scale > 1) + templateResized = template.Resize(scale, Emgu.CV.CvEnum.Inter.Cubic); + else + templateResized = template; frame.ROI = roi; - Image match = frame.MatchTemplate(template, Emgu.CV.CvEnum.TemplateMatchingType.CcoeffNormed); + Image match = frame.MatchTemplate(templateResized, Emgu.CV.CvEnum.TemplateMatchingType.CcoeffNormed); match.MinMax(out _, out double[] max, out _, out Point[] maxLoc); if (max[0] < threshold) return Point.Empty; return maxLoc[0]; diff --git a/MarioMaker2OCR/VideoProcessor.cs b/MarioMaker2OCR/VideoProcessor.cs index b9a12ac..79148c3 100644 --- a/MarioMaker2OCR/VideoProcessor.cs +++ b/MarioMaker2OCR/VideoProcessor.cs @@ -92,14 +92,14 @@ private void initializeTemplates() // Start Clear Detail screen templates clearDetailTemplates.Add( - new EventTemplate("./templates/480/worldrecord.png", "worldrecord", 0.8, new Rectangle[] { + new EventTemplate("./templates/480/worldrecord.png", "worldrecord", 0.6, new Rectangle[] { new Rectangle(new Point(445,85), new Size(115, 130)), - }) + }, 1.31) ); clearDetailTemplates.Add( - new EventTemplate("./templates/480/firstclear.png", "firstclear", 0.8, new Rectangle[] { + new EventTemplate("./templates/480/firstclear.png", "firstclear", 0.6, new Rectangle[] { new Rectangle(new Point(445,85), new Size(115, 130)), - }) + }, 1.31) ); //Start Templates that run on black screen immediately following a clear @@ -383,7 +383,7 @@ public void processingLoop() log.Info("Detected level clear."); // HACK: Apart from taking up more CPU to do a comparision like the Level Select screen this is the best solution imo // Match happens during transition, so 500ms is long enough to get to the screen, but not long enough to exit and miss it. - Thread.Sleep(593); + Thread.Sleep(500); cap.Retrieve(currentFrame); ClearScreenEventArgs args = new ClearScreenEventArgs(); @@ -395,7 +395,6 @@ public void processingLoop() Dictionary topHues = getHues(data, topOfScreen, skip); if (isMostlyYellow(topHues)) args.commentsEnabled = true; new Thread(new ParameterizedThreadStart(onClearScreen)).Start(args); - } else if (isBlackFrame(hues)) { @@ -588,6 +587,10 @@ protected virtual void onBlackScreenStart() { var buffer = copyFrameBuffer(); var levelFrame = getLevelScreenImageFromBuffer(buffer); + + // Do not process anything if the buffer is empty + if (levelFrame == null) return; + double levelScreenMatch = ImageLibrary.CompareImages(levelFrame, levelDetailScreen); if(levelScreenMatch > 0.90) { @@ -680,12 +683,15 @@ protected virtual void onClearScreen(object a) this.lastEvent = "clear"; if (frameBuffer[0] == null) return; + e.clearTime = OCRLibrary.GetClearTimeFromFrame(e.frame, e.commentsEnabled); + Image grayscaleFrame = e.frame.Mat.ToImage().Resize(640, 480, Inter.Cubic); foreach (var tmpl in templates["clear"]) { var loc = tmpl.getLocation(grayscaleFrame); if (loc != Point.Empty) { + log.Info(String.Format("Detected {0}", tmpl.eventType)); TemplateMatchEventArgs args = new TemplateMatchEventArgs { frame = e.frame,