Feature/video classification (#36)

* feat: Add support for video input types * feat: Added Output for videoClassification Modality * chore/ refactored SCSS to use className instead of element tags. * Refactor SampleInputsTab to use onSampleInputClickPreview for handling sample video input clicks
c3sr · Jul 29, 2024 · a947882 · a947882
1 parent 5f7f588
commit a947882
Show file tree

Hide file tree

Showing 27 changed files with 544 additions and 84 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/src/components/Experiment/QuickInput/QuickInput.js b/src/components/Experiment/QuickInput/QuickInput.js
@@ -3,11 +3,14 @@ import "./QuickImageInput.scss";
 import QuickImageInput from "./QuickImageInput";
 import QuickTextInput from "./QuickTextInput";
 import QuickAudioInput from "./QuickAudioInput";
+import QuickVideoInput from "./QuickVideoInput";
 import Task from "../../../helpers/Task";
 import { TaskInputTypes } from "../../../helpers/TaskInputTypes";
 // import { TaskControls } from "../../HomePage/TaskControls";
 import QuickMultiInput from "./QuickMultiInput";
 
+// TODO: QuickText/Image/Audio/VideoInput should be refactored into a single component
+
 export default function QuickInput(props) {
   const task = Task.getStaticTask(props.model.output.type);
   const [URLValidity, setURLValidity] = useState(false);
@@ -29,6 +32,8 @@ export default function QuickInput(props) {
       return <QuickTextInput hideUpload={task.hideUpload} {...props} />;
     case TaskInputTypes.Audio:
       return <QuickAudioInput  {...props} />;
+    case TaskInputTypes.Video:
+      return <QuickVideoInput {...props} />;
     case TaskInputTypes.Image:
     default:
       return <QuickImageInput {...props} />;

diff --git a/src/components/Experiment/QuickInput/QuickInput.stories.js b/src/components/Experiment/QuickInput/QuickInput.stories.js
@@ -22,6 +22,7 @@ import {
   textClassification,
   audioToAudio,
   audioClassification,
+  videoClassification,
 } from "../../../helpers/TaskIDs";
 import {
   SampleImageClassificationInputs,
@@ -40,6 +41,7 @@ import {
   SampleTextClassificationInputs,
   SampleAudioToAudioInputs,
   SampleAudioClassificationInputs,
+  SampleVideoClassificationInputs,
 } from "../../../helpers/sampleImages";
 import { QuickInputType } from "./quickInputType";
 import { TaskInputTypes } from "../../../helpers/TaskInputTypes";
@@ -293,4 +295,14 @@ AudioClassification.args = {
       type: audioClassification,
     },
   },
+};
+
+export const VideoClassification = Template.bind({});
+VideoClassification.args = {
+  sampleInputs: SampleVideoClassificationInputs,
+  model: {
+    output: {
+      type: videoClassification,
+    },
+  },
 };
diff --git a/src/components/Experiment/QuickInput/QuickVideoInput.js b/src/components/Experiment/QuickInput/QuickVideoInput.js
@@ -0,0 +1,71 @@
+import React from "react";
+import "./QuickVideoInput.scss";
+import Task from "../../../helpers/Task";
+import useQuickInputControl from "./useQuickInputControl";
+import useBEMNaming from "../../../common/useBEMNaming";
+import { QuickInputTabContent } from "./QuickInputTabContent";
+import { QuickInputTabTitle } from "./QuickInputTabTitle";
+import { QuickInputType } from "./quickInputType";
+
+export default function QuickVideoInput(props) {
+    const {
+        tabIsSelected,
+        selectedInputs,
+        addInput,
+        getTabs,
+        removeInput,
+        selectTab,
+        selectInput,
+        runModel,
+    } = useQuickInputControl(props);
+    const { getBlock, getElement } = useBEMNaming("quick-video-input");
+
+    const task = Task.getStaticTask(props.model.output.type);
+    const tabs = getTabs(QuickInputType.Video);
+
+    return (
+        <div className={getBlock()}>
+            {!props.hideHeader && (
+                <>
+                    <h2 className={getElement("title")}>Try this model</h2>
+                    <div className={getElement("subtitle")}>{task.inputText}</div>
+                </>
+            )}
+            <div className={getElement("tabs")}>
+                <div className={getElement("tab-titles")} role="tablist">
+                    {tabs.map((tab, index) => (
+                        <QuickInputTabTitle
+                            key={index}
+                            tab={tab}
+                            index={index}
+                            tabIsSelected={tabIsSelected}
+                            selectTab={selectTab}
+                            getElement={getElement}
+                        />
+                    ))}
+                </div>
+                {tabs.map((tab, index) => (
+                    <QuickInputTabContent
+                        key={index}
+                        tab={tab}
+                        index={index}
+                        getElement={getElement}
+                        {...props}
+                        removeInput={removeInput}
+                        addInput={addInput}
+                        selectInput={selectInput}
+                        tabIsSelected={tabIsSelected}
+                        selectedInputs={selectedInputs}
+                    />
+                ))}
+            </div>
+            <button
+                className={getElement("run-model")}
+                disabled={selectedInputs.length === 0 || selectedInputs[0] === ""}
+                onClick={() => runModel()}
+            >
+                Run model and see results
+            </button>
+        </div>
+    );
+}
diff --git a/src/components/Experiment/QuickInput/QuickVideoInput.scss b/src/components/Experiment/QuickInput/QuickVideoInput.scss
@@ -0,0 +1,22 @@
+@import "../../../App";
+
+.quick-video-input {
+  @include tabs;
+  @include quick-input-container;
+
+  &__subtitle {
+    @include body;
+    margin-top: 8px;
+  }
+
+  &__run-model {
+    @include primaryButtonWithIcon("after", "arrow-right-white.svg");
+
+    align-items: center;
+    align-self: flex-end;
+    display: flex;
+    flex-direction: row;
+    justify-content: center;
+    margin-top: 24px;
+  }
+}
diff --git a/src/components/Experiment/QuickInput/Tabs/SampleInput/SampleInputsTab.js b/src/components/Experiment/QuickInput/Tabs/SampleInput/SampleInputsTab.js
@@ -6,6 +6,8 @@ import useBEMNaming from "../../../../../common/useBEMNaming";
 import { QuickInputType } from "../../quickInputType";
 import { ReactComponent as DocumentIcon } from "../../../../../resources/icons/icon-document.svg";
 import { imageTo3D } from '../../../../../helpers/TaskIDs';
+import URLInputPreview from '../URLInput/URLInputPreview';
+import { TaskInputTypes } from '../../../../../helpers/TaskInputTypes';
 
 
 export default function SampleInputsTab(props) {
@@ -24,6 +26,12 @@ export default function SampleInputsTab(props) {
         return className;
     };
 
+    const inputHandlerForPreview = (src) => {
+        props?.inputPreviewProps?.setURLValidity(true);
+        props?.inputPreviewProps?.setSelectedInputSrc(src);
+
+    };
+
     const makeSampleInput = (url, index) => {
         switch (sampleInputType) {
             case QuickInputType.Image:
@@ -34,16 +42,23 @@ export default function SampleInputsTab(props) {
                 return makeSampleAudioInput(url, index);
             case QuickInputType.Document:
                 return makeSampleDocumentInput(url, index);
+            case QuickInputType.Video:
+                return makeSampleVideoInput(url, index);
             default:
                 return makeDefaultErrorInput();
         }
     };
 
+    const onSampleInputClickPreview = (index, url) => {
+        selectInput(index);
+        inputHandlerForPreview(url.src);
+    };
+
     // TODO: Rename "url" to "input" or similar
     function makeSampleImageInput(url, index) {
         return (
             <button onClick={() => selectInput(index)} key={index} className={getElement(getInputClassName(url))}>
-                <img src={url.src} alt={url.alt} />
+                <img src={url.src} alt={url.description} />
             </button>
         );
     }
@@ -76,6 +91,15 @@ export default function SampleInputsTab(props) {
         );
     }
 
+    function makeSampleVideoInput(url, index) {
+        return (
+            <button onClick={() => onSampleInputClickPreview(index, url)} key={index} className={getElement(getInputClassName(url))}>
+                <video src={url.src} alt={url.alt} autoPlay muted={true} loop className={getElement("sample-video-content")} />
+
+            </button>
+        );
+    }
+
     function makeDefaultErrorInput() {
         return (
             <div>No input type defined</div>
@@ -92,6 +116,9 @@ export default function SampleInputsTab(props) {
             <div className={getElement('list')}>
                 {sampleInputs.map(makeSampleInput)}
             </div>
+            {sampleInputType === QuickInputType.Video &&
+                <URLInputPreview inputPreviewProps={props.inputPreviewProps} inputType={TaskInputTypes.Video} selectedInputs={props.values} />
+            }
         </div>
     );
 
@@ -105,6 +132,8 @@ export default function SampleInputsTab(props) {
                 return "Select an audio file";
             case QuickInputType.Document:
                 return "Select a document";
+            case QuickInputType.Video:
+                return "Select a video";
             default:
                 return "Error: no input type set";
         }

diff --git a/src/components/Experiment/QuickInput/Tabs/SampleInput/SampleInputsTab.scss b/src/components/Experiment/QuickInput/Tabs/SampleInput/SampleInputsTab.scss
@@ -128,6 +128,31 @@
       box-shadow: -2px 4px 4px rgba(0, 0, 0, 0.25);
     }
 
+    &--unselected {
+      opacity: 0.5;
+    }
+  }
+  &__input-video {
+    line-height: 0;
+    width: 100%;
+    max-width: 240px;
+
+    & .sample-inputs__sample-video-content {
+      border: 6px solid transparent;
+      width: 100%;
+    }
+
+    &:not(:first-child) {
+      margin-left: 4px;
+    }
+
+    &--selected {
+      & .sample-inputs__sample-video-content {
+        border-color: $azul;
+        box-shadow: -2px 4px 4px rgba(0, 0, 0, 0.25);
+      }
+    }
+
     &--unselected {
       opacity: 0.5;
     }

diff --git a/src/components/Experiment/QuickInput/Tabs/URLInput/URLInputPreview.js b/src/components/Experiment/QuickInput/Tabs/URLInput/URLInputPreview.js
@@ -11,10 +11,13 @@ const URLInputPreview = (props) => {
                 <h3 className={getElement("title")}> Input Preview</h3>
                 <div className={getElement("preview")}>
                     {props.inputType === TaskInputTypes.Image ? (
-                        <img src={props?.inputPreviewProps?.selectedInputSrc} alt="Preview" />
+                        <img src={props?.inputPreviewProps?.selectedInputSrc} alt="Preview" className={getElement("img")} />
                     ) : props.inputType === TaskInputTypes.Audio ? (
-                        <audio controls src={props?.inputPreviewProps?.selectedInputSrc} title="Preview" />
-                    ) : null}
+                        <audio controls src={props?.inputPreviewProps?.selectedInputSrc} title="Preview" className={getElement("audio")} />
+                    ) : props.inputType === TaskInputTypes.Video ? (
+                        <video src={props?.inputPreviewProps?.selectedInputSrc} controls className={getElement("video")} />
+                    ) :
+                        null}
 
                 </div>
             </div>}

diff --git a/src/components/Experiment/QuickInput/Tabs/URLInput/URLInputPreview.scss b/src/components/Experiment/QuickInput/Tabs/URLInput/URLInputPreview.scss
@@ -1,14 +1,18 @@
 .url-inputs-preview {
   width: 100%;
   margin: 20px auto;
-}
 
-img {
-  margin: 10px auto;
-  display: block;
-}
+  &__img {
+    margin: 10px auto;
+    display: block;
+  }
+
+  &__audio {
+    margin: 10px auto;
+    display: block;
+  }
 
-audio {
-  margin: 10px auto;
-  display: block;
+  &__video {
+    max-width: min(100%, 1000px);
+  }
 }
diff --git a/src/components/Experiment/QuickInput/Tabs/URLInput/useURLInputControl.js b/src/components/Experiment/QuickInput/Tabs/URLInput/useURLInputControl.js
@@ -1,4 +1,5 @@
 import ImageVerifier from "../../../../../helpers/imageVerifier";
+import VideoVerifier from "../../../../../helpers/videoVerifier";
 import AudioVerifier from "../../../../../helpers/audioVerifier";
 import Task from "../../../../../helpers/Task";
 import { useState } from "react";
@@ -25,6 +26,9 @@ export default function useURLInputControl(props) {
         case TaskInputTypes.Audio:
           verifier = new AudioVerifier(tempUrl);
           break;
+        case TaskInputTypes.Video:
+          verifier = new VideoVerifier(tempUrl);
+          break;
         default:
           break;
       }

diff --git a/src/components/Experiment/QuickInput/quickInputType.js b/src/components/Experiment/QuickInput/quickInputType.js
@@ -3,4 +3,5 @@ export const QuickInputType = {
     Text: "text",
     Audio: "audio",
     Document: "document",
-}
+    Video: "video",
+};