Integrate LlmService

Better label for making judgements

Look at unauthorized use case

typo!

app/assets/javascripts/components/book_listing/book_listing.html

@@ -14,8 +14,8 @@
         <i
           class="bi bi-book-half"
           aria-hidden="true"
-          title="Make Judgements"
-          alt="Make Judgements"
+          title="Judge Documents"
+          alt="Judge Documents"
         ></i>
       </a>
 

app/assets/javascripts/components/import_to_cases/_modal_snapshot.html

@@ -9,7 +9,7 @@ <h3 class="modal-title">Import Snapshot</h3>
   </p>
   <p>
     The CSV file should have the headers: <code>Snapshot Name,Snapshot Time,Case ID,Query Text,Doc ID,Doc Position</code>.
-    The <code>Case ID</code> must already exist in Quepid.  You can have multiple snapshots in a single file, seperated by the <code>Snapshot Name</code> field.
+    The <code>Case ID</code> must already exist in Quepid.  You can have multiple snapshots in a single file, separated by the <code>Snapshot Name</code> field.
     The file should look similar to:
     <pre>
 Snapshot Name,Snapshot Time,Case ID,Query Text,Doc ID,Doc Position

app/assets/javascripts/components/judgements/_modal.html

@@ -117,7 +117,7 @@ <h3 class="modal-title">Judgements</h3>
 
   <a class="btn btn-default pull-left" href="books/{{ctrl.activeBookId}}/judge" target="_self" ng-show="ctrl.canUpdateCase && ctrl.activeBookId">
     <i class="glyphicon glyphicon-book"></i>
-    Make Judgements!
+    Judge Documents!
   </a>
   <a class="btn btn-default pull-left" ng-click="ctrl.refreshRatingsFromBook()" ng-disabled="processingPrompt.inProgress || ctrl.populateBook || !ctrl.activeBookId || ctrl.share.acase.bookId !== ctrl.activeBookId" >
     <i class="glyphicon glyphicon-refresh" ng-class="{'spintime': processingPrompt.inProgress}"></i>

app/assets/javascripts/services/settingsSvc.js

@@ -344,7 +344,7 @@ angular.module('QuepidApp')
             if (newUrl === null || angular.isUndefined(newUrl)) {
               useTMDBDemoSettings = true;
             }
-            // We actually have seperate demos for Solr based on http and https urls.
+            // We actually have separate demos for Solr based on http and https urls.
             else if (newUrl === this.tmdbSettings['solr'].insecureSearchUrl || newUrl === this.tmdbSettings['solr'].secureSearchUrl) {
               useTMDBDemoSettings = true;
             }

app/controllers/ai_judges/prompts_controller.rb

@@ -24,16 +24,11 @@ def update
       @ai_judge = User.find(params[:ai_judge_id])
       @ai_judge.update(ai_judge_params)
 
-      # Hey scott, here we make a call!
-      llm_service = LlmService.new(@ai_judge.openai_key)
-      result = llm_service.make_judgement @ai_judge.prompt, 'user prompt'
-      pp result
-
       @query_doc_pair = QueryDocPair.new(query_doc_pair_params)
 
-      @judgement = Judgement.new
-      @judgement.rating = result[:rating]
-      @judgement.explanation = result[:explanation]
+      llm_service = LlmService.new(@ai_judge.openai_key)
+      @judgement = llm_service.make_judgement @ai_judge, @query_doc_pair
+      pp @judgement
 
       render :edit
     end

app/controllers/ai_judges_controller.rb

@@ -3,7 +3,7 @@
 class AiJudgesController < ApplicationController
   before_action :set_team
 
-  DEFAULT_PROMPT = <<~TEXT
+  DEFAULT_SYSTEM_PROMPT = <<~TEXT
     You are evaluating the results from a search engine. For each query, you will be provided with multiple documents. Your task is to evaluate each document and assign a judgment on a scale of 0 to 2, where:
     - 0 indicates the document is irrelevant to the query.
     - 1 indicates the document is somewhat relevant to the query.
@@ -60,7 +60,7 @@ class AiJudgesController < ApplicationController
 
   def new
     @ai_judge = User.new
-    @ai_judge.prompt = DEFAULT_PROMPT
+    @ai_judge.prompt = DEFAULT_SYSTEM_PROMPT
   end
 
   def edit

app/controllers/books_controller.rb

@@ -209,7 +209,7 @@ def run_judge_judy
     ai_judge = @book.ai_judges.where(id: params[:ai_judge_id]).first
 
     judge_all = deserialize_bool_param(params[:judge_all])
-    number_of_pairs = params[:number_of_pairs]
+    number_of_pairs = params[:number_of_pairs].to_i
     number_of_pairs = nil if judge_all
 
     RunJudgeJudyJob.perform_later(@book, ai_judge, number_of_pairs)

app/jobs/run_judge_judy_job.rb

@@ -14,23 +14,42 @@ class RunJudgeJudyJob < ApplicationJob
   #
   # @example Judge all pairs
   #   RunJudgeJudyJob.perform_later(book, ai_judge, nil)
+  # rubocop:disable Metrics/MethodLength
   def perform book, judge, number_of_pairs
     counter = 0
+    puts "the key is #{judge.openai_key}"
+    llm_service = LlmService.new judge.openai_key, {}
     loop do
-      break if number_of_pairs && counter >= (number_of_pairs.to_i)
+      break if number_of_pairs && counter >= number_of_pairs
 
       query_doc_pair = SelectionStrategy.random_query_doc_based_on_strategy(book, judge)
       break if query_doc_pair.nil?
 
-      judgement = Judgement.new(query_doc_pair: query_doc_pair, user: judge, updated_at: Time.zone.now)
-      judgement.rating = 4
-      judgement.explanation = "Eric writing code.  Judge is #{judge.email}"
+      begin
+        judgement = llm_service.make_judgement(judge, query_doc_pair)
+      rescue RuntimeError => e
+        case e.message
+        when /401/
+          raise # we can't do anything about this, so pass it up
+        else
+          judgement.explanation = e.full_message
+          judgement.unrateable = true
+        end
+      end
+
       judgement.save!
-      sleep 1
-      broadcast_update(book, counter += 1, query_doc_pair, judge)
+      counter += 1
+
+      if number_of_pairs.nil?
+        broadcast_update_kraken_mode(book, counter, query_doc_pair, judge)
+      else
+        broadcast_update(book, number_of_pairs - counter, query_doc_pair, judge)
+      end
     end
+    broadcast_complete(book, judge)
     UpdateCaseJob.perform_later book
   end
+  # rubocop:enable Metrics/MethodLength
 
   private
 
@@ -42,4 +61,22 @@ def broadcast_update book, counter, query_doc_pair, judge
       locals:  { book: book, counter: counter, qdp: query_doc_pair, judge: judge }
     )
   end
+
+  def broadcast_update_kraken_mode book, counter, query_doc_pair, judge
+    Turbo::StreamsChannel.broadcast_render_to(
+      :notifications,
+      target:  'notifications',
+      partial: 'books/update_kraken_mode',
+      locals:  { book: book, counter: counter, qdp: query_doc_pair, judge: judge }
+    )
+  end
+
+  def broadcast_complete book, judge
+    Turbo::StreamsChannel.broadcast_render_to(
+      :notifications,
+      target:  'notifications',
+      partial: 'books/complete',
+      locals:  { book: book, judge: judge }
+    )
+  end
 end

app/services/llm_service.rb

@@ -8,18 +8,35 @@ def initialize openai_key, _opts = {}
     @openai_key = openai_key
   end
 
-  def make_judgement _system_prompt, _user_prompt
-    # scott write code.
+  def make_judgement judge, query_doc_pair
+    user_prompt = make_user_prompt query_doc_pair
+    results = get_llm_response user_prompt, judge.prompt
+    puts 'Here are the results'
+    puts results
+    judgement = Judgement.new(query_doc_pair: query_doc_pair, user: judge)
+    judgement.rating = results[:judgment]
+    judgement.explanation = results[:explanation]
 
-    {
-      explanation: 'Hi scott',
-      rating:      rand(4),
-    }
+    judgement
+  end
+
+  def make_user_prompt query_doc_pair
+    fields = JSON.parse(query_doc_pair.document_fields).to_yaml
+
+    user_prompt = <<~TEXT
+      Query: #{query_doc_pair.query_text}
+
+      doc1:
+        #{fields}
+    TEXT
+
+    user_prompt
   end
 
   # rubocop:disable Metrics/MethodLength
   def get_llm_response user_prompt, system_prompt
     uri = URI('https://api.openai.com/v1/chat/completions')
+
     headers = {
       'Content-Type'  => 'application/json',
       'Authorization' => "Bearer #{@openai_key}",
@@ -38,20 +55,19 @@ def get_llm_response user_prompt, system_prompt
     end
     if response.is_a?(Net::HTTPSuccess)
       json_response = JSON.parse(response.body)
-      # puts json_response
       content = json_response['choices']&.first&.dig('message', 'content')
-      # puts content
       parsed_content = begin
         JSON.parse(content)
       rescue StandardError
         {}
       end
 
+      parsed_content = parsed_content['response'] if parsed_content['response']
       # puts "here is parsed"
       # puts parsed_content
       {
-        explanation: parsed_content['response']['explanation'],
-        judgment:    parsed_content['response']['judgment_value'],
+        explanation: parsed_content['explanation'],
+        judgment:    parsed_content['judgment'],
       }
     else
       raise "Error: #{response.code} - #{response.message}"

app/views/ai_judges/prompts/_form.html.erb

@@ -91,3 +91,77 @@
 
 
 <% end %>
+
+
+<script language="javascript">
+  const codemirrorEditor = CodeMirror.fromTextArea(document.getElementById('query_doc_pair_document_fields'), {
+    mode: "application/json",
+    lineNumbers: true,
+    gutters: ["CodeMirror-lint-markers"],
+    lint: true
+  });
+  codemirrorEditor.setSize(null, 400);
+
+  let content = codemirrorEditor.getValue();
+  try {
+    codemirrorEditor.setValue(JSON.stringify(JSON.parse(content), null, 2));
+  } catch(e) {
+    console.error("Invalid JSON");
+  }
+
+
+  // Function to validate JavaScript syntax
+  function validateSyntax(code) {
+    try {
+      // We can NOT validate this until we have working functions.
+      // like setScore etc defined in this space.
+      // Use the built-in `eval` function to check the syntax
+      //eval(code);
+      return true;
+    } catch (error) {
+      return false;
+    }
+  }
+
+  // Validate syntax on code change
+  codemirrorEditor.on("change", function() {
+    var code = codemirrorEditor.getValue();
+    var isValidSyntax = validateSyntax(code);
+    var errorAnnotation = isValidSyntax ? [] : [{
+      from: CodeMirror.Pos(0, 0),
+      to: CodeMirror.Pos(0, 0),
+      message: "Syntax Error",
+      severity: "error"
+    }];
+    CodeMirror.signal(codemirrorEditor, "lint", codemirrorEditor, errorAnnotation);
+  });
+
+  // Add linting annotations to show syntax errors
+  codemirrorEditor.on("lint", function(_, annotations) {
+    codemirrorEditor.clearGutter("CodeMirror-lint-markers");
+    annotations.forEach(function(annotation) {
+      codemirrorEditor.setGutterMarker(annotation.from.line, "CodeMirror-lint-markers", createMarker(annotation));
+    });
+  });
+
+  // Helper function to create gutter marker
+  function createMarker(annotation) {
+    var marker = document.createElement("div");
+    marker.className = "lint-marker " + annotation.severity;
+    marker.title = annotation.message;
+    return marker;
+  }  
+</script>
+
+<style>
+.lint-marker {
+  width: 16px;
+  height: 16px;
+  background-color: red;
+  border-radius: 50%;
+}
+
+.lint-marker.error {
+  background-color: red;
+}
+</style>

app/views/books/_complete.turbo_stream.erb

@@ -0,0 +1,14 @@
+<%= turbo_stream.replace "notification-book-#{book.id}" do %>
+  <span id="notification-book-<%= book.id %>">    
+    <div class="alert alert-info" role="alert">
+      Judging by <i class="bi bi-robot"></i> <%= judge.name %> completed.  
+      Please <%= link_to book_path(book) do %>
+        reload the page
+        <i
+          class="bi arrow-clockwise"
+          aria-hidden="true"
+        ></i>
+      <% end %>.
+    </div>
+  </span>
+<% end %>

app/views/books/_update_kraken_mode.turbo_stream.erb

@@ -0,0 +1,13 @@
+<%= turbo_stream.replace "notification-book-#{book.id}" do %>
+  <span id="notification-book-<%= book.id %>">
+
+    <div class="alert alert-warning" role="alert" >  
+      <%= image_tag "kraken.png", size:"60x60" %>    
+      Query/Doc Pairs So Far: <code><%= counter %></code>. 
+      <br/>
+      <p class="text-truncate">
+        Processing query <code><%= qdp.query_text %></code>.
+      </p>
+    </div>
+  </span>
+<% end %>

app/views/judgements/_moar_judgements_needed.html.erb

@@ -23,7 +23,7 @@
     </div>
   <% else %>
     <div class="alert alert-success" role="alert">
-    Congratulations, all query/doc pairs have been judged by three seperate judges.
+    Congratulations, all query/doc pairs have been judged by three separate judges.
     </div>      
   <% end %>
 <% end %>

db/migrate/20231222115905_add_all_query_attributes_to_query_docs.rb

@@ -1,6 +1,6 @@
 class AddAllQueryAttributesToQueryDocs < ActiveRecord::Migration[7.1]
   # As we do more automatic syncing of books and cases it becomes more
-  # apparent that having "query" and "ratings" as seperate tables for supporting Cases,
+  # apparent that having "query" and "ratings" as separate tables for supporting Cases,
   # while having "query_doc_pair" and "judgements" for supporting Books is
   # maybe not the right way.   
   #