app:
  description: 'LLM-as-a-judge for Exercise 5: grades ONE criterion of a chatbot answer and returns {"reasoning", "verdict"} as JSON'
  icon: ⚖️
  icon_background: '#E0F2FE'
  icon_type: emoji
  mode: workflow
  name: 'Exercise 5: RAG Judge'
  use_icon_as_answer_icon: false
dependencies:
- current_identifier: null
  type: marketplace
  value:
    marketplace_plugin_unique_identifier: langgenius/azure_openai:0.0.69@c744f6bf60699c296307a4420094a7f88b4f6e7fdf353b67b2b88ddb455074b2
    version: null
kind: app
version: 0.7.0
workflow:
  conversation_variables: []
  environment_variables: []
  features:
    file_upload:
      enabled: false
    opening_statement: ''
    retriever_resource:
      enabled: false
    sensitive_word_avoidance:
      enabled: false
    speech_to_text:
      enabled: false
    suggested_questions: []
    suggested_questions_after_answer:
      enabled: false
    text_to_speech:
      enabled: false
      language: ''
      voice: ''
  graph:
    edges:
    - data:
        isInIteration: false
        isInLoop: false
        sourceType: start
        targetType: llm
      id: start-source-judge-target
      source: start
      sourceHandle: source
      target: judge
      targetHandle: target
      type: custom
      zIndex: 0
    - data:
        isInIteration: false
        isInLoop: false
        sourceType: llm
        targetType: end
      id: judge-source-end-target
      source: judge
      sourceHandle: source
      target: end
      targetHandle: target
      type: custom
      zIndex: 0
    nodes:
    - data:
        desc: What the tests send to the judge
        selected: false
        title: Start
        type: start
        variables:
        - label: criterion
          max_length: 4000
          options: []
          required: true
          type: paragraph
          variable: criterion
        - label: question
          max_length: 4000
          options: []
          required: true
          type: paragraph
          variable: question
        - label: answer
          max_length: 20000
          options: []
          required: true
          type: paragraph
          variable: answer
        - label: context
          max_length: 60000
          options: []
          required: false
          type: paragraph
          variable: context
        - label: reference
          max_length: 4000
          options: []
          required: false
          type: paragraph
          variable: reference
      height: 194
      id: start
      position:
        x: 80
        y: 280
      positionAbsolute:
        x: 80
        y: 280
      selected: false
      sourcePosition: right
      targetPosition: left
      type: custom
      width: 242
    - data:
        context:
          enabled: false
          variable_selector: []
        desc: Grades one criterion. Temperature 0, reasoning first, binary verdict, JSON only.
        model:
          completion_params:
            temperature: 0
          mode: chat
          name: gpt-35-turbo-16k
          provider: langgenius/azure_openai/azure_openai
        prompt_template:
        - id: judge-system
          role: system
          text: 'You are a strict quality judge for a chatbot that answers questions
            about Jira issues. You grade exactly ONE criterion.


            Rules:

            - Use ONLY the information given below. Never use your own knowledge
            about the topic.

            - The CONTEXT is what the chatbot retrieved from its knowledge base. The
            REFERENCE is what a correct answer must contain. Either may be empty.

            - Work step by step: first check the ANSWER against the CRITERION, then
            decide.

            - PASS only if the criterion is fully met. If you are unsure, FAIL.

            - Answer length and writing style do not matter, only the criterion.


            Reply with JSON only, no markdown, in exactly this shape:

            {"reasoning": "<1 to 3 short sentences>", "verdict": "PASS"}

            or

            {"reasoning": "<1 to 3 short sentences>", "verdict": "FAIL"}'
        - id: judge-user
          role: user
          text: 'CRITERION:

            {{#start.criterion#}}


            QUESTION:

            {{#start.question#}}


            CONTEXT:

            {{#start.context#}}


            REFERENCE:

            {{#start.reference#}}


            ANSWER:

            {{#start.answer#}}'
        selected: false
        title: Judge
        type: llm
        variables: []
        vision:
          enabled: false
      height: 98
      id: judge
      position:
        x: 400
        y: 280
      positionAbsolute:
        x: 400
        y: 280
      selected: false
      sourcePosition: right
      targetPosition: left
      type: custom
      width: 242
    - data:
        desc: ''
        outputs:
        - value_selector:
          - judge
          - text
          value_type: string
          variable: result
        selected: false
        title: End
        type: end
      height: 90
      id: end
      position:
        x: 720
        y: 280
      positionAbsolute:
        x: 720
        y: 280
      selected: false
      sourcePosition: right
      targetPosition: left
      type: custom
      width: 242
    viewport:
      x: 0
      y: 0
      zoom: 1