{
  "measuredAt": "2026-10-17T10:52:57Z",
  "baseUrl": "https://www.flonno.com/v1",
  "operatorRun": true,
  "scope": "Synthetic smoke evaluation; 3 trials per main case, concurrency 2. Not independent or statistically representative; no full-load or general accuracy claim.",
  "requests": [
    {
      "case": "tool",
      "index": 0,
      "durationMs": 1179.41,
      "firstEventMs": null,
      "content": "",
      "reasoningCharacters": 0,
      "calls": [
        {
          "function": {
            "arguments": "{\"product\":{\"sku\": \"ABC-17\", \"warehouse\": \"Zurich\"}}",
            "name": "lookup_stock"
          },
          "id": "call_a033d48374b8cd58",
          "type": "function"
        }
      ],
      "usage": {
        "completion_tokens": 43,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 329,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 372
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "tool",
      "index": 1,
      "durationMs": 1861.6,
      "firstEventMs": null,
      "content": "",
      "reasoningCharacters": 0,
      "calls": [
        {
          "function": {
            "arguments": "{\"product\":{\"sku\": \"ABC-17\", \"warehouse\": \"Zurich\"}}",
            "name": "lookup_stock"
          },
          "id": "call_984a25c6860a5b63",
          "type": "function"
        }
      ],
      "usage": {
        "completion_tokens": 43,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 329,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 372
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "tool",
      "index": 2,
      "durationMs": 2165.51,
      "firstEventMs": null,
      "content": "",
      "reasoningCharacters": 0,
      "calls": [
        {
          "function": {
            "arguments": "{\"product\":{\"sku\": \"ABC-17\", \"warehouse\": \"Zurich\"}}",
            "name": "lookup_stock"
          },
          "id": "call_cd5365e00717b9be",
          "type": "function"
        }
      ],
      "usage": {
        "completion_tokens": 43,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 329,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 372
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "stream_tool",
      "index": 0,
      "durationMs": 2443.51,
      "firstEventMs": 2200.13,
      "content": "",
      "reasoningCharacters": 0,
      "calls": [
        {
          "id": "call_8eb9ba7cd6357695",
          "type": "function",
          "function": {
            "name": "lookup_stock",
            "arguments": "{\"product\":{\"sku\": \"ABC-17\", \"warehouse\": \"Zurich\"}}"
          }
        }
      ],
      "usage": {
        "completion_tokens": 43,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 329,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 372
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "stream_tool",
      "index": 1,
      "durationMs": 2460.0,
      "firstEventMs": 2218.63,
      "content": "",
      "reasoningCharacters": 0,
      "calls": [
        {
          "id": "call_e16fe1be036c7a30",
          "type": "function",
          "function": {
            "name": "lookup_stock",
            "arguments": "{\"product\":{\"sku\": \"ABC-17\", \"warehouse\": \"Zurich\"}}"
          }
        }
      ],
      "usage": {
        "completion_tokens": 43,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 329,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 372
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "stream_tool",
      "index": 2,
      "durationMs": 2508.79,
      "firstEventMs": 2221.25,
      "content": "",
      "reasoningCharacters": 0,
      "calls": [
        {
          "id": "call_419f334a21d90e66",
          "type": "function",
          "function": {
            "name": "lookup_stock",
            "arguments": "{\"product\":{\"sku\": \"ABC-17\", \"warehouse\": \"Zurich\"}}"
          }
        }
      ],
      "usage": {
        "completion_tokens": 43,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 329,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 372
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "json",
      "index": 0,
      "durationMs": 3238.71,
      "firstEventMs": 1671.79,
      "content": "{\"city\": \"Zurich\", \"count\": 3}",
      "reasoningCharacters": 0,
      "calls": [],
      "usage": {
        "completion_tokens": 15,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 25,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 40
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "json",
      "index": 1,
      "durationMs": 3399.08,
      "firstEventMs": 1726.98,
      "content": "{\"city\": \"Zurich\", \"count\": 3}",
      "reasoningCharacters": 0,
      "calls": [],
      "usage": {
        "completion_tokens": 15,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 25,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 40
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "json",
      "index": 2,
      "durationMs": 3255.89,
      "firstEventMs": 1694.28,
      "content": "{\"city\": \"Zurich\", \"count\": 3}",
      "reasoningCharacters": 0,
      "calls": [],
      "usage": {
        "completion_tokens": 15,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 25,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 40
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "parallel_tool",
      "index": 0,
      "durationMs": 2256.18,
      "firstEventMs": 2012.49,
      "content": "",
      "reasoningCharacters": 0,
      "calls": [
        {
          "id": "call_df1d1bba4caa9cce",
          "type": "function",
          "function": {
            "name": "lookup_stock",
            "arguments": "{\"product\":{\"sku\": \"ABC-17\", \"warehouse\": \"Zurich\"}}"
          }
        },
        {
          "id": "call_f0815db78b4077f5",
          "type": "function",
          "function": {
            "name": "lookup_stock",
            "arguments": "{\"product\":{\"sku\": \"XYZ-42\", \"warehouse\": \"Bern\"}}"
          }
        }
      ],
      "usage": {
        "completion_tokens": 84,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 339,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 423
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "parallel_tool",
      "index": 1,
      "durationMs": 3217.43,
      "firstEventMs": 2930.01,
      "content": "",
      "reasoningCharacters": 0,
      "calls": [
        {
          "id": "call_872cb6f5ea482f1a",
          "type": "function",
          "function": {
            "name": "lookup_stock",
            "arguments": "{\"product\":{\"sku\": \"ABC-17\", \"warehouse\": \"Zurich\"}}"
          }
        },
        {
          "id": "call_b8f39f237c3b8e46",
          "type": "function",
          "function": {
            "name": "lookup_stock",
            "arguments": "{\"product\":{\"sku\": \"XYZ-42\", \"warehouse\": \"Bern\"}}"
          }
        }
      ],
      "usage": {
        "completion_tokens": 84,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 339,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 423
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "parallel_tool",
      "index": 2,
      "durationMs": 3154.51,
      "firstEventMs": 2912.76,
      "content": "",
      "reasoningCharacters": 0,
      "calls": [
        {
          "id": "call_a88322f54a99ad38",
          "type": "function",
          "function": {
            "name": "lookup_stock",
            "arguments": "{\"product\":{\"sku\": \"ABC-17\", \"warehouse\": \"Zurich\"}}"
          }
        },
        {
          "id": "call_c46f9ba4c3e81aab",
          "type": "function",
          "function": {
            "name": "lookup_stock",
            "arguments": "{\"product\":{\"sku\": \"XYZ-42\", \"warehouse\": \"Bern\"}}"
          }
        }
      ],
      "usage": {
        "completion_tokens": 84,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 339,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 423
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "thinking_json",
      "index": 0,
      "durationMs": 2773.52,
      "firstEventMs": 1314.55,
      "content": "{\"city\":\"Zurich\",\"count\":3}",
      "reasoningCharacters": 150,
      "calls": [],
      "usage": {
        "completion_tokens": 57,
        "completion_tokens_details": {
          "reasoning_tokens": 44
        },
        "prompt_tokens": 23,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 80
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "thinking_json",
      "index": 1,
      "durationMs": 2786.86,
      "firstEventMs": 1318.73,
      "content": "{\"city\":\"Zurich\",\"count\":3}",
      "reasoningCharacters": 150,
      "calls": [],
      "usage": {
        "completion_tokens": 57,
        "completion_tokens_details": {
          "reasoning_tokens": 44
        },
        "prompt_tokens": 23,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 80
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "thinking_json",
      "index": 2,
      "durationMs": 2780.01,
      "firstEventMs": 1328.99,
      "content": "{\"city\":\"Zurich\",\"count\":3}",
      "reasoningCharacters": 150,
      "calls": [],
      "usage": {
        "completion_tokens": 57,
        "completion_tokens_details": {
          "reasoning_tokens": 44
        },
        "prompt_tokens": 23,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 80
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "concurrency_2_json",
      "index": 0,
      "durationMs": 2570.97,
      "firstEventMs": 1339.31,
      "content": "{\"city\": \"Zurich\", \"count\": 3}",
      "reasoningCharacters": 0,
      "calls": [],
      "usage": {
        "completion_tokens": 15,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 25,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 40
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "case": "concurrency_2_json",
      "index": 1,
      "durationMs": 4709.63,
      "firstEventMs": 3620.42,
      "content": "{\"city\": \"Zurich\", \"count\": 3}",
      "reasoningCharacters": 0,
      "calls": [],
      "usage": {
        "completion_tokens": 15,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 25,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 40
      },
      "finished": true,
      "done": true,
      "passed": true
    },
    {
      "durationMs": 1820.48,
      "firstEventMs": null,
      "content": "",
      "reasoningCharacters": 0,
      "calls": [
        {
          "function": {
            "arguments": "{\"product\":{\"sku\": \"ABC-17\", \"warehouse\": \"Zurich\"}}",
            "name": "lookup_stock"
          },
          "id": "call_adb6fcf8dc0f09b0",
          "type": "function"
        }
      ],
      "usage": {
        "completion_tokens": 43,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 335,
        "prompt_tokens_details": {
          "cached_tokens": 0
        },
        "total_tokens": 378
      },
      "finished": true,
      "done": true,
      "case": "loop_call",
      "passed": true
    },
    {
      "durationMs": 555.77,
      "firstEventMs": 502.26,
      "content": "The stock for SKU **ABC-17** in the **Zurich** warehouse is **17 units**.",
      "reasoningCharacters": 0,
      "calls": [],
      "usage": {
        "completion_tokens": 25,
        "completion_tokens_details": {
          "reasoning_tokens": 0
        },
        "prompt_tokens": 401,
        "prompt_tokens_details": {
          "cached_tokens": 377
        },
        "total_tokens": 426
      },
      "finished": true,
      "done": true,
      "case": "loop_result",
      "passed": true
    }
  ],
  "billingVerification": {
    "balanceNano": "999676370",
    "pendingCount": 0,
    "settledCount": 19,
    "usageLedgerNano": "323630",
    "initialBalanceNano": "1000000000",
    "expectedUsageNano": "323630",
    "walletDebitNano": "323630",
    "matched": true
  },
  "gatewayImage": "flonno:b2fa9010a80ad5019a30f76bb2092cee5486d52c",
  "limitations": [
    "Operator-run, not independent; 3 trials per main case.",
    "Synthetic prompts, no customer data. No statistical guarantee or general task-accuracy benchmark.",
    "Concurrency tested at 2; no saturation/load test or SLA.",
    "tool_choice=required is unsupported. The supported automatic tool selection was evaluated.",
    "Full JSON Schema compliance and named forced-tool selection were not evaluated.",
    "Public minute probes disable thinking; this separate evaluation includes medium reasoning."
  ]
}