Show Available Models
Show available models. Right now we are only serving one model at a time.
Responses
Response samples
- 200
{- "object": "list",
- "data": [
- {
- "id": "mistralai/Mistral-7B-v0.1",
- "object": "model",
- "created": 1695658438,
- "owned_by": "vllm",
- "root": "mistralai/Mistral-7B-v0.1",
- "parent": null,
- "permission": [
- {
- "id": "modelperm-b1325bff197f4f95a1c50aa2d4a484d3",
- "object": "model_permission",
- "created": 1695658438,
- "allow_create_engine": false,
- "allow_sampling": true,
- "allow_logprobs": true,
- "allow_search_indices": false,
- "allow_view": true,
- "allow_fine_tuning": false,
- "organization": "*",
- "group": null,
- "is_blocking": false
}
]
}
]
}
Create Chat Completion
Completion API similar to OpenAI's API.
See https://platform.openai.com/docs/api-reference/chat/create for the full API specification. This API mimics the OpenAI ChatCompletion API.
NOTE: Currently we do not support the following features: - function_call (Users should implement this by themselves) - logit_bias (to be supported by vLLM engine)
Request Body schema: application/json
model required | string (Model) |
required | Messages (string) or Array of Messages (objects) (Messages) |
temperature | number (Temperature) Default: 0.7 |
top_p | number (Top P) Default: 1 |
n | integer (N) Default: 1 |
max_tokens | integer (Max Tokens) Default: 8192 |
Array of Stop (strings) or Stop (string) (Stop) | |
stream | boolean (Stream) Default: false |
presence_penalty | number (Presence Penalty) Default: 0 |
frequency_penalty | number (Frequency Penalty) Default: 0 |
object (Logit Bias) | |
user | string (User) |
best_of | integer (Best Of) |
top_k | integer (Top K) Default: -1 |
ignore_eos | boolean (Ignore Eos) Default: false |
use_beam_search | boolean (Use Beam Search) Default: false |
Responses
Request samples
- Payload
{- "model": "string",
- "messages": "string",
- "temperature": 0.7,
- "top_p": 1,
- "n": 1,
- "max_tokens": 8192,
- "stop": [
- "string"
], - "stream": false,
- "presence_penalty": 0,
- "frequency_penalty": 0,
- "logit_bias": {
- "property1": 0,
- "property2": 0
}, - "user": "string",
- "best_of": 0,
- "top_k": -1,
- "ignore_eos": false,
- "use_beam_search": false
}
Response samples
- 200
- 422
{- "id": "cmpl-2759a099e3c9429ca88b66b8ab9a9965",
- "object": "chat.completion",
- "created": 1695318445,
- "model": "mistralai/Mistral-7B-v0.1",
- "choices": [
- {
- "index": 0,
- "message": {
- "role": "assistant",
- "content": "beavers, busy builders,\ndam and pond, their kingdom."
}, - "finish_reason": "stop"
}
], - "usage": {
- "prompt_tokens": 17,
- "total_tokens": 35,
- "completion_tokens": 18
}
}
Create Completion
Completion API similar to OpenAI's API.
See https://platform.openai.com/docs/api-reference/completions/create for the API specification. This API mimics the OpenAI Completion API.
NOTE: Currently we do not support the following features: - suffix (the language models we currently support do not support suffix) - logit_bias (to be supported by vLLM engine)
Request Body schema: application/json
model required | string (Model) |
required | Array of Prompt (integers) or Array of Prompt (integers) or Prompt (string) or Array of Prompt (strings) (Prompt) |
suffix | string (Suffix) |
max_tokens | integer (Max Tokens) Default: 16 |
temperature | number (Temperature) Default: 1 |
top_p | number (Top P) Default: 1 |
n | integer (N) Default: 1 |
stream | boolean (Stream) Default: false |
logprobs | integer (Logprobs) |
echo | boolean (Echo) Default: false |
Array of Stop (strings) or Stop (string) (Stop) | |
presence_penalty | number (Presence Penalty) Default: 0 |
frequency_penalty | number (Frequency Penalty) Default: 0 |
best_of | integer (Best Of) |
object (Logit Bias) | |
user | string (User) |
top_k | integer (Top K) Default: -1 |
ignore_eos | boolean (Ignore Eos) Default: false |
use_beam_search | boolean (Use Beam Search) Default: false |
Responses
Request samples
- Payload
{- "model": "string",
- "prompt": [
- 0
], - "suffix": "string",
- "max_tokens": 16,
- "temperature": 1,
- "top_p": 1,
- "n": 1,
- "stream": false,
- "logprobs": 0,
- "echo": false,
- "stop": [
- "string"
], - "presence_penalty": 0,
- "frequency_penalty": 0,
- "best_of": 0,
- "logit_bias": {
- "property1": 0,
- "property2": 0
}, - "user": "string",
- "top_k": -1,
- "ignore_eos": false,
- "use_beam_search": false
}
Response samples
- 200
- 422
{- "id": "cmpl-605c15936dd441aeb08f765035c9b88e",
- "object": "text_completion",
- "created": 1695660046,
- "model": "mistralai/Mistral-7B-v0.1",
- "choices": [
- {
- "index": 0,
- "text": " the “Mistral”.\n\nIt is a cold and very dry wind that comes from the north-northwest, blowing from the Mediterranean Sea to",
- "logprobs": {
- "text_offset": [
- 0,
- 4,
- 6,
- 7,
- 10,
- 13,
- 15,
- 21,
- 27,
- 29,
- 32,
- 34,
- 39,
- 43,
- 48,
- 52,
- 57,
- 62,
- 68,
- 73,
- 77,
- 83,
- 84,
- 85,
- 89,
- 93,
- 94,
- 102,
- 107,
- 111,
- 125,
- 129
], - "token_logprobs": [
- -0.13521480560302734,
- -1.726870059967041,
- -0.5047908425331116,
- -0.00127948890440166,
- -0.000016212332411669195,
- -1.4993023872375488,
- -0.52951580286026,
- -0.029912520200014114,
- -0.8960363268852234,
- -0.3480968475341797,
- -0.06601884216070175,
- -0.35511890053749084,
- -0.9667784571647644,
- -4.5353264808654785,
- -0.2424573302268982,
- -0.018769746646285057,
- -0.5242247581481934,
- -2.609259605407715,
- -0.00840850081294775,
- -0.026951059699058533,
- -0.21602609753608704,
- -1.3617815971374512,
- -1.4199233055114746,
- -0.062266286462545395,
- -0.15606233477592468,
- -1.16431725025177,
- -1.035098671913147,
- -0.6018842458724976,
- -0.004355231299996376,
- -1.2897896766662598,
- -0.18369901180267334,
- -0.494781494140625
], - "tokens": [
- "▁the",
- "▁“",
- "M",
- "ist",
- "ral",
- "”.",
- "<0x0A>",
- "<0x0A>",
- "It",
- "▁is",
- "▁a",
- "▁cold",
- "▁and",
- "▁very",
- "▁dry",
- "▁wind",
- "▁that",
- "▁comes",
- "▁from",
- "▁the",
- "▁north",
- "-",
- "n",
- "orth",
- "west",
- ",",
- "▁blowing",
- "▁from",
- "▁the",
- "▁Mediterranean",
- "▁Sea",
- "▁to"
], - "top_logprobs": [
- {
- "▁the": -0.13521480560302734
}, - {
- "▁“": -1.726870059967041
}, - {
- "M": -0.5047908425331116
}, - {
- "ist": -0.00127948890440166
}, - {
- "ral": -0.000016212332411669195
}, - {
- "”.": -1.4993023872375488
}, - {
- "<0x0A>": -0.52951580286026
}, - {
- "<0x0A>": -0.029912520200014114
}, - {
- "It": -0.8960363268852234
}, - {
- "▁is": -0.3480968475341797
}, - {
- "▁a": -0.06601884216070175
}, - {
- "▁cold": -0.35511890053749084
}, - {
- "▁and": -0.9667784571647644
}, - {
- "▁very": -4.5353264808654785
}, - {
- "▁dry": -0.2424573302268982
}, - {
- "▁wind": -0.018769746646285057
}, - {
- "▁that": -0.5242247581481934
}, - {
- "▁comes": -2.609259605407715
}, - {
- "▁from": -0.00840850081294775
}, - {
- "▁the": -0.026951059699058533
}, - {
- "▁north": -0.21602609753608704
}, - {
- "-": -1.3617815971374512
}, - {
- "n": -1.4199233055114746
}, - {
- "orth": -0.062266286462545395
}, - {
- "west": -0.15606233477592468
}, - {
- ",": -1.16431725025177
}, - {
- "▁blowing": -1.035098671913147
}, - {
- "▁from": -0.6018842458724976
}, - {
- "▁the": -0.004355231299996376
}, - {
- "▁Mediterranean": -1.2897896766662598
}, - {
- "▁Sea": -0.18369901180267334
}, - {
- "▁to": -0.494781494140625
}
]
}, - "finish_reason": "length"
}
], - "usage": {
- "prompt_tokens": 11,
- "total_tokens": 43,
- "completion_tokens": 32
}
}