VllmServerConfigΒΆ

class VllmServerConfig ( BaseServerConfig )

vLLM Omni Docker server config.

Polymorphic Type:

type: vllm

All BaseServerConfig types:

Fields:

env_pathOptional [ str ] = None

Path to a Python environment directory (virtualenv/conda).

modelstr = "meta-llama/Meta-Llama-3-8B-Instruct"

Model name exposed to the benchmark client.

hoststr = "localhost"

Host address for the server

portint = 8000

Port number for the server

api_keystr = "token-abc123"

API key for server authentication

gpu_idsOptional [ list [ int ] ] = None

List of GPU IDs to use (None means auto-assign)

startup_timeoutfloat = 300.0

Timeout in seconds for server startup

health_check_intervalfloat = 2.0

Interval in seconds between health checks

require_contiguous_gpusbool = True

Require contiguous GPU allocation (e.g., GPUs 0,1,2 instead of 0,2,5)

tensor_parallel_sizeint = 1

Number of GPUs for tensor parallelism

dtypestr = "auto"

Data type for model weights (auto, float16, bfloat16, etc.)

max_model_lenOptional [ int ] = None

Maximum model context length

additional_argsOptional [ str ] = "{}"

Additional engine-specific arguments as JSON string, dict, or None.

api_baseOptional [ str ] = None

External API base URL for the server.

health_urlOptional [ str ] = None

Health endpoint URL for the server.

setup_dirOptional [ str ] = None

Source checkout used by subprocess engines.

image : str = "vllm-omni:0.21-local"

container_name : Optional [ str ] = None

container_port : Optional [ int ] = None

engine_args : list [ str ] = []

volumes : list [ str ] = []

docker_gpus : Optional [ str ] = None

docker_runtime : Optional [ str ] = "nvidia"

ipc_mode : Optional [ str ] = "host"

hf_model : str = ""

deploy_config : str = ""

container_deploy_config : Optional [ str ] = None

env : Optional [ typing.Dict[str, str] ] = None

pass_env : list [ str ] = []

bootstrap : Optional [ str ] = None