Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 48 additions & 1 deletion backend/node/create_nodes/providers_schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
},
"plugin": {
"$comment": "Plugin to be loaded by the Studio to interact with the provider",
"enum": ["openai-compatible", "ollama", "anthropic", "google"]
"enum": ["openai-compatible", "ollama", "anthropic", "google", "custom"]
},
"plugin_config": {
"type": "object"
Expand Down Expand Up @@ -453,6 +453,53 @@
}
}
}
},
{
"if": {
"properties": {
"plugin": { "const": "custom" }
}
},
"then": {
"properties": {
"plugin_config": {
"type": "object",
"additionalProperties": false,
"properties": {
"api_key_env_var": {
"type": "string",
"$comment": "Environment variable that contains the API key",
"minLength": 1
},
"api_url": {
"type": ["string", "null"],
"$comment": "URL of the API endpoint"
}
},
"required": ["api_key_env_var", "api_url"]
},
"config": {
"type": "object",
"additionalProperties": false,
"properties": {
"temperature": {
"type": "number",
"minimum": 0,
"maximum": 1,
"multipleOf": 0.05,
"default": 0.75
},
"max_tokens": {
"type": "integer",
"minimum": 100,
"maximum": 2000,
"multipleOf": 10,
"default": 500
}
}
}
}
}
}
],
"required": ["provider", "model", "config", "plugin", "plugin_config"],
Expand Down
20 changes: 20 additions & 0 deletions backend/validators/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import typing
import contextlib
import dataclasses
import os

from copy import deepcopy
from pathlib import Path
Expand Down Expand Up @@ -145,6 +146,25 @@ async def _get_snap_from_validators(
host_data["mock_response"] = val.llmprovider.plugin_config[
"mock_response"
]
if val.llmprovider.plugin == "custom":
plugin_config = {
str(k): str(v) for k, v in val.llmprovider.plugin_config.items()
}
plugin_config["api_key_env_var"] = str(
os.getenv(val.llmprovider.plugin_config["api_key_env_var"])
)

host_data["custom_plugin_data"] = {
"model": str(val.llmprovider.model),
"config": {
str(k): str(v) if isinstance(v, (int, float, bool)) else v
for k, v in val.llmprovider.config.items()
},
"plugin_config": plugin_config,
}
val.llmprovider.plugin = (
"openai-compatible" # so genvm thinks it is an implemented plugin
)
current_validators.append(SingleValidatorSnapshot(val, host_data))
return Snapshot(
nodes=current_validators, genvm_config_path=self._genvm_config.new_path
Expand Down
90 changes: 77 additions & 13 deletions backend/validators/greyboxing.lua
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,65 @@ local function get_mock_response_from_table(table, message)
return "no match"
end

local function handle_custom_plugin(ctx, args, mapped_prompt)
local custom_plugin_data = ctx.host_data.custom_plugin_data
local content
if mapped_prompt.prompt.system_message then
content = mapped_prompt.prompt.system_message .. mapped_prompt.prompt.user_message
else
content = mapped_prompt.prompt.user_message
end
local payload = {
model = custom_plugin_data.model,
messages = {
{role = "user", content = content}
}
}

-- Convert config values to appropriate types because we could only send strings to genvm
for k, v in pairs(custom_plugin_data.config) do
local num = tonumber(v)
if num then
payload[k] = num
elseif v == "true" then
payload[k] = true
elseif v == "false" then
payload[k] = false
else
payload[k] = v
end
end

local api_request = lib.rs.request(ctx, {
method = 'POST',
url = custom_plugin_data.plugin_config.api_url,
headers = {
["Content-Type"] = "application/json",
["Authorization"] = "Bearer " .. custom_plugin_data.plugin_config.api_key_env_var
},
body = lib.rs.json_stringify(payload),
json = true
})

local response_data = api_request.body
local response_status = api_request.status

if response_status ~= 200 then
return false, lib.rs.json_stringify(response_data.error)
end
Comment thread
kstroobants marked this conversation as resolved.
Resolved

local result
if args.template == "EqComparative" or args.template == "EqNonComparativeValidator" then
-- These equivalence principles ask also for a reasoning but we only want the result
local json_response = lib.rs.json_parse(response_data.choices[1].message.content)
result = json_response.result
else
result = response_data.choices[1].message.content
end
Comment thread
kstroobants marked this conversation as resolved.
Resolved

return true, result
end

local function just_in_backend(ctx, args, mapped_prompt)
---@cast mapped_prompt MappedPrompt
---@cast args LLMExecPromptPayload | LLMExecPromptTemplatePayload
Expand Down Expand Up @@ -43,19 +102,24 @@ local function just_in_backend(ctx, args, mapped_prompt)
mapped_prompt.prompt.use_max_completion_tokens = false

for i = 1,3 do
local request = {
provider = provider_id,
model = model,
prompt = mapped_prompt.prompt,
format = mapped_prompt.format,
}

local success, result = pcall(function ()
return llm.rs.exec_prompt_in_provider(
ctx,
request
)
end)
local success, result
if ctx.host_data.custom_plugin_data then
success, result = handle_custom_plugin(ctx, args, mapped_prompt)
else
local request = {
provider = provider_id,
model = model,
prompt = mapped_prompt.prompt,
format = mapped_prompt.format,
}

success, result = pcall(function ()
return llm.rs.exec_prompt_in_provider(
ctx,
request
)
end)
end

lib.log{level = "debug", message = "executed with", success = success, type = type(result), res = result}
if success then
Expand Down
62 changes: 61 additions & 1 deletion backend/validators/llm.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@
import os
import sys
import dataclasses

import aiohttp
from pathlib import Path
import json

from dotenv import load_dotenv

Expand Down Expand Up @@ -116,6 +117,9 @@ async def provider_available(
if url is None:
return False

if plugin == "custom":
return await self.call_custom_model(model, url, key_env)

exe_path = Path(os.environ["GENVM_BIN"]).joinpath("genvm-modules")

try:
Expand Down Expand Up @@ -149,3 +153,59 @@ async def provider_available(
f"ERROR: Wrong input provider_available {model=}, {url=}, {plugin=}, {key_env=}, {e=}"
)
return False

async def call_custom_model(self, model: str, url: str, key_env: str) -> bool:
"""
Call a custom model to check if it is available.
"""
try:
prompt = {
"model": model,
"messages": [
{
"role": "user",
"content": 'Respond with two letters "ok" (without quotes) and only this word, lowercase',
}
],
}

api_key = os.environ.get(key_env)
if not api_key:
print(f"ERROR: missing API key for {key_env}")
return False

async with aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=30)
) as session, session.post(
url, json=prompt, headers={"Authorization": f"Bearer {api_key}"}
) as response:
if response.status != 200:
print(
f"ERROR: Custom model check failed with status {response.status}"
)
return False

response_data = await response.json()
try:
result = response_data["choices"][0]["message"]["content"]
if isinstance(result, str) and result.strip().lower() == "ok":
return True
elif (
isinstance(result, dict)
and "result" in result
and result["result"].strip().lower() == "ok"
):
return True

print(
f"ERROR: Custom model check failed: got '{result}' instead of 'ok'"
)
return False

except (KeyError, IndexError, json.JSONDecodeError) as e:
print(f"ERROR: Invalid response format: {e}")
return False

except Exception as e:
print(f"ERROR: Custom model check failed with error: {e}")
return False
49 changes: 48 additions & 1 deletion frontend/src/assets/schemas/providers_schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
},
"plugin": {
"$comment": "Plugin to be loaded by the Studio to interact with the provider",
"enum": ["openai-compatible", "ollama", "anthropic", "google"]
"enum": ["openai-compatible", "ollama", "anthropic", "google", "custom"]
},
"plugin_config": {
"type": "object"
Expand Down Expand Up @@ -450,6 +450,53 @@
}
}
}
},
{
"if": {
"properties": {
"plugin": { "const": "custom" }
}
},
"then": {
"properties": {
"plugin_config": {
"type": "object",
"additionalProperties": false,
"properties": {
"api_key_env_var": {
"type": "string",
"$comment": "Environment variable that contains the API key",
"minLength": 1
},
"api_url": {
"type": ["string", "null"],
"$comment": "URL of the API endpoint"
}
},
"required": ["api_key_env_var", "api_url"]
},
"config": {
"type": "object",
"additionalProperties": false,
"properties": {
"temperature": {
"type": "number",
"minimum": 0,
"maximum": 1,
"multipleOf": 0.05,
"default": 0.75
},
"max_tokens": {
"type": "integer",
"minimum": 100,
"maximum": 2000,
"multipleOf": 10,
"default": 500
}
}
}
}
}
}
],
"required": ["provider", "model", "config", "plugin", "plugin_config"],
Expand Down
28 changes: 14 additions & 14 deletions frontend/src/components/Simulator/ValidatorModal.vue
Original file line number Diff line number Diff line change
Expand Up @@ -119,20 +119,20 @@ const providerOptions = computed(() => {
});

const modelOptions = computed(() => {
return uniqBy(nodeStore.nodeProviders, 'model')
.filter(
(provider: any) => provider.provider === newValidatorData.value.provider,
)
.map((provider: any) => {
const isDisabled = !provider?.is_model_available;
return {
value: provider.model,
label: isDisabled
? `${provider.model} (missing configuration)`
: provider.model,
disabled: !provider?.is_model_available,
};
});
const providers = nodeStore.nodeProviders.filter(
(provider: any) => provider.provider === newValidatorData.value.provider,
);

return uniqBy(providers, 'model').map((provider: any) => {
const isDisabled = !provider?.is_model_available;
return {
value: provider.model,
label: isDisabled
? `${provider.model} (missing configuration)`
: provider.model,
disabled: !provider?.is_model_available,
};
});
});

const handleChangeProvider = () => {
Expand Down
3 changes: 2 additions & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,5 @@ python-dotenv==1.1.0
black==25.1.0
pytest==8.3.5
pytest-mock==3.14.0
pre-commit==4.2.0
pre-commit==4.2.0
aiohttp==3.11.16