Skip to content

Feat/add litellm provider #268

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
RheagalFire wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Feat/add litellm provider #268

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
50c10b0
feat: add LiteLLM as AI gateway provider
RheagalFire Jul 16, 2026
6c10e4d
test: add comprehensive edge case tests for LiteLLM provider
RheagalFire Jul 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ const (

// sensitiveConfigKeys lists the config keys that should be stored in the
// secure credential store rather than in the plaintext YAML config file.
var sensitiveConfigKeys = []string{"openai.api_key", "gemini.api_key"}
var sensitiveConfigKeys = []string{"openai.api_key", "gemini.api_key", "litellm.api_key"}

// migrateCredentialsToStore moves any plaintext API keys found in the YAML
// config into the secure credential store and clears them from the config file.
Expand Down
66 changes: 35 additions & 31 deletions cmd/config_list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,37 +18,41 @@ func init() {

// availableKeys is a map of configuration keys and their descriptions
var availableKeys = map[string]string{
"git.diff_unified": "Number of context lines in git diff output (default: 3)",
"git.exclude_list": "Files to exclude from git diff command",
"git.template_file": "Path to template file for commit messages",
"git.template_string": "Template string for formatting commit messages",
"openai.socks": "SOCKS proxy URL for API connections",
"openai.api_key": "Authentication key for OpenAI API access",
"openai.api_key_helper": "Shell command to dynamically generate API key",
"openai.api_key_helper_refresh_interval": "Interval in seconds to refresh credentials from apiKeyHelper (default: 900)",
"openai.model": "AI model identifier to use for requests",
"openai.org_id": "Organization ID for multi-org OpenAI accounts",
"openai.proxy": "HTTP proxy URL for API connections",
"output.lang": "Language for summarization output (default: English)",
"openai.base_url": "Custom base URL for API requests",
"openai.timeout": "Maximum duration to wait for API response",
"openai.max_tokens": "Maximum token limit for generated completions",
"openai.temperature": "Randomness control parameter (0-1): lower values for focused results, higher for creative variety",
"openai.provider": "Service provider selection ('openai' or 'azure')",
"openai.skip_verify": "Option to bypass TLS certificate verification",
"openai.headers": "Additional custom HTTP headers for API requests",
"openai.api_version": "Specific API version to target",
"openai.top_p": "Nucleus sampling parameter: controls diversity by limiting to top percentage of probability mass",
"openai.frequency_penalty": "Parameter to reduce repetition by penalizing tokens based on their frequency",
"openai.presence_penalty": "Parameter to encourage topic diversity by penalizing previously used tokens",
"openai.stream": "Enable streaming output for real-time token display",
"prompt.folder": "Directory path for custom prompt templates",
"gemini.project_id": "VertexAI project for Gemini provider",
"gemini.location": "VertexAI location for Gemini provider",
"gemini.backend": "Gemini backend (BackendGeminiAPI or BackendVertexAI)",
"gemini.api_key": "API key for Gemini provider",
"gemini.api_key_helper": "Shell command to dynamically generate Gemini API key",
"gemini.api_key_helper_refresh_interval": "Interval in seconds to refresh Gemini credentials from apiKeyHelper (default: 900)",
"git.diff_unified": "Number of context lines in git diff output (default: 3)",
"git.exclude_list": "Files to exclude from git diff command",
"git.template_file": "Path to template file for commit messages",
"git.template_string": "Template string for formatting commit messages",
"openai.socks": "SOCKS proxy URL for API connections",
"openai.api_key": "Authentication key for OpenAI API access",
"openai.api_key_helper": "Shell command to dynamically generate API key",
"openai.api_key_helper_refresh_interval": "Interval in seconds to refresh credentials from apiKeyHelper (default: 900)",
"openai.model": "AI model identifier to use for requests",
"openai.org_id": "Organization ID for multi-org OpenAI accounts",
"openai.proxy": "HTTP proxy URL for API connections",
"output.lang": "Language for summarization output (default: English)",
"openai.base_url": "Custom base URL for API requests",
"openai.timeout": "Maximum duration to wait for API response",
"openai.max_tokens": "Maximum token limit for generated completions",
"openai.temperature": "Randomness control parameter (0-1): lower values for focused results, higher for creative variety",
"openai.provider": "Service provider selection ('openai' or 'azure')",
"openai.skip_verify": "Option to bypass TLS certificate verification",
"openai.headers": "Additional custom HTTP headers for API requests",
"openai.api_version": "Specific API version to target",
"openai.top_p": "Nucleus sampling parameter: controls diversity by limiting to top percentage of probability mass",
"openai.frequency_penalty": "Parameter to reduce repetition by penalizing tokens based on their frequency",
"openai.presence_penalty": "Parameter to encourage topic diversity by penalizing previously used tokens",
"openai.stream": "Enable streaming output for real-time token display",
"prompt.folder": "Directory path for custom prompt templates",
"gemini.project_id": "VertexAI project for Gemini provider",
"gemini.location": "VertexAI location for Gemini provider",
"gemini.backend": "Gemini backend (BackendGeminiAPI or BackendVertexAI)",
"gemini.api_key": "API key for Gemini provider",
"gemini.api_key_helper": "Shell command to dynamically generate Gemini API key",
"gemini.api_key_helper_refresh_interval": "Interval in seconds to refresh Gemini credentials from apiKeyHelper (default: 900)",
"litellm.api_key": "API key for LiteLLM proxy (master key or virtual key)",
"litellm.api_key_helper": "Shell command to dynamically generate LiteLLM API key",
"litellm.api_key_helper_refresh_interval": "Interval in seconds to refresh LiteLLM credentials from apiKeyHelper (default: 900)",
"litellm.base_url": "LiteLLM proxy base URL (default: http://localhost:4000/v1)",
}

// configListCmd represents the command to list the configuration values.
Expand Down
18 changes: 18 additions & 0 deletions cmd/config_set.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,14 @@ func init() {
configSetCmd.Flags().String("gemini.api_key_helper", "", availableKeys["gemini.api_key_helper"])
configSetCmd.Flags().
Int("gemini.api_key_helper_refresh_interval", 900, availableKeys["gemini.api_key_helper_refresh_interval"])
// LiteLLM flags
configSetCmd.Flags().String("litellm.api_key", "", availableKeys["litellm.api_key"])
configSetCmd.Flags().
String("litellm.api_key_helper", "", availableKeys["litellm.api_key_helper"])
configSetCmd.Flags().
Int("litellm.api_key_helper_refresh_interval", 900, availableKeys["litellm.api_key_helper_refresh_interval"])
configSetCmd.Flags().
String("litellm.base_url", "http://localhost:4000/v1", availableKeys["litellm.base_url"])

_ = viper.BindPFlag("openai.base_url", configSetCmd.Flags().Lookup("base_url"))
_ = viper.BindPFlag("openai.org_id", configSetCmd.Flags().Lookup("org_id"))
Expand Down Expand Up @@ -87,6 +95,16 @@ func init() {
"gemini.api_key_helper_refresh_interval",
configSetCmd.Flags().Lookup("gemini.api_key_helper_refresh_interval"),
)
_ = viper.BindPFlag("litellm.api_key", configSetCmd.Flags().Lookup("litellm.api_key"))
_ = viper.BindPFlag(
"litellm.api_key_helper",
configSetCmd.Flags().Lookup("litellm.api_key_helper"),
)
_ = viper.BindPFlag(
"litellm.api_key_helper_refresh_interval",
configSetCmd.Flags().Lookup("litellm.api_key_helper_refresh_interval"),
)
_ = viper.BindPFlag("litellm.base_url", configSetCmd.Flags().Lookup("litellm.base_url"))
}

// configSetCmd updates the config value.
Expand Down
58 changes: 58 additions & 0 deletions cmd/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/appleboy/CodeGPT/core"
"github.com/appleboy/CodeGPT/provider/anthropic"
"github.com/appleboy/CodeGPT/provider/gemini"
"github.com/appleboy/CodeGPT/provider/litellm"
"github.com/appleboy/CodeGPT/provider/openai"
"github.com/appleboy/CodeGPT/util"

Expand Down Expand Up @@ -205,6 +206,63 @@ func GetClient(ctx context.Context, p core.Platform) (core.Generative, error) {
return NewOpenAI(ctx)
case core.Anthropic:
return NewAnthropic(ctx)
case core.LiteLLM:
return NewLiteLLM(ctx)
}
return nil, errors.New("invalid provider")
}

// NewLiteLLM creates a new LiteLLM client that connects to a LiteLLM proxy server,
// providing access to 100+ LLM providers through a unified OpenAI-compatible API.
func NewLiteLLM(ctx context.Context) (*litellm.Client, error) {
_ = ctx

var apiKey string

// Try litellm-specific key helper first
if helper := viper.GetString("litellm.api_key_helper"); helper != "" {
var refreshInterval time.Duration
if viper.IsSet("litellm.api_key_helper_refresh_interval") {
refreshInterval = time.Duration(
viper.GetInt("litellm.api_key_helper_refresh_interval"),
) * time.Second
} else {
refreshInterval = util.DefaultRefreshInterval
}
key, err := util.GetAPIKeyFromHelperWithCache(ctx, helper, refreshInterval)
if err != nil {
return nil, err
}
apiKey = key
} else {
// Try litellm.api_key first, fall back to openai.api_key
key, err := getAPIKey("litellm.api_key")
if err != nil {
return nil, err
}
apiKey = key
if apiKey == "" {
openaiKey, err := getAPIKey("openai.api_key")
if err != nil {
return nil, err
}
apiKey = openaiKey
}
}

return litellm.New(
litellm.WithToken(apiKey),
litellm.WithModel(viper.GetString("openai.model")),
litellm.WithBaseURL(viper.GetString("litellm.base_url")),
litellm.WithProxyURL(viper.GetString("openai.proxy")),
litellm.WithSocksURL(viper.GetString("openai.socks")),
litellm.WithTimeout(viper.GetDuration("openai.timeout")),
litellm.WithMaxTokens(viper.GetInt("openai.max_tokens")),
litellm.WithTemperature(float32(viper.GetFloat64("openai.temperature"))),
litellm.WithTopP(float32(viper.GetFloat64("openai.top_p"))),
litellm.WithFrequencyPenalty(float32(viper.GetFloat64("openai.frequency_penalty"))),
litellm.WithPresencePenalty(float32(viper.GetFloat64("openai.presence_penalty"))),
litellm.WithSkipVerify(viper.GetBool("openai.skip_verify")),
litellm.WithHeaders(viper.GetStringSlice("openai.headers")),
)
}
4 changes: 3 additions & 1 deletion core/platform.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ const (
Gemini Platform = "gemini"
// Anthropic represents the Anthropic platform.
Anthropic Platform = "anthropic"
// LiteLLM represents the LiteLLM AI gateway platform.
LiteLLM Platform = "litellm"
)

// String returns the string representation of the Platform.
Expand All @@ -22,7 +24,7 @@ func (p Platform) String() string {
// IsValid returns true if the Platform is valid.
func (p Platform) IsValid() bool {
switch p {
case OpenAI, Azure, Gemini, Anthropic:
case OpenAI, Azure, Gemini, Anthropic, LiteLLM:
return true
}
return false
Expand Down
Loading