Skip to content

Commit dd7db8b

Browse files
alexrudallalexrudall
authored
Merge pull request alexrudall#466 from alexrudall/groq
Add Groq
2 parents b060d86 + 69ba82e commit dd7db8b

File tree

6 files changed

+189
-20
lines changed

6 files changed

+189
-20
lines changed

‎README.md

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ Stream text with GPT-4, transcribe and translate audio with Whisper, or create i
2727
- [Faraday middleware](#faraday-middleware)
2828
- [Azure](#azure)
2929
- [Ollama](#ollama)
30+
- [Groq](#groq)
3031
- [Counting Tokens](#counting-tokens)
3132
- [Models](#models)
3233
- [Chat](#chat)
@@ -239,6 +240,27 @@ client.chat(
239240
# => Hi! It's nice to meet you. Is there something I can help you with, or would you like to chat?
240241
```
241242

243+
#### Groq
244+
245+
[Groq API Chat](https://console.groq.com/docs/quickstart) is broadly compatible with the OpenAI API, with a [few minor differences](https://console.groq.com/docs/openai). Get an access token from [here](https://console.groq.com/keys), then:
246+
247+
````ruby
248+
client = OpenAI::Client.new(
249+
access_token: "groq_access_token_goes_here",
250+
uri_base: "https://api.groq.com/"
251+
)
252+
253+
client.chat(
254+
parameters: {
255+
model: "llama3", # Required.
256+
messages: [{ role: "user", content: "Hello!"}], # Required.
257+
temperature: 0.7,
258+
stream: proc do |chunk, _bytesize|
259+
print chunk.dig("choices", 0, "delta", "content")
260+
end
261+
})
262+
```
263+
242264
### Counting Tokens
243265

244266
OpenAI parses prompt text into [tokens](https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them), which are words or portions of words. (These tokens are unrelated to your API access_token.) Counting tokens can help you estimate your [costs](https://openai.com/pricing). It can also help you ensure your prompt text size is within the max-token limits of your model's context window, and choose an appropriate [`max_tokens`](https://platform.openai.com/docs/api-reference/chat/create#chat/create-max_tokens) completion parameter so your response will fit as well.
@@ -247,7 +269,7 @@ To estimate the token-count of your text:
247269
248270
```ruby
249271
OpenAI.rough_token_count("Your text")
250-
```
272+
````
251273
252274
If you need a more accurate count, try [tiktoken_ruby](https://github.com/IAPark/tiktoken_ruby).
253275

‎spec/fixtures/cassettes/groq_llama3-8b-8192_streamed_chat.yml

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
---
2+
http_interactions:
3+
- request:
4+
method: post
5+
uri: https://api.groq.com/openai/v1/chat/completions
6+
body:
7+
encoding: UTF-8
8+
string: '{"model":"llama3-8b-8192","messages":[{"role":"user","content":"Hello!"}],"stream":true}'
9+
headers:
10+
Content-Type:
11+
- application/json
12+
Authorization:
13+
- Bearer <OPENAI_ACCESS_TOKEN>
14+
Accept-Encoding:
15+
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
16+
Accept:
17+
- "*/*"
18+
User-Agent:
19+
- Ruby
20+
response:
21+
status:
22+
code: 200
23+
message: OK
24+
headers:
25+
Date:
26+
- Sun, 28 Apr 2024 14:14:21 GMT
27+
Content-Type:
28+
- text/event-stream
29+
Transfer-Encoding:
30+
- chunked
31+
Connection:
32+
- keep-alive
33+
Cache-Control:
34+
- no-cache
35+
Vary:
36+
- Origin
37+
X-Ratelimit-Limit-Requests:
38+
- '14400'
39+
X-Ratelimit-Limit-Tokens:
40+
- '12000'
41+
X-Ratelimit-Remaining-Requests:
42+
- '14399'
43+
X-Ratelimit-Remaining-Tokens:
44+
- '11994'
45+
X-Ratelimit-Reset-Requests:
46+
- 6s
47+
X-Ratelimit-Reset-Tokens:
48+
- 30ms
49+
X-Request-Id:
50+
- req_01hwjgx2mmfwvv1cjs7mbnnzx9
51+
Via:
52+
- 1.1 google
53+
Alt-Svc:
54+
- h3=":443"; ma=86400
55+
Cf-Cache-Status:
56+
- DYNAMIC
57+
Set-Cookie:
58+
- __cf_bm=QZ_zCo.sZtGo4pDeVUp1lu3BJ7JIp80ODZNCgOwO3Xg-1714313661-1.0.1.1-NBJfGde2UFdXGgwSBnU15KWpgRDPsJ0yiN0Mqc3GqOFjrvn_eYFjgV8YAm56.QYYiZmQsstvnUdxJ3LaGaUlVg;
59+
path=/; expires=Sun, 28-Apr-24 14:44:21 GMT; domain=.groq.com; HttpOnly; Secure;
60+
SameSite=None
61+
Server:
62+
- cloudflare
63+
Cf-Ray:
64+
- 87b7a87d3f4edcbb-LHR
65+
body:
66+
encoding: UTF-8
67+
string: |+
68+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"x_groq":{"id":"req_01hwjgx2mmfwvv1cjs7mbnnzx9"}}
69+
70+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}]}
71+
72+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}]}
73+
74+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" It"},"logprobs":null,"finish_reason":null}]}
75+
76+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":"'s"},"logprobs":null,"finish_reason":null}]}
77+
78+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" nice"},"logprobs":null,"finish_reason":null}]}
79+
80+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" to"},"logprobs":null,"finish_reason":null}]}
81+
82+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" meet"},"logprobs":null,"finish_reason":null}]}
83+
84+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}]}
85+
86+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}]}
87+
88+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" Is"},"logprobs":null,"finish_reason":null}]}
89+
90+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" there"},"logprobs":null,"finish_reason":null}]}
91+
92+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" something"},"logprobs":null,"finish_reason":null}]}
93+
94+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}]}
95+
96+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" can"},"logprobs":null,"finish_reason":null}]}
97+
98+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" help"},"logprobs":null,"finish_reason":null}]}
99+
100+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}]}
101+
102+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" with"},"logprobs":null,"finish_reason":null}]}
103+
104+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" or"},"logprobs":null,"finish_reason":null}]}
105+
106+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" would"},"logprobs":null,"finish_reason":null}]}
107+
108+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}]}
109+
110+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" like"},"logprobs":null,"finish_reason":null}]}
111+
112+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" to"},"logprobs":null,"finish_reason":null}]}
113+
114+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":" chat"},"logprobs":null,"finish_reason":null}]}
115+
116+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}]}
117+
118+
data: {"id":"chatcmpl-1b853bd2-b065-41bf-a336-fd07da175604","object":"chat.completion.chunk","created":1714313661,"model":"llama3-8b-8192","system_fingerprint":"fp_873a560973","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"x_groq":{"id":"req_01hwjgx2mmfwvv1cjs7mbnnzx9","usage":{"queue_time":0.090000872,"prompt_tokens":12,"prompt_time":0.005,"completion_tokens":24,"completion_time":0.029,"total_tokens":36,"total_time":0.034}}}
119+
120+
data: [DONE]
121+
122+
recorded_at: Sun, 28 Apr 2024 14:14:21 GMT
123+
recorded_with: VCR 6.1.0
124+
...

‎spec/fixtures/cassettes/llama3_chat.yml renamed to ‎spec/fixtures/cassettes/ollama_llama3_chat.yml

Lines changed: 7 additions & 7 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎spec/openai/client/chat_spec.rb

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,10 @@
1111
end
1212
let(:parameters) { { model: model, messages: messages, stream: stream } }
1313
let(:content) { response.dig("choices", 0, "message", "content") }
14-
let(:cassette) { "#{model} #{'streamed' if stream} chat".downcase }
14+
let(:provider) { nil }
15+
let(:cassette) do
16+
"#{"#{provider}_" if provider}#{model} #{'streamed' if stream} chat".downcase
17+
end
1518

1619
context "with model: gpt-3.5-turbo" do
1720
let(:model) { "gpt-3.5-turbo" }
@@ -176,11 +179,12 @@ def call(chunk)
176179

177180
context "with Ollama + model: llama3" do
178181
let(:uri_base) { "http://localhost:11434" }
182+
let(:provider) { "ollama" }
179183
let(:model) { "llama3" }
180184

181185
it "succeeds" do
182186
VCR.use_cassette(cassette) do
183-
vcr_skip do
187+
tap do
184188
Faraday.new(url: uri_base).get
185189
rescue Faraday::ConnectionFailed
186190
pending "This test needs `ollama serve` running locally with #{model} installed"
@@ -190,6 +194,35 @@ def call(chunk)
190194
end
191195
end
192196
end
197+
198+
context "with Groq + model: llama3" do
199+
let(:uri_base) { "https://api.groq.com/openai" }
200+
let(:provider) { "groq" }
201+
let(:model) { "llama3-8b-8192" }
202+
let(:response) do
203+
OpenAI::Client.new({ uri_base: uri_base }).chat(
204+
parameters: parameters
205+
)
206+
end
207+
let(:chunks) { [] }
208+
let(:stream) do
209+
proc do |chunk, _bytesize|
210+
chunks << chunk
211+
end
212+
end
213+
214+
it "succeeds" do
215+
VCR.use_cassette(cassette) do
216+
tap do
217+
response
218+
rescue Faraday::UnauthorizedError
219+
pending "This test needs the `OPENAI_ACCESS_TOKEN` to be a Groq API key"
220+
end
221+
222+
expect(chunks.dig(0, "choices", 0, "index")).to eq(0)
223+
end
224+
end
225+
end
193226
end
194227
end
195228
end

‎spec/spec_helper.rb

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,6 @@
4949
end
5050
end
5151
end
52-
53-
c.include VCRHelpers
5452
end
5553

5654
RSPEC_ROOT = File.dirname __FILE__

‎spec/support/vcr_skip.rb

Lines changed: 0 additions & 8 deletions
This file was deleted.

0 commit comments

Comments
 (0)