Skip to content

Commit a2f2b65

Browse files
sycaisycai
authored
chore: deprecate Claude 3.5 in the codebase (#2472)
Fixes b/486207478
1 parent 8a1a82f commit a2f2b65

File tree

3 files changed

+72
-132
lines changed

3 files changed

+72
-132
lines changed

‎bigframes/ml/llm.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -877,7 +877,7 @@ class Claude3TextGenerator(base.RetriableRemotePredictor):
877877
The model for natural language tasks. Possible values are "claude-3-sonnet", "claude-3-haiku", "claude-3-5-sonnet" and "claude-3-opus".
878878
"claude-3-sonnet" (deprecated) is Anthropic's dependable combination of skills and speed. It is engineered to be dependable for scaled AI deployments across a variety of use cases.
879879
"claude-3-haiku" is Anthropic's fastest, most compact vision and text model for near-instant responses to simple queries, meant for seamless AI experiences mimicking human interactions.
880-
"claude-3-5-sonnet" is Anthropic's most powerful AI model and maintains the speed and cost of Claude 3 Sonnet, which is a mid-tier model.
880+
"claude-3-5-sonnet" (deprecated) is Anthropic's most powerful AI model and maintains the speed and cost of Claude 3 Sonnet, which is a mid-tier model.
881881
"claude-3-opus" (deprecated) is Anthropic's second-most powerful AI model, with strong performance on highly complex tasks.
882882
https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude#available-claude-models
883883
If no setting is provided, "claude-3-sonnet" will be used by default

‎notebooks/remote_functions/remote_function_vertex_claude_model.ipynb‎

Lines changed: 62 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -28,20 +28,6 @@
2828
"</table>"
2929
]
3030
},
31-
{
32-
"cell_type": "code",
33-
"execution_count": null,
34-
"metadata": {},
35-
"outputs": [],
36-
"source": [
37-
"import sys\n",
38-
"\n",
39-
"# Python 3.13 is not yet a supported runtime for remote functions.\n",
40-
"# See: https://cloud.google.com/functions/docs/runtime-support#python for the supported runtimes.\n",
41-
"if sys.version_info >= (3, 13, 0):\n",
42-
" sys.exit(0)"
43-
]
44-
},
4531
{
4632
"cell_type": "markdown",
4733
"metadata": {},
@@ -155,30 +141,6 @@
155141
"execution_count": 4,
156142
"metadata": {},
157143
"outputs": [
158-
{
159-
"data": {
160-
"text/html": [
161-
"Query job c4c27713-51c8-4293-8454-5c904df79318 is DONE. 0 Bytes processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:c4c27713-51c8-4293-8454-5c904df79318&page=queryresults\">Open Job</a>"
162-
],
163-
"text/plain": [
164-
"<IPython.core.display.HTML object>"
165-
]
166-
},
167-
"metadata": {},
168-
"output_type": "display_data"
169-
},
170-
{
171-
"data": {
172-
"text/html": [
173-
"Query job 0b1b71d8-8546-45f2-b403-707161fe4002 is DONE. 0 Bytes processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:0b1b71d8-8546-45f2-b403-707161fe4002&page=queryresults\">Open Job</a>"
174-
],
175-
"text/plain": [
176-
"<IPython.core.display.HTML object>"
177-
]
178-
},
179-
"metadata": {},
180-
"output_type": "display_data"
181-
},
182144
{
183145
"data": {
184146
"text/html": [
@@ -267,13 +229,15 @@
267229
},
268230
{
269231
"cell_type": "code",
270-
"execution_count": 6,
232+
"execution_count": 5,
271233
"metadata": {},
272234
"outputs": [
273235
{
274236
"data": {
275237
"text/html": [
276-
"Query job 488a116f-44b2-4ff7-9f95-bd36473dab0f is DONE. 0 Bytes processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:488a116f-44b2-4ff7-9f95-bd36473dab0f&page=queryresults\">Open Job</a>"
238+
"\n",
239+
" Query processed 0 Bytes in a moment of slot time. [<a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:9bc70627-6891-44a4-b7d7-8a28e213cdec&page=queryresults\">Job bigframes-dev:us-east5.9bc70627-6891-44a4-b7d7-8a28e213cdec details</a>]\n",
240+
" "
277241
],
278242
"text/plain": [
279243
"<IPython.core.display.HTML object>"
@@ -301,24 +265,24 @@
301265
" \"content\": message,\n",
302266
" }\n",
303267
" ],\n",
304-
" model=\"claude-3-5-sonnet@20240620\",\n",
268+
" model=\"claude-3-haiku@20240307\",\n",
305269
" )\n",
306270
" content_text = message.content[0].text if message.content else \"\"\n",
307271
" return content_text"
308272
]
309273
},
310274
{
311275
"cell_type": "code",
312-
"execution_count": 7,
276+
"execution_count": 6,
313277
"metadata": {},
314278
"outputs": [
315279
{
316280
"data": {
317281
"text/plain": [
318-
"'bigframes-dev._b52b272a35b88e236e1f96fbe3f560c83a8fee85.bigframes_session265649_de1176dd4c57f40ba959503af3981682'"
282+
"'bigframes-dev._e9a5162ae4daa9f50fda3f95febaa9781131f3b8.bigframes_sessionc10c73_49262141176cbf70037559ae84e834d3'"
319283
]
320284
},
321-
"execution_count": 7,
285+
"execution_count": 6,
322286
"metadata": {},
323287
"output_type": "execute_result"
324288
}
@@ -330,16 +294,16 @@
330294
},
331295
{
332296
"cell_type": "code",
333-
"execution_count": 8,
297+
"execution_count": 7,
334298
"metadata": {},
335299
"outputs": [
336300
{
337301
"data": {
338302
"text/plain": [
339-
"'projects/bigframes-dev/locations/us-east5/functions/bigframes-session265649-de1176dd4c57f40ba959503af3981682'"
303+
"'projects/bigframes-dev/locations/us-east5/functions/bigframes-sessionc10c73-49262141176cbf70037559ae84e834d3'"
340304
]
341305
},
342-
"execution_count": 8,
306+
"execution_count": 7,
343307
"metadata": {},
344308
"output_type": "execute_result"
345309
}
@@ -351,49 +315,40 @@
351315
},
352316
{
353317
"cell_type": "code",
354-
"execution_count": 9,
318+
"execution_count": 8,
355319
"metadata": {},
356320
"outputs": [
357321
{
358322
"data": {
359323
"text/html": [
360-
"Query job 58b230a8-6536-4bac-ab02-dcf574692dd6 is DONE. 0 Bytes processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:58b230a8-6536-4bac-ab02-dcf574692dd6&page=queryresults\">Open Job</a>"
361-
],
362-
"text/plain": [
363-
"<IPython.core.display.HTML object>"
364-
]
365-
},
366-
"metadata": {},
367-
"output_type": "display_data"
368-
},
369-
{
370-
"data": {
371-
"text/html": [
372-
"Query job 46d6a1e9-426a-4615-8eb5-98d34d08ec07 is DONE. 1.3 kB processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:46d6a1e9-426a-4615-8eb5-98d34d08ec07&page=queryresults\">Open Job</a>"
373-
],
374-
"text/plain": [
375-
"<IPython.core.display.HTML object>"
376-
]
377-
},
378-
"metadata": {},
379-
"output_type": "display_data"
380-
},
381-
{
382-
"data": {
383-
"text/html": [
384-
"Query job cf8fcbaa-b233-47cd-b4e3-60876b24879f is DONE. 0 Bytes processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:cf8fcbaa-b233-47cd-b4e3-60876b24879f&page=queryresults\">Open Job</a>"
385-
],
386-
"text/plain": [
387-
"<IPython.core.display.HTML object>"
388-
]
389-
},
390-
"metadata": {},
391-
"output_type": "display_data"
392-
},
393-
{
394-
"data": {
395-
"text/html": [
396-
"Query job 2620a544-d1df-4b30-bec4-4221e79ddf0f is DONE. 1.4 kB processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:2620a544-d1df-4b30-bec4-4221e79ddf0f&page=queryresults\">Open Job</a>"
324+
"\n",
325+
" Query started with request ID bigframes-dev:us-east5.821579f4-63ea-4072-a3ce-318e43768432.<details><summary>SQL</summary><pre>SELECT\n",
326+
"`bfuid_col_3` AS `bfuid_col_3`,\n",
327+
"`bfuid_col_4` AS `bfuid_col_4`,\n",
328+
"`bfuid_col_5` AS `bfuid_col_5`\n",
329+
"FROM\n",
330+
"(SELECT\n",
331+
" `t1`.`bfuid_col_3`,\n",
332+
" `t1`.`bfuid_col_4`,\n",
333+
" `t1`.`bfuid_col_5`,\n",
334+
" `t1`.`bfuid_col_6` AS `bfuid_col_7`\n",
335+
"FROM (\n",
336+
" SELECT\n",
337+
" `t0`.`level_0`,\n",
338+
" `t0`.`column_0`,\n",
339+
" `t0`.`bfuid_col_6`,\n",
340+
" `t0`.`level_0` AS `bfuid_col_3`,\n",
341+
" `t0`.`column_0` AS `bfuid_col_4`,\n",
342+
" `bigframes-dev._e9a5162ae4daa9f50fda3f95febaa9781131f3b8.bigframes_sessionc10c73_49262141176cbf70037559ae84e834d3`(`t0`.`column_0`) AS `bfuid_col_5`\n",
343+
" FROM (\n",
344+
" SELECT\n",
345+
" *\n",
346+
" FROM UNNEST(ARRAY&lt;STRUCT&lt;`level_0` INT64, `column_0` STRING, `bfuid_col_6` INT64&gt;&gt;[STRUCT(0, &#x27;What is the capital of France?&#x27;, 0), STRUCT(1, &#x27;Explain the concept of photosynthesis in simple terms.&#x27;, 1), STRUCT(2, &#x27;Write a haiku about artificial intelligence.&#x27;, 2)]) AS `level_0`\n",
347+
" ) AS `t0`\n",
348+
") AS `t1`)\n",
349+
"ORDER BY `bfuid_col_7` ASC NULLS LAST\n",
350+
"LIMIT 10</pre></details>\n",
351+
" "
397352
],
398353
"text/plain": [
399354
"<IPython.core.display.HTML object>"
@@ -436,13 +391,12 @@
436391
" <tr>\n",
437392
" <th>1</th>\n",
438393
" <td>Explain the concept of photosynthesis in simpl...</td>\n",
439-
" <td>Photosynthesis is the process plants use to ma...</td>\n",
394+
" <td>Photosynthesis is the process by which plants ...</td>\n",
440395
" </tr>\n",
441396
" <tr>\n",
442397
" <th>2</th>\n",
443398
" <td>Write a haiku about artificial intelligence.</td>\n",
444-
" <td>Here's a haiku about artificial intelligence:\n",
445-
"...</td>\n",
399+
" <td>Here is a haiku about artificial intelligence:...</td>\n",
446400
" </tr>\n",
447401
" </tbody>\n",
448402
"</table>\n",
@@ -457,14 +411,13 @@
457411
"\n",
458412
" answers \n",
459413
"0 The capital of France is Paris. \n",
460-
"1 Photosynthesis is the process plants use to ma... \n",
461-
"2 Here's a haiku about artificial intelligence:\n",
462-
"... \n",
414+
"1 Photosynthesis is the process by which plants ... \n",
415+
"2 Here is a haiku about artificial intelligence:... \n",
463416
"\n",
464417
"[3 rows x 2 columns]"
465418
]
466419
},
467-
"execution_count": 9,
420+
"execution_count": 8,
468421
"metadata": {},
469422
"output_type": "execute_result"
470423
}
@@ -484,17 +437,30 @@
484437
},
485438
{
486439
"cell_type": "code",
487-
"execution_count": null,
440+
"execution_count": 9,
488441
"metadata": {},
489-
"outputs": [],
442+
"outputs": [
443+
{
444+
"data": {
445+
"text/html": [
446+
"Session sessionc10c73 closed."
447+
],
448+
"text/plain": [
449+
"<IPython.core.display.HTML object>"
450+
]
451+
},
452+
"metadata": {},
453+
"output_type": "display_data"
454+
}
455+
],
490456
"source": [
491457
"bpd.close_session()"
492458
]
493459
}
494460
],
495461
"metadata": {
496462
"kernelspec": {
497-
"display_name": "venv",
463+
"display_name": "venv (3.14.2)",
498464
"language": "python",
499465
"name": "python3"
500466
},
@@ -508,7 +474,7 @@
508474
"name": "python",
509475
"nbconvert_exporter": "python",
510476
"pygments_lexer": "ipython3",
511-
"version": "3.10.9"
477+
"version": "3.14.2"
512478
}
513479
},
514480
"nbformat": 4,

‎tests/system/load/test_llm.py‎

Lines changed: 9 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -98,18 +98,10 @@ def test_llm_gemini_w_ground_with_google_search(llm_remote_text_df):
9898

9999

100100
# (b/366290533): Claude models are of extremely low capacity. The tests should reside in small tests. Moving these here just to protect BQML's shared capacity(as load test only runs once per day.) and make sure we still have minimum coverage.
101-
@pytest.mark.parametrize(
102-
"model_name",
103-
("claude-3-haiku", "claude-3-5-sonnet"),
104-
)
105101
@pytest.mark.flaky(retries=3, delay=120)
106-
def test_claude3_text_generator_create_load(
107-
dataset_id, model_name, session, session_us_east5, bq_connection
108-
):
109-
if model_name in ("claude-3-5-sonnet",):
110-
session = session_us_east5
102+
def test_claude3_text_generator_create_load(dataset_id, session, bq_connection):
111103
claude3_text_generator_model = llm.Claude3TextGenerator(
112-
model_name=model_name, connection_name=bq_connection, session=session
104+
model_name="claude-3-haiku", connection_name=bq_connection, session=session
113105
)
114106
assert claude3_text_generator_model is not None
115107
assert claude3_text_generator_model._bqml_model is not None
@@ -120,40 +112,28 @@ def test_claude3_text_generator_create_load(
120112
)
121113
assert f"{dataset_id}.temp_text_model" == reloaded_model._bqml_model.model_name
122114
assert reloaded_model.connection_name == bq_connection
123-
assert reloaded_model.model_name == model_name
115+
assert reloaded_model.model_name == "claude-3-haiku"
124116

125117

126-
@pytest.mark.parametrize(
127-
"model_name",
128-
("claude-3-haiku", "claude-3-5-sonnet"),
129-
)
130118
@pytest.mark.flaky(retries=3, delay=120)
131119
def test_claude3_text_generator_predict_default_params_success(
132-
llm_text_df, model_name, session, session_us_east5, bq_connection
120+
llm_text_df, session, bq_connection
133121
):
134-
if model_name in ("claude-3-5-sonnet",):
135-
session = session_us_east5
136122
claude3_text_generator_model = llm.Claude3TextGenerator(
137-
model_name=model_name, connection_name=bq_connection, session=session
123+
model_name="claude-3-haiku", connection_name=bq_connection, session=session
138124
)
139125
df = claude3_text_generator_model.predict(llm_text_df).to_pandas()
140126
utils.check_pandas_df_schema_and_index(
141127
df, columns=utils.ML_GENERATE_TEXT_OUTPUT, index=3, col_exact=False
142128
)
143129

144130

145-
@pytest.mark.parametrize(
146-
"model_name",
147-
("claude-3-haiku", "claude-3-5-sonnet"),
148-
)
149131
@pytest.mark.flaky(retries=3, delay=120)
150132
def test_claude3_text_generator_predict_with_params_success(
151-
llm_text_df, model_name, session, session_us_east5, bq_connection
133+
llm_text_df, session, bq_connection
152134
):
153-
if model_name in ("claude-3-5-sonnet",):
154-
session = session_us_east5
155135
claude3_text_generator_model = llm.Claude3TextGenerator(
156-
model_name=model_name, connection_name=bq_connection, session=session
136+
model_name="claude-3-haiku", connection_name=bq_connection, session=session
157137
)
158138
df = claude3_text_generator_model.predict(
159139
llm_text_df, max_output_tokens=100, top_k=20, top_p=0.5
@@ -163,20 +143,14 @@ def test_claude3_text_generator_predict_with_params_success(
163143
)
164144

165145

166-
@pytest.mark.parametrize(
167-
"model_name",
168-
("claude-3-haiku", "claude-3-5-sonnet"),
169-
)
170146
@pytest.mark.flaky(retries=3, delay=120)
171147
def test_claude3_text_generator_predict_multi_col_success(
172-
llm_text_df, model_name, session, session_us_east5, bq_connection
148+
llm_text_df, session, bq_connection
173149
):
174-
if model_name in ("claude-3-5-sonnet",):
175-
session = session_us_east5
176150

177151
llm_text_df["additional_col"] = 1
178152
claude3_text_generator_model = llm.Claude3TextGenerator(
179-
model_name=model_name, connection_name=bq_connection, session=session
153+
model_name="claude-3-haiku", connection_name=bq_connection, session=session
180154
)
181155
df = claude3_text_generator_model.predict(llm_text_df).to_pandas()
182156
utils.check_pandas_df_schema_and_index(

0 commit comments

Comments
 (0)