Even if calling calling an AIP Logic from a custom widget (via a function) gives more control over the logic ran in the backend, is it possible to call an LLM directly from a custom widget ? If yes, how ?
The idea is to call a python function that calls an LLM, and to call this function from Custom Widget. The result can be streamed to the custom widget.
Here is a code snippet for the function
from anthropic import Anthropic
from openai import OpenAI
from typing import Iterable
from functions.api import function
from functions.aliases import model
from foundry_sdk.v2.language_models.utils import (
get_anthropic_base_url,
get_foundry_token,
get_http_client,
get_openai_base_url,
)
@function
def get_chat_completion(prompt: str, model_key: str ="gpt56Sol") -> str:
client = OpenAI(
api_key=get_foundry_token(preview=True),
base_url=get_openai_base_url(preview=True),
http_client=get_http_client(preview=True),
)
completion = client.chat.completions.create(
model=model(model_key).rid,
messages=[
{
"role": "user",
"content": prompt,
},
],
)
return str(completion.choices[0].message.content)
@function
def create_chat_completion_as_stream(prompt: str, model_key: str ="gpt56Sol") -> Iterable[str]:
client = OpenAI(
api_key=get_foundry_token(preview=True),
base_url=get_openai_base_url(preview=True),
http_client=get_http_client(preview=True),
)
stream = client.chat.completions.create(
model=model(model_key).rid,
messages=[{"role": "user", "content": prompt}],
stream=True,
)
for event in stream:
if event.choices:
content = event.choices[0].delta.content
if content:
yield content
Don’t forget to import the models you want to be used in the left sidebar in code repository/VSCode.
The widget itself can then call those functions, and consume the stream as part of the UI displayed to end users.
import {
Button,
Callout,
Card,
Classes,
FormGroup,
Icon,
InputGroup,
Intent,
Spinner,
Tag,
TextArea,
} from "@blueprintjs/core";
import {
$ontologyRid,
createChatCompletionAsStream,
} from "@custom-widget/sdk";
import { Queries } from "@osdk/foundry.functions";
import {
type FoundryWidgetClientContext,
useFoundryWidgetContext,
} from "@osdk/widget.client-react";
import clsx from "clsx";
import React, { useCallback, useRef, useState } from "react";
import { client } from "./client.js";
import type MainConfig from "./main.config.js";
import { useDarkTheme } from "./useDarkTheme.js";
import css from "./Widget.module.css";
const DEFAULT_PROMPT =
"Explain in two short paragraphs how token streaming improves the user experience.";
const DEFAULT_MODEL_KEY = "gpt56Sol";
const useWidgetContext: () => FoundryWidgetClientContext<typeof MainConfig> =
useFoundryWidgetContext.withTypes<typeof MainConfig>();
const extractTextChunks = (value: unknown): string[] => {
if (typeof value === "string") return [value];
if (!Array.isArray(value)) return [];
return value.filter((item): item is string => typeof item === "string");
};
export const Widget: React.FC = () => {
const { parameters } = useWidgetContext();
const [prompt, setPrompt] = useState(
parameters.values.initialPrompt ?? DEFAULT_PROMPT,
);
const [modelKey, setModelKey] = useState(
parameters.values.modelKey ?? DEFAULT_MODEL_KEY,
);
const [chunks, setChunks] = useState<string[]>([]);
const [isStreaming, setIsStreaming] = useState(false);
const [streamError, setStreamError] = useState<string | null>(null);
const runIdRef = useRef(0);
const isDarkTheme = useDarkTheme();
const handleStream = useCallback(async () => {
const normalizedPrompt = prompt.trim();
const normalizedModelKey = modelKey.trim();
if (normalizedPrompt.length === 0) {
setStreamError("Enter a prompt before starting the stream.");
return;
}
const runId = runIdRef.current + 1;
runIdRef.current = runId;
setChunks([]);
setStreamError(null);
setIsStreaming(true);
try {
const stream = await Queries.streamingExecute(
client,
createChatCompletionAsStream.apiName,
{
ontology: $ontologyRid,
parameters: {
prompt: normalizedPrompt,
model_key: normalizedModelKey || undefined,
},
},
{ preview: true },
);
for await (const message of stream) {
if (runId !== runIdRef.current) break;
if (message.type === "error") {
throw new Error(
message.errorDescription ??
message.errorName ??
"The streaming function returned an unknown error.",
);
}
const nextChunks = extractTextChunks(message.value);
if (nextChunks.length > 0 && runId === runIdRef.current) {
setChunks((current) => [...current, ...nextChunks]);
}
}
} catch (error: unknown) {
if (runId === runIdRef.current) {
console.error("LLM streaming error:", error);
setStreamError(
error instanceof Error
? error.message
: "Streaming failed. Please try again.",
);
}
} finally {
if (runId === runIdRef.current) setIsStreaming(false);
}
}, [modelKey, prompt]);
const handleCancel = useCallback(() => {
runIdRef.current += 1;
setIsStreaming(false);
}, []);
const handleClear = useCallback(() => {
setChunks([]);
setStreamError(null);
}, []);
const response = chunks.join("");
return (
<main className={clsx(css.container, isDarkTheme && Classes.DARK)}>
<section className={css.content} aria-labelledby="widget-title">
<header className={css.header}>
<div className={css.titleBlock}>
<div className={css.iconBadge} aria-hidden="true">
<Icon icon="chat" size={20} />
</div>
<div>
<h2 id="widget-title" className={css.title}>
LLM streaming tester
</h2>
<p className={css.subtitle}>
Run the Python query function and inspect tokens as they arrive.
</p>
</div>
</div>
<Tag minimal={true} intent={Intent.PRIMARY}>
createChatCompletionAsStream
</Tag>
</header>
<Card className={css.controls} compact={true}>
<form
onSubmit={(event) => {
event.preventDefault();
void handleStream();
}}
>
<FormGroup
label="Model alias"
labelFor="model-key"
helperText="Leave blank to use the function default (gpt56Sol)."
>
<InputGroup
id="model-key"
value={modelKey}
onChange={(event) => setModelKey(event.currentTarget.value)}
placeholder={DEFAULT_MODEL_KEY}
disabled={isStreaming}
fill={true}
/>
</FormGroup>
<FormGroup label="Prompt" labelFor="prompt">
<TextArea
id="prompt"
value={prompt}
onChange={(event) => setPrompt(event.currentTarget.value)}
placeholder="Ask the model something…"
disabled={isStreaming}
fill={true}
className={css.promptInput}
rows={5}
/>
</FormGroup>
<div className={css.actions}>
{isStreaming ? (
<Button
type="button"
icon="stop"
intent={Intent.DANGER}
onClick={handleCancel}
>
Cancel
</Button>
) : (
<Button type="submit" icon="play" intent={Intent.PRIMARY}>
{response.length > 0 ? "Stream again" : "Start streaming"}
</Button>
)}
<Button
type="button"
icon="trash"
minimal={true}
onClick={handleClear}
disabled={isStreaming || response.length === 0}
>
Clear response
</Button>
</div>
</form>
</Card>
{streamError != null && (
<Callout title="Streaming failed" intent={Intent.DANGER} icon="error">
{streamError}
</Callout>
)}
<Card className={css.responseCard} compact={true}>
<div className={css.responseHeader}>
<div className={css.responseTitle}>
<span>Response</span>
{isStreaming && <Spinner size={14} />}
</div>
<div className={css.responseMetadata} aria-live="polite">
<Tag minimal={true}>{chunks.length} chunks</Tag>
<Tag minimal={true}>{response.length} characters</Tag>
</div>
</div>
<div
className={clsx(
css.responseBody,
response.length === 0 && css.emptyResponse,
)}
aria-live="polite"
aria-busy={isStreaming}
>
{response.length > 0 ? (
<p className={css.responseText}>
{response}
{isStreaming && <span className={css.cursor} aria-hidden="true" />}
</p>
) : (
<div className={css.emptyState}>
<Icon icon="feed" size={24} />
<span>The streamed response will appear here.</span>
</div>
)}
</div>
</Card>
</section>
</main>
);
};