💻 Примеры: CLIP и генерация изображений

Пример 1. CLIP: сходство текст-изображение

Это адаптация llm-course/lesson-ai-07/ai7-1.py: модель CLIP получает список текстов и список изображений, кодирует их в векторы и сравнивает.

# clip_similarity.py
from io import BytesIO

import clip
import requests
import torch
from PIL import Image


def get_device():
    return "cuda" if torch.cuda.is_available() else "cpu"


def load_image_from_url(url):
    response = requests.get(url, timeout=20)
    response.raise_for_status()
    return Image.open(BytesIO(response.content)).convert("RGB")


def text_to_image_similarity(model, preprocess, text_queries, images, device):
    text_inputs = clip.tokenize(text_queries).to(device)
    image_inputs = torch.stack([preprocess(img) for img in images]).to(device)

    with torch.no_grad():
        text_features = model.encode_text(text_inputs)
        image_features = model.encode_image(image_inputs)

        text_features /= text_features.norm(dim=-1, keepdim=True)
        image_features /= image_features.norm(dim=-1, keepdim=True)

        scores = (100.0 * text_features @ image_features.T).softmax(dim=-1)
    return scores.cpu().numpy()


def main():
    device = get_device()
    model, preprocess = clip.load("ViT-B/32", device=device)
    model.eval()

    image_urls = [
        "https://cdn.shopify.com/s/files/1/0086/0795/7054/files/Golden-Retriever.jpg?v=1645179525",
        "https://miro.medium.com/v2/resize:fit:1400/1*tMKkGydXuiOBOb15srANvg@2x.jpeg",
    ]
    images = [load_image_from_url(url) for url in image_urls]
    text_queries = ["a dog", "a car", "a sunset", "a person"]

    similarities = text_to_image_similarity(
        model, preprocess, text_queries, images, device
    )

    for i, text in enumerate(text_queries):
        print(f"\nText: {text!r}")
        for j, score in enumerate(similarities[i]):
            print(f"  Image {j + 1}: {score * 100:.2f}%")


if __name__ == "__main__":
    main()

В исходном коде урока используются две онлайн-картинки: собака и закат. Попробуйте заменить URL на свои изображения и добавить описания, которые могут запутать модель.

Пример 2. CLIP: zero-shot классификация

Классификация делается без обучения на новых классах: мы просто даём варианты текстовых меток.

# zero_shot_clip.py
labels = [
    "a photo of a dog",
    "a photo of a cat",
    "a photo of a car",
    "a photo of a sunset",
]

image_input = preprocess(image).unsqueeze(0).to(device)
text_inputs = clip.tokenize(labels).to(device)

with torch.no_grad():
    image_features = model.encode_image(image_input)
    text_features = model.encode_text(text_inputs)
    image_features /= image_features.norm(dim=-1, keepdim=True)
    text_features /= text_features.norm(dim=-1, keepdim=True)
    probabilities = (100.0 * image_features @ text_features.T).softmax(dim=-1)

for label, probability in zip(labels, probabilities.cpu().numpy()[0]):
    print(f"{label}: {probability * 100:.2f}%")

Пример 3. Генерация изображений через Hugging Face

В исходном ai7-2.py используется HTTP-запрос к Hugging Face Inference API. Ниже — современный вариант через InferenceClient, который соответствует текущей документации Hugging Face.

# generate_image.py
import os

from dotenv import load_dotenv
from huggingface_hub import InferenceClient


def setup_client():
    load_dotenv()
    token = os.getenv("HF_TOKEN")
    if not token:
        raise RuntimeError("Set HF_TOKEN in .env")
    return InferenceClient(provider="fal-ai", api_key=token)


def generate_image(client, prompt, filename):
    image = client.text_to_image(
        prompt,
        model="black-forest-labs/FLUX.1-dev",
    )
    image.save(filename)
    return filename


def main():
    client = setup_client()
    prompts = [
        "A serene landscape with mountains and a lake at sunset, realistic photo",
        "A futuristic city with flying cars and neon lights, cinematic concept art",
        "A small robot reading a Python book, watercolor illustration",
    ]

    for index, prompt in enumerate(prompts, start=1):
        filename = f"generated_image_{index}.png"
        generate_image(client, prompt, filename)
        print(f"Saved {filename}: {prompt}")


if __name__ == "__main__":
    main()