How Google’s TIPSv2 Outperforms Competitors Locally?

python3 -m venv .venv
source .venv/bin/activate
python -m pip install --upgrade pip

# GPU wheels
pip install torch torchvision --index-url https://download.pytorch.org/whl/cu121

# Common Python packages
pip install jupyter numpy pillow scikit-learn matplotlib requests tqdm

jupyter notebook

import torch
import torch.nn.functional as F
from PIL import Image
from torchvision import transforms

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# Replace these stubs with the real TIPSv2 model loader
class TIPSv2:
    def __init__(self, image_encoder, text_encoder, processor):
        self.image_encoder = image_encoder.to(device).eval()
        self.text_encoder = text_encoder.to(device).eval()
        self.processor = processor

    @torch.inference_mode()
    def encode_image(self, image):
        x = self.processor(image).to(device)
        # Returns a dict with 'cls' and 'patches'
        return self.image_encoder(x)

    @torch.inference_mode()
    def encode_text(self, texts):
        # Returns a tensor of shape [N, D]
        return self.text_encoder(texts)

# Example processor that resizes to 448 and normalizes
def build_image_processor(img_size=448):
    return transforms.Compose([
        transforms.Resize((img_size, img_size), interpolation=transforms.InterpolationMode.BICUBIC),
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.48145466, 0.4578275, 0.40821073],
                             std=[0.26862954, 0.26130258, 0.27577711]),
        lambda x: x.unsqueeze(0)
    ])

# TODO: plug in the real encoders from the official release
image_encoder = ...  # e.g., load_image_encoder(checkpoint_path)
text_encoder  = ...  # e.g., load_text_encoder(checkpoint_path)
processor     = build_image_processor()

tips = TIPSv2(image_encoder, text_encoder, processor)

from pathlib import Path

img_path = Path("path_to_your_image.jpg")
image = Image.open(img_path).convert("RGB")

with torch.inference_mode():
    out = tips.encode_image(image)

# Expected keys: 'cls' -> [1, D], 'patches' -> [1, N, D]
cls = out["cls"]           # shape [1, 768] for a base model
patches = out["patches"]   # shape [1, 1024, 768] for a 32x32 grid

print("CLS shape:", tuple(cls.shape))
print("Patches shape:", tuple(patches.shape))

def l2_normalize(x, dim=-1, eps=1e-12):
    return x / (x.norm(dim=dim, keepdim=True) + eps)

def cosine_sim(a, b):
    return a @ b.t()

labels = [
    "cat",
    "dog",
    "bird",
    "a cat standing on rocky ground outdoors",
    "a paw raised"
]

with torch.inference_mode():
    img_vec = l2_normalize(cls, dim=-1)                   # [1, D]
    txt_vec = l2_normalize(tips.encode_text(labels), -1)  # [L, D]
    sims = cosine_sim(img_vec, txt_vec).squeeze(0)        # [L]

topk = torch.topk(sims, k=min(5, len(labels)))
for idx in topk.indices.tolist():
    print(labels[idx], float(sims[idx]))

queries = [
    "a close up of a pet on a rock",
    "an animal outdoors",
    "a person riding a bike",
    "a scenic mountain view",
    "a raised paw"
]

with torch.inference_mode():
    q_vec = l2_normalize(tips.encode_text(queries), -1)   # [Q, D]
    scores = cosine_sim(img_vec, q_vec).squeeze(0)        # [Q]

order = torch.argsort(scores, descending=True).tolist()
for i in order:
    print(queries[i], float(scores[i]))