Image Analysis with AI by Automating Captions, Detection and Similarity Search in Python

				
					import os
os.environ['LC_ALL'] = 'C.UTF-8'
os.environ['LANG'] = 'C.UTF-8'

!pip install sentence-transformers
!pip install -q transformers ipyplot==1.1.0 ftfy

from PIL import Image
import requests
from transformers import CLIPProcessor, CLIPModel
import torch
from io import BytesIO
import ipyplot
from sentence_transformers import util

def initialize_clip_model():
    model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
    processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
    return model, processor

def fetch_images(api_key, search_keyword, number_to_retrieve):
    api_url = f"https://pixabay.com/api/?key={api_key}&q={search_keyword.lower()}&image_type=photo&safesearch=true&per_page={number_to_retrieve}"
    response = requests.get(api_url)
    output = response.json()
    images = [Image.open(BytesIO(requests.get(item['webformatURL']).content)).convert("RGB") for item in output["hits"]]
    return images

def perform_semantic_search(model, processor, phrase, images, top_k = 3):
    with torch.no_grad():
        inputs = processor(text=[semantic_search_phrase], images=all_images, return_tensors="pt", padding=True)
        outputs = model(**inputs)
    logits_per_image = outputs.logits_per_image
    probs = logits_per_image.softmax(dim=1)
    values, indices = logits_per_image.squeeze().topk(top_k) # Top-3
    top_images, top_scores = [], []
    for score, index in zip(values, indices):
        top_images.append(images[index.item()])  # Convert tensor to integer
        top_scores.append(round(score.item(), top_k))
    return top_images, top_scores

def display_images(images, scores):
    print(f"Scores: {scores}")
    ipyplot.plot_images(images, img_width=300)

def perform_visual_search(model, processor, target_image_url, candidate_images, top_k=3):
    # Load the target image
    target_image = Image.open(requests.get(target_image_url, stream=True).raw).convert("RGB")

    # Preprocess the target image and candidate images
    target_inputs = processor(images=target_image, return_tensors="pt")
    candidate_inputs = [processor(images=img, return_tensors="pt") for img in candidate_images]

    # Compute embeddings for the target image and candidate images
    with torch.no_grad():
        target_outputs = model.get_image_features(**target_inputs)
        candidate_outputs = [model.get_image_features(**inputs) for inputs in candidate_inputs]

    # Compute cosine similarities
    similarities = [util.cos_sim(target_outputs, output).item() for output in candidate_outputs]

    # Sort the candidate images based on their similarity scores
    sorted_indices = sorted(range(len(similarities)), key=lambda i: similarities[i], reverse=True)
    top_images = [candidate_images[idx] for idx in sorted_indices[:top_k]]
    top_scores = [similarities[idx] for idx in sorted_indices[:top_k]]

    return top_images, top_scores

# Main execution
model, processor = initialize_clip_model()

# Setup search query
pixabay_api_key = "22176616-358d1b190a298ff59f96b35a1"  # Use your actual API key
semantic_search_phrase = "small dog with hat"
pixabay_search_keyword = "dog with hat"
no_to_retrieve = 100

all_images = fetch_images(pixabay_api_key, pixabay_search_keyword, no_to_retrieve)
print("Total number of images retrieved from Pixabay: ", len(all_images))
print("Images retrieved from Pixabay:")

ipyplot.plot_images(all_images, max_images=no_to_retrieve, img_width=150)

print("Images with The highest Semantic Similarity")

top_images, top_scores = perform_semantic_search(model, processor, semantic_search_phrase, all_images)
display_images(top_images, top_scores)

# URL of the target image for visual similarity search
target_image_url = "https://i.ibb.co/zfmc38w/istockphoto-152514029-612x612.jpg"

# Perform visual similarity search based on the target image
print("Images with the highest Visual Similarity:")
top_visual_images, top_visual_scores = perform_visual_search(model, processor, target_image_url, all_images, top_k=3)
display_images(top_visual_images, top_visual_scores)