from transformers import BlipProcessor, BlipForConditionalGeneration from sentence_transformers import SentenceTransformer import gradio as gr from PIL import Image import json import torch import faiss # ----------------------------- # 1. Load BLIP Model # ----------------------------- blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base") # ----------------------------- # 2. Load Vector DB (Fish Facts) # ----------------------------- # Sample fish knowledge base fish_data = [ { "name": "pufferfish", "text": "Contains tetrodotoxin in organs, highly poisonous" }, { "name": "stonefish", "text": "Has venomous dorsal spines causing intense pain" }, { "name": "lionfish", "text": "Venomous spines harmful to humans" }, { "name": "spotted trunkfish", "text": "Skin secretes toxins dangerous if ingested" }, { "name": "moray eel", "text": "May cause ciguatera poisoning if eaten" }, { "name": "barracuda", "text": "Associated with ciguatera poisoning" }, { "name": "surgeonfish", "text": "Can carry ciguatoxin harmful when consumed" }, { "name": "triggerfish", "text": "Some species carry ciguatoxin" }, { "name": "amberjack", "text": "Linked to ciguatera poisoning due to accumulated toxins" }, { "name": "grouper", "text": "May carry toxins causing ciguatera poisoning" }, { "name": "sardines", "text": "Small oily fish rich in omega-3, safe to consume" }, { "name": "mackerel", "text": "Rich in omega-3 fatty acids, commonly consumed" }, { "name": "salmon", "text": "Widely consumed nutritious fish, high in omega-3" }, { "name": "herring", "text": "Nutritious, consumed pickled or smoked" }, { "name": "rainbow trout", "text": "Mild flavor, good protein and omega-3 source" }, { "name": "anchovies", "text": "Small strong-flavored fish, commonly eaten" }, { "name": "cod", "text": "Lean white fish, good protein source" }, { "name": "tilapia", "text": "Mild flavor, widely farmed freshwater fish" }, { "name": "haddock", "text": "White fish, similar to cod, widely consumed" }, { "name": "catfish", "text": "Freshwater fish commonly consumed worldwide" }, { "name": "stargazer (uranoscopus spp.)", "text": "Bury themselves in sand and can deliver electric shocks as well as venom. Some species are considered delicacies after proper preparation." }, { "name": "rabbitfish (siganidae spp.)", "text": "Reef-associated fish with venomous spines capable of inflicting painful wounds." }, { "name": "soapfish (rypticus spp.)", "text": "Produce a bitter toxin called grammistin from their skin to deter predators, which may cause illness in humans." }, { "name": "sea bream (sarpa salpa)", "text": "Known as \"salema,\" this fish can cause hallucinogenic effects when consumed due to the presence of toxins." }, { "name": "blue-ringed octopus (hapalochlaena spp.)", "text": "While not a fish, this marine animal is noteworthy for its venomous bite, which can be fatal to humans." }, { "name": "greenland shark (somniosus microcephalus)", "text": "Flesh contains high levels of toxins and is poisonous if not properly processed before consumption." }, { "name": "sturgeon (acipenser spp.)", "text": "Primarily known for caviar; generally considered safe to eat when properly prepared." }, { "name": "sea bass (dicentrarchus labrax)", "text": "Popular in culinary dishes; generally considered safe to eat when properly prepared." }, { "name": "red snapper (lutjanus campechanus)", "text": "Highly prized for its taste; generally considered safe to eat when properly prepared." }, { "name": "parrotfish (scaridae spp.)", "text": "Known for their vibrant colors and role in reef ecosystems; generally considered safe to eat when properly prepared." }, { "name": "arctic char (salvelinus alpinus)", "text": "A cold-water fish related to salmon and trout, known for its delicate flavor and pink flesh. Commonly found in Arctic and sub-Arctic regions." }, { "name": "black crappie (pomoxis nigromaculatus)", "text": "A freshwater fish popular among anglers, recognized by its dark, speckled pattern. Inhabits lakes, ponds, and slow-moving rivers." }, { "name": "blue catfish (ictalurus furcatus)", "text": "A large freshwater fish with a bluish-gray body and a deeply forked tail. Native to major river systems in the United States." }, { "name": "bluegill (lepomis macrochirus)", "text": "A small, freshwater sunfish with a distinctive blue patch on its gill cover. Commonly found in North American lakes and ponds." }, { "name": "golden shiner (notemigonus crysoleucas)", "text": "A small, golden-colored freshwater fish often used as bait. Native to eastern North America." }, { "name": "goldfish (carassius auratus)", "text": "A popular ornamental freshwater fish known for its bright orange color and adaptability. Originally domesticated in China." }, { "name": "goby fish (family gobiidae)", "text": "A diverse family of small fish, some species of which are toxic. Found in marine and freshwater environments worldwide." }, { "name": "goliath tigerfish (hydrocynus goliath)", "text": "A large, aggressive freshwater fish native to African rivers, known for its sharp teeth and predatory behavior." }, { "name": "goblin shark (mitsukurina owstoni)", "text": "A rare, deep-sea shark with a distinctive elongated snout and protrusible jaws. Considered a \"living fossil\" due to its ancient lineage." }, { "name": "greenland shark (somniosus microcephalus)", "text": "A large, slow-moving shark native to the North Atlantic. Its flesh contains toxins and is poisonous if not properly processed before consumption." }, { "name": "almaco jack (seriola rivoliana)", "text": "A pelagic fish found in warm seas, known for its strong fighting ability when caught by anglers. Often marketed as \"Kampachi\" in seafood cuisine." }, { "name": "black bullhead (ameiurus melas)", "text": "A freshwater catfish native to North America, characterized by its dark coloration and adaptability to various habitats. Commonly found in slow-moving waters." }, { "name": "brook stickleback (culaea inconstans)", "text": "A small freshwater fish native to North America, recognized by its five to six dorsal spines. Often inhabits streams and ponds with abundant vegetation." }, { "name": "bonytail chub (gila elegans)", "text": "An endangered freshwater fish native to the Colorado River basin, known for its streamlined body and historical significance in the region's ecosystem." }, { "name": "brassy minnow (hybognathus hankinsoni)", "text": "A small freshwater fish found in North America, notable for its brassy coloration and preference for clear, vegetated streams and ponds." }, { "name": "boxfish (family ostraciidae)", "text": "Marine fish known for their box-like shape and rigid body. Some species can secrete toxins when stressed, which can be harmful to other fish in aquariums." }, { "name": "blowfish (family tetraodontidae)", "text": "Also known as pufferfish, these fish can inflate their bodies as a defense mechanism. Many species contain tetrodotoxin, a potent neurotoxin harmful to humans if ingested." }, { "name": "cowfish (family ostraciidae)", "text": "Closely related to boxfish, cowfish are characterized by horn-like structures on their heads. Some species can release toxins when threatened, posing risks in aquarium settings." }, { "name": "porcupinefish (family diodontidae)", "text": "Similar to pufferfish, porcupinefish can inflate their bodies and have spines that radiate outward when inflated. They contain tetrodotoxin, making them dangerous if consumed." }, { "name": "ocellate river stingray (potamotrygon motoro)", "text": "A freshwater stingray native to South American rivers, recognized by its distinctive eye-like spots. Possesses venomous barbs that can cause painful injuries." }, { "name": "milkfish (chanos chanos)", "text": "Also known as \"bony fish,\" milkfish are freshwater species that can accumulate ciguatoxin in their muscles, leading to severe food poisoning in humans if consumed." }, { "name": "escolar (lepidocybium flavobrunneum)", "text": "A deep-sea fish known for its rich, oily flesh. However, it contains high levels of indigestible wax esters (gempylotoxin) that can cause gastrointestinal distress in humans." }, { "name": "tiger shark (galeocerdo cuvier)", "text": "A large predatory shark found in tropical and temperate waters. While not typically consumed, its flesh can contain high levels of toxins, and it is considered dangerous due to its aggressive behavior." }, { "name": "titan triggerfish (balistoides viridescens)", "text": "A large reef fish known for its aggressive behavior, especially when guarding its nest. While not poisonous to eat, it can inflict serious injuries to divers." }, { "name": "yellowmargin triggerfish (pseudobalistes flavimarginatus)", "text": "Similar to the Titan Triggerfish, this species is known for its territorial nature and can be aggressive towards intruders." }, { "name": "portuguese man o' war (physalia physalis)", "text": "Often mistaken for a jellyfish, this marine hydrozoan has long tentacles that deliver painful stings. While not a fish, it's notable for its venomous nature." }, { "name": "cone snail (genus conus)", "text": "Marine snails known for their beautiful shells and venomous harpoons used to capture prey. Some species' venom can be fatal to humans." }, { "name": "fire coral (millepora spp.)", "text": "Despite its name, fire coral is a hydrozoan with stinging nematocysts that can cause painful reactions upon contact." }, { "name": "crown-of-thorns starfish (acanthaster planci)", "text": "A large starfish covered in venomous spines. Contact can cause severe pain and swelling." }, { "name": "blister beetle (family meloidae)", "text": "Insects known for secreting a toxic substance called cantharidin, which can cause blistering upon contact with skin." }, { "name": "angelfish", "text": "Freshwater species popular in aquariums, known for their distinctive shape and vibrant colors." }, { "name": "clownfish", "text": "Marine fish recognized for their bright orange color with white bands; often associated with sea anemones." }, { "name": "goldfish", "text": "Common freshwater aquarium fish, typically orange, bred for various colorations and fin types." }, { "name": "catfish", "text": "Diverse group of ray-finned fish characterized by prominent barbels resembling cat whiskers." }, { "name": "salmon", "text": "Anadromous fish known for migrating from the ocean to freshwater to spawn; valued for their rich, oily flesh." }, { "name": "swordfish", "text": "Large predatory fish recognized by their elongated, flat bill; popular in culinary dishes." }, { "name": "tuna", "text": "Fast-swimming pelagic fish, widely consumed worldwide; includes species like albacore, yellowfin, and bluefin." }, { "name": "trout", "text": "Freshwater fish related to salmon, found in cold-water streams and lakes; prized by anglers and for culinary use." }, { "name": "guppy", "text": "Small freshwater fish popular in home aquariums; known for their vibrant colors and live-bearing reproduction." }, { "name": "betta", "text": "Also known as Siamese fighting fish; recognized for their vivid colors and aggressive behavior." }, { "name": "seahorse", "text": "Marine fish with a horse-like head and prehensile tail; males carry embryos in a specialized pouch." }, { "name": "pufferfish", "text": "Known for their ability to inflate when threatened; some species contain tetrodotoxin, a potent neurotoxin." }, { "name": "tilapia", "text": "Freshwater fish widely farmed for its mild flavor and adaptability to various environments." }, { "name": "mackerel", "text": "Fast-swimming, migratory fish known for their rich, oily meat; includes species like Atlantic and king mackerel." }, { "name": "cod", "text": "Demersal fish valued for their mild flavor and flaky white flesh; includes Atlantic and Pacific varieties." }, { "name": "bass", "text": "Encompasses various freshwater and marine species popular among anglers and in cuisine." }, { "name": "snapper", "text": "Marine fish known for their sharp teeth and vibrant colors; includes species like red snapper." }, { "name": "haddock", "text": "Bottom-dwelling fish related to cod; commonly used in dishes like fish and chips." }, { "name": "perch", "text": "Freshwater fish with a spiny dorsal fin; includes species like European perch and yellow perch." }, { "name": "mahi-mahi", "text": "Also known as dolphinfish; surface-dwelling fish recognized for their dazzling colors and high-speed swimming." }, { "name": "carp", "text": "Freshwater fish native to Europe and Asia; often farmed and consumed in various cuisines." }, { "name": "halibut", "text": "Large flatfish found in both the North Atlantic and North Pacific; prized for its firm, white flesh." }, { "name": "flounder", "text": "Flatfish species known for both eyes being on one side of their body; inhabit coastal waters." }, { "name": "sardine", "text": "Small, oily fish often consumed canned; rich in omega-3 fatty acids." }, { "name": "wrasse", "text": "Colorful reef fish known for their role in cleaning parasites from other fish." } ] texts = [item["text"] for item in fish_data] names = [item["name"] for item in fish_data] # Encode fish data embedder = SentenceTransformer('all-MiniLM-L6-v2') embeddings = embedder.encode(texts) # Create FAISS index dim = embeddings[0].shape[0] index = faiss.IndexFlatL2(dim) index.add(embeddings) # Toxic fish list toxic_list = ['Pufferfish','Stonefish','Lionfish','Spotted Trunkfish','Moray Eel','Barracuda','Surgeonfish','Triggerfish','Amberjack','Grouper','Sardines','Mackerel','Salmon','Herring','Rainbow Trout','Anchovies','Cod','Tilapia','Haddock','Catfish','Stargazer','Rabbitfish','Soapfish','Sea Bream','Blue-ringed Octopus','Greenland Shark','Sturgeon','Sea Bass','Red Snapper','Parrotfish','Arctic Char','Black Crappie','Blue Catfish','Bluegill','Golden Shiner','Goldfish','Goby Fish','Goliath Tigerfish','Goblin Shark','Greenland Shark','Almaco Jack','Black Bullhead','Brook Stickleback','Bonytail Chub','Brassy Minnow','Boxfish','Blowfish','Cowfish','Porcupinefish','Ocellate River Stingray','Milkfish','Escolar','Tiger Shark','Titan Triggerfish','Yellowmargin Triggerfish','Portuguese Man o War','Cone Snail','Fire Coral','Crown-of-Thorns Starfish','Blister Beetle','Angelfish','Clownfish','Goldfish','Swordfish','Tuna','Guppy','Betta','Seahorse','Bass','Snapper','Perch','Mahi-mahi','Carp','Halibut','Flounder','Wrasse'] # ----------------------------- # 3. Fish Identifier Function # ----------------------------- def identify_fish(image): # Step 1: Generate caption from image inputs = blip_processor(image, return_tensors="pt") out = blip_model.generate(**inputs) caption = blip_processor.decode(out[0], skip_special_tokens=True) # Step 2: Search fish knowledge base query_embed = embedder.encode([caption]) D, I = index.search(query_embed, k=1) matched_fish = fish_data[I[0][0]] name = matched_fish["name"] info = matched_fish["text"] # Step 3: Classify toxicity is_poisonous = "Yes 🧪" if name in toxic_list else "No ✅" # Step 4: Return result return f"**Image Caption:** {caption}\n\n**Detected Fish:** {name.title()}\n**Poisonous:** {is_poisonous}\n**Fact:** {info}" # ----------------------------- # 4. Gradio UI # ----------------------------- demo = gr.Interface( fn=identify_fish, inputs=gr.Image(type="pil"), outputs="markdown", title="🧠 Smart Fish Identifier (BLIP + RAG)", description="Upload a fish image. We use BLIP to describe the fish, then match it with our fish fact knowledge base using RAG to determine if it's poisonous." ) if __name__ == '__main__': demo.launch()