improve text inference

submission_script.py

@@ -0,0 +1,92 @@
+import requests
+import json
+from pprint import pprint
+import time
+import sys
+
+def evaluate_text_model(space_url: str, max_retries=3, retry_delay=5):
+    """
+    Evaluate a text classification model through its API endpoint
+    """
+    params = {
+        "dataset_name": "QuotaClimat/frugalaichallenge-text-train",
+        "test_size": 0.2,
+        "test_seed": 42,
+    }
+    
+    # Construct API URL
+    if "localhost" in space_url:
+        api_url = f"{space_url}/text"
+    else:
+        api_url = f"https://{space_url.replace('/', '-')}.hf.space/text"
+
+    headers = {
+        'Content-Type': 'application/json',
+        'Accept': 'application/json'
+    }
+
+    for attempt in range(max_retries):
+        try:
+            print(f"\nAttempt {attempt + 1} of {max_retries}")
+            print(f"Making request to: {api_url}")
+            
+            # Health check
+            health_url = f"https://{space_url.replace('/', '-')}.hf.space/health"
+            health_response = requests.get(health_url, timeout=30)
+            
+            if health_response.status_code != 200:
+                print(f"Space not ready (status: {health_response.status_code})")
+                time.sleep(retry_delay)
+                continue
+
+            # Make API call
+            response = requests.post(
+                api_url,
+                json=params,
+                headers=headers,
+                timeout=300
+            )
+            
+            if response.status_code == 200:
+                return response.json()
+            else:
+                print(f"Error: Status {response.status_code}")
+                print(f"Response: {response.text}")
+                time.sleep(retry_delay)
+                
+        except requests.exceptions.RequestException as e:
+            print(f"Request error: {str(e)}")
+            if attempt < max_retries - 1:
+                time.sleep(retry_delay)
+        except Exception as e:
+            print(f"Unexpected error: {str(e)}")
+            if attempt < max_retries - 1:
+                time.sleep(retry_delay)
+    
+    return None
+
+def main():
+    # Space URL
+    space_url = "Tonic/frugal-ai-submission-template"
+    
+    print("\nStarting model evaluation...")
+    results = evaluate_text_model(space_url)
+    
+    if results:
+        print("\nEvaluation Results:")
+        print("-" * 50)
+        print(f"Accuracy: {results.get('accuracy', 'N/A'):.4f}")
+        print(f"Energy (Wh): {results.get('energy_consumed_wh', 'N/A'):.6f}")
+        print(f"Emissions (gCO2eq): {results.get('emissions_gco2eq', 'N/A'):.6f}")
+        print("\nFull Results:")
+        pprint(results)
+    else:
+        print("\nEvaluation failed!")
+        print("Troubleshooting:")
+        print(f"1. Check space status: https://{space_url.replace('/', '-')}.hf.space")
+        print("2. Verify API implementation")
+        print("3. Try again later")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

tasks/text.py

@@ -2,7 +2,6 @@ from fastapi import APIRouter
 from datetime import datetime
 from datasets import load_dataset
 from sklearn.metrics import accuracy_score
-import random
 import torch
 from transformers import AutoTokenizer, AutoModelForSequenceClassification
 from torch.utils.data import Dataset, DataLoader
@@ -12,23 +11,45 @@ from .utils.emissions import tracker, clean_emissions_data, get_space_info
 
 router = APIRouter()
 
-DESCRIPTION = "Random Baseline"
+DESCRIPTION = "Climate Guard Toxic Agent Model"
 ROUTE = "/text"
 
-@router.post(ROUTE, tags=["Text Task"], 
-             description=DESCRIPTION)
+class TextDataset(Dataset):
+    def __init__(self, texts, labels, tokenizer, max_len=128):
+        self.texts = texts
+        self.labels = labels
+        self.tokenizer = tokenizer
+        self.max_len = max_len
+
+    def __len__(self):
+        return len(self.texts)
+
+    def __getitem__(self, idx):
+        text = str(self.texts[idx])
+        label = self.labels[idx]
+        
+        encoding = self.tokenizer(
+            text,
+            max_length=self.max_len,
+            padding='max_length',
+            truncation=True,
+            return_tensors="pt"
+        )
+        
+        return {
+            'input_ids': encoding['input_ids'].squeeze(0),
+            'attention_mask': encoding['attention_mask'].squeeze(0),
+            'labels': torch.tensor(label, dtype=torch.long)
+        }
+
+@router.post(ROUTE, tags=["Text Task"], description=DESCRIPTION)
 async def evaluate_text(request: TextEvaluationRequest):
     """
     Evaluate text classification for climate disinformation detection.
-    
-    Current Model: Random Baseline
-    - Makes random predictions from the label space (0-7)
-    - Used as a baseline for comparison
     """
-    # Get space info
     username, space_url = get_space_info()
 
-    # Define the label mapping
+    # Label mapping
     LABEL_MAPPING = {
         "0_not_relevant": 0,
         "1_not_happening": 1,
@@ -40,99 +61,79 @@ async def evaluate_text(request: TextEvaluationRequest):
         "7_fossil_fuels_needed": 7
     }
 
-    # Load and prepare the dataset
+    # Load dataset
     dataset = load_dataset(request.dataset_name)
-
+    
     # Convert string labels to integers
     dataset = dataset.map(lambda x: {"label": LABEL_MAPPING[x["label"]]})
-
-    # Split dataset
-    train_test = dataset["train"]
+    
+    # Get test dataset
     test_dataset = dataset["test"]
     
     # Start tracking emissions
     tracker.start()
-    tracker.start_task("inference")
-
-    # Load the model and tokenizer
-    model_name = "Tonic/climate-guard-toxic-agent"
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    model = AutoModelForSequenceClassification.from_pretrained(model_name)
-
-    class TextDataset(Dataset):
-        def __init__(self, texts, labels, tokenizer, max_len=128):
-            self.texts = texts
-            self.labels = labels
-            self.tokenizer = tokenizer
-            self.max_len = max_len
-
-        def __len__(self):
-            return len(self.texts)
-
-        def __getitem__(self, idx):
-            text = self.texts[idx]
-            label = self.labels[idx]
-            encodings = self.tokenizer(
-                text,
-                max_length=self.max_len,
-                padding='max_length',
-                truncation=True,
-                return_tensors="pt"
-            )
-            return {
-                'input_ids': encodings['input_ids'].squeeze(0),
-                'attention_mask': encodings['attention_mask'].squeeze(0),
-                'labels': torch.tensor(label, dtype=torch.long)
-            }
-
-    # Create dataset and dataloader
-    test_dataset = TextDataset(texts, labels, tokenizer)
-    test_loader = DataLoader(test_dataset, batch_size=16)
-
-    # Model inference
-    model.eval()
-    predictions = []
-    ground_truth = []
-    DEVICE = 'cpu'
-    with torch.no_grad():
-        for batch in test_loader:
-            input_ids = batch['input_ids'].to(DEVICE)
-            attention_mask = batch['attention_mask'].to(DEVICE)
-            labels = batch['labels'].to(DEVICE)
-    
-            outputs = model(input_ids=input_ids, attention_mask=attention_mask)
-            _, predicted = torch.max(outputs.logits, 1)
-    
-            predictions.extend(predicted.cpu().numpy())
-            ground_truth.extend(labels.cpu().numpy())
-
-    #--------------------------------------------------------------------------------------------
-    # YOUR MODEL INFERENCE STOPS HERE
-    #--------------------------------------------------------------------------------------------   
-
     
-    # Stop tracking emissions
-    emissions_data = tracker.stop_task()
-    
-    # Calculate accuracy
-    accuracy = accuracy_score(ground_truth, predictions)
-    
-    # Prepare results dictionary
-    results = {
-        "username": username,
-        "space_url": space_url,
-        "submission_timestamp": datetime.now().isoformat(),
-        "model_description": DESCRIPTION,
-        "accuracy": float(accuracy),
-        "energy_consumed_wh": emissions_data.energy_consumed * 1000,
-        "emissions_gco2eq": emissions_data.emissions * 1000,
-        "emissions_data": clean_emissions_data(emissions_data),
-        "api_route": ROUTE,
-        "dataset_config": {
-            "dataset_name": request.dataset_name,
-            "test_size": request.test_size,
-            "test_seed": request.test_seed
+    try:
+        # Load model and tokenizer
+        model_name = "Tonic/climate-guard-toxic-agent"
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModelForSequenceClassification.from_pretrained(model_name)
+        
+        # Prepare dataset
+        test_data = TextDataset(
+            texts=test_dataset["text"],
+            labels=test_dataset["label"],
+            tokenizer=tokenizer
+        )
+        
+        test_loader = DataLoader(test_data, batch_size=16)
+        
+        # Model inference
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        model = model.to(device)
+        model.eval()
+        
+        predictions = []
+        ground_truth = []
+        
+        with torch.no_grad():
+            for batch in test_loader:
+                input_ids = batch['input_ids'].to(device)
+                attention_mask = batch['attention_mask'].to(device)
+                labels = batch['labels'].to(device)
+                
+                outputs = model(input_ids=input_ids, attention_mask=attention_mask)
+                _, predicted = torch.max(outputs.logits, 1)
+                
+                predictions.extend(predicted.cpu().numpy())
+                ground_truth.extend(labels.cpu().numpy())
+        
+        # Calculate accuracy
+        accuracy = accuracy_score(ground_truth, predictions)
+        
+        # Stop tracking emissions
+        emissions_data = tracker.stop()
+        
+        # Prepare results
+        results = {
+            "username": username,
+            "space_url": space_url,
+            "submission_timestamp": datetime.now().isoformat(),
+            "model_description": DESCRIPTION,
+            "accuracy": float(accuracy),
+            "energy_consumed_wh": emissions_data.energy_consumed * 1000,
+            "emissions_gco2eq": emissions_data.emissions * 1000,
+            "emissions_data": clean_emissions_data(emissions_data),
+            "api_route": ROUTE,
+            "dataset_config": {
+                "dataset_name": request.dataset_name,
+                "test_size": request.test_size,
+                "test_seed": request.test_seed
+            }
         }
-    }
-    
-    return results 
+        
+        return results
+        
+    except Exception as e:
+        tracker.stop()
+        raise e