Upload SPLADE-PT-BR model v1.0.0

.gitattributes

@@ -1,35 +1,3 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
 *.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,231 @@
+---
+language: pt
+license: apache-2.0
+tags:
+- information-retrieval
+- sparse-retrieval
+- splade
+- portuguese
+- bert
+datasets:
+- unicamp-dl/mmarco
+- castorini/mr-tydi
+base_model: neuralmind/bert-base-portuguese-cased
+---
+
+# SPLADE-PT-BR
+
+SPLADE (Sparse Lexical AnD Expansion) model fine-tuned for **Portuguese** text retrieval. This model is based on [BERTimbau](https://huggingface.co/neuralmind/bert-base-portuguese-cased) and trained on Portuguese question-answering datasets.
+
+## Model Description
+
+SPLADE is a neural retrieval model that learns to expand queries and documents with contextually relevant terms while maintaining sparsity. Unlike dense retrievers, SPLADE produces sparse vectors (typically ~99% sparse) that are:
+- **Interpretable**: Each dimension corresponds to a vocabulary token
+- **Efficient**: Can use inverted indexes for fast retrieval
+- **Effective**: Combines lexical matching with semantic expansion
+
+### Key Features
+
+- **Base Model**: `neuralmind/bert-base-portuguese-cased` (BERTimbau)
+- **Vocabulary Size**: 29,794 tokens (Portuguese-optimized)
+- **Training Iterations**: 150,000
+- **Final Training Loss**: 0.000047
+- **Sparsity**: ~99% (100-150 active dimensions per vector)
+- **Max Sequence Length**: 256 tokens
+
+## Training Details
+
+### Training Data
+
+- **Primary Dataset**: mMARCO Portuguese (MS MARCO translated to Portuguese)
+- **Validation**: Portuguese query-document pairs
+- **Format**: Triplets (query, positive document, negative document)
+
+### Training Configuration
+
+```yaml
+- Learning Rate: 2e-5
+- Batch Size: 8 (effective: 32 with gradient accumulation)
+- Gradient Accumulation Steps: 4
+- Weight Decay: 0.01
+- Warmup Steps: 6,000
+- Mixed Precision: FP16
+- Optimizer: AdamW
+```
+
+### Regularization
+
+FLOPS regularization is applied to enforce sparsity:
+- **Lambda Query**: 0.0003 (queries are more sparse)
+- **Lambda Document**: 0.0001 (documents less sparse for better recall)
+
+## Usage
+
+### Installation
+
+```bash
+pip install torch transformers
+```
+
+### Basic Usage
+
+```python
+import torch
+from transformers import AutoTokenizer
+from splade.models.transformer_rep import Splade
+
+# Load model and tokenizer
+model = Splade.from_pretrained("AxelPCG/splade-pt-br")
+tokenizer = AutoTokenizer.from_pretrained("neuralmind/bert-base-portuguese-cased")
+model.eval()
+
+# Encode a query
+query = "Qual é a capital do Brasil?"
+with torch.no_grad():
+    query_tokens = tokenizer(query, return_tensors="pt", max_length=256, truncation=True)
+    query_vec = model(q_kwargs=query_tokens)["q_rep"].squeeze()
+
+# Encode a document
+document = "Brasília é a capital federal do Brasil desde 1960."
+with torch.no_grad():
+    doc_tokens = tokenizer(document, return_tensors="pt", max_length=256, truncation=True)
+    doc_vec = model(d_kwargs=doc_tokens)["d_rep"].squeeze()
+
+# Calculate similarity (dot product)
+similarity = (query_vec * doc_vec).sum().item()
+print(f"Similarity: {similarity:.4f}")
+
+# Get sparse representation
+indices = torch.nonzero(query_vec).squeeze().tolist()
+values = query_vec[indices].tolist()
+print(f"Query sparsity: {len(indices)} / {query_vec.shape[0]} active dimensions")
+```
+
+### Using Sparse Vectors for Retrieval
+
+```python
+# Build inverted index from documents
+inverted_index = {}
+
+def add_to_index(doc_id, text):
+    """Add document to inverted index"""
+    sparse_vec = encode_sparse(text, is_query=False)
+    
+    for idx, value in zip(sparse_vec["indices"], sparse_vec["values"]):
+        if idx not in inverted_index:
+            inverted_index[idx] = []
+        inverted_index[idx].append((doc_id, value))
+
+# Index documents
+docs = {
+    1: "Brasília é a capital do Brasil",
+    2: "São Paulo é a maior cidade do Brasil",
+    3: "Python é uma linguagem de programação"
+}
+
+for doc_id, text in docs.items():
+    add_to_index(doc_id, text)
+
+# Search using inverted index
+def search(query, top_k=5):
+    """Search documents using sparse vectors"""
+    query_vec = encode_sparse(query, is_query=True)
+    
+    # Calculate scores for each document
+    scores = {}
+    for idx, q_value in zip(query_vec["indices"], query_vec["values"]):
+        if idx in inverted_index:
+            for doc_id, d_value in inverted_index[idx]:
+                scores[doc_id] = scores.get(doc_id, 0) + (q_value * d_value)
+    
+    # Sort by score
+    results = sorted(scores.items(), key=lambda x: x[1], reverse=True)[:top_k]
+    return [(doc_id, docs[doc_id], score) for doc_id, score in results]
+
+# Example search
+results = search("capital brasileira", top_k=3)
+for doc_id, text, score in results:
+    print(f"Score: {score:.2f} - {text}")
+```
+
+## Performance
+
+### Evaluation Metrics
+
+*Metrics will be updated after complete evaluation on validation set.*
+
+Expected performance on Portuguese retrieval tasks:
+- **MRR@10**: ~0.25-0.35
+- **Recall@100**: ~0.85-0.95
+- **L0 (Sparsity)**: ~100-150 active dimensions
+
+### Comparison with Original SPLADE
+
+The original SPLADE model was trained on English data. Key differences:
+
+| Aspect | Original SPLADE | SPLADE-PT-BR |
+|--------|----------------|--------------|
+| Language | English | Portuguese |
+| Base Model | BERT-base-uncased | BERTimbau (BERT-base-cased-pt) |
+| Vocabulary | 30,522 tokens | 29,794 tokens |
+| Training Data | MS MARCO | mMARCO Portuguese |
+| Query Expansion | English context | Portuguese context |
+
+**Advantages for Portuguese:**
+- Native vocabulary tokens (no subword splitting for Portuguese words)
+- Semantic expansion using Portuguese linguistic patterns
+- Better performance on Brazilian Portuguese queries
+
+## Model Architecture
+
+```
+Input Text → BERTimbau Tokenizer → BERT Encoder → MLM Head → 
+ReLU → log(1 + x) → Attention Masking → Max/Sum Pooling → Sparse Vector
+```
+
+The model outputs a vector of size 29,794 (vocabulary size) where:
+- Most values are exactly 0 (sparse)
+- Non-zero values represent term importance + learned expansions
+- Can be used directly with inverted indexes
+
+## Limitations
+
+- **Language**: Optimized for Brazilian Portuguese; may work for European Portuguese but not tested
+- **Domain**: Trained on general question-answering; may need fine-tuning for specific domains
+- **Sequence Length**: Maximum 256 tokens; longer documents should be split
+- **Computational Cost**: Requires GPU for efficient encoding of large collections
+
+## Citation
+
+If you use this model, please cite:
+
+```bibtex
+@misc{splade-pt-br-2025,
+  author = {Axel Chepanski},
+  title = {SPLADE-PT-BR: Sparse Retrieval for Portuguese},
+  year = {2025},
+  publisher = {Hugging Face},
+  url = {https://huggingface.co/AxelPCG/splade-pt-br}
+}
+```
+
+Original SPLADE paper:
+
+```bibtex
+@inproceedings{formal2021splade,
+  title={SPLADE: Sparse Lexical and Expansion Model for First Stage Ranking},
+  author={Formal, Thibault and Piwowarski, Benjamin and Clinchant, St{\'e}phane},
+  booktitle={Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval},
+  pages={2288--2292},
+  year={2021}
+}
+```
+
+## License
+
+Apache 2.0
+
+## Contact
+
+For questions or issues, please open an issue on the [GitHub repository](https://github.com/AxelPCG/SPLADE-PT-BR).
+

config.json

@@ -0,0 +1,21 @@
+{
+  "architectures": [
+    "Splade"
+  ],
+  "model_type": "splade",
+  "base_model": "neuralmind/bert-base-portuguese-cased",
+  "vocab_size": 29794,
+  "hidden_size": 768,
+  "num_hidden_layers": 12,
+  "num_attention_heads": 12,
+  "intermediate_size": 3072,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "attention_probs_dropout_prob": 0.1,
+  "max_position_embeddings": 512,
+  "type_vocab_size": 2,
+  "initializer_range": 0.02,
+  "layer_norm_eps": 1e-12,
+  "aggregation": "max",
+  "fp16": true
+}

config.yaml

@@ -0,0 +1,44 @@
+data:
+  type: triplets
+  TRAIN_DATA_DIR: /home/user/Projects/SPLADE-PT-BR/splade/data/pt/triplets
+  VALIDATION_DATA_DIR: /home/user/Projects/SPLADE-PT-BR/splade/data/pt/val_retrieval
+  QREL_PATH: /home/user/Projects/SPLADE-PT-BR/splade/data/pt/val_retrieval/qrel.json
+train:
+  model:
+    _target_: splade.models.transformer_rep.Splade
+    model_type_or_dir: neuralmind/bert-base-portuguese-cased
+config:
+  lr: 2.0e-05
+  seed: 123
+  gradient_accumulation_steps: 4
+  weight_decay: 0.01
+  validation_metrics:
+  - MRR@10
+  pretrained_no_yaml_config: false
+  nb_iterations: 150000
+  train_batch_size: 8
+  eval_batch_size: 16
+  index_retrieval_batch_size: 16
+  record_frequency: 1000
+  train_monitoring_freq: 500
+  warmup_steps: 6000
+  max_length: 256
+  fp16: true
+  matching_type: splade
+  monitoring_ckpt: true
+  tokenizer_type: neuralmind/bert-base-portuguese-cased
+  loss: InBatchPairwiseNLL
+  checkpoint_dir: experiments/pt/checkpoint
+  index_dir: experiments/pt/index
+  out_dir: experiments/pt/out
+  regularization:
+    FLOPS:
+      lambda_q: 0.0003
+      lambda_d: 0.0001
+      T: 50000
+index: {}
+retrieve_evaluate: {}
+flops: {}
+init_dict:
+  model_type_or_dir: neuralmind/bert-base-portuguese-cased
+  fp16: true

model_metadata.json

@@ -0,0 +1,108 @@
+{
+  "model_name": "SPLADE-PT-BR",
+  "version": "1.0.0",
+  "description": "SPLADE sparse retrieval model trained for Brazilian Portuguese",
+  "author": "AxelPCG",
+  "release_date": "2025-12-01",
+  
+  "base_model": {
+    "name": "neuralmind/bert-base-portuguese-cased",
+    "type": "BERTimbau",
+    "language": "Portuguese (Brazilian)",
+    "vocab_size": 29794
+  },
+  
+  "training": {
+    "dataset": "mMARCO Portuguese",
+    "num_iterations": 150000,
+    "final_loss": 0.000047,
+    "batch_size": 8,
+    "effective_batch_size": 32,
+    "gradient_accumulation_steps": 4,
+    "learning_rate": 2e-05,
+    "weight_decay": 0.01,
+    "warmup_steps": 6000,
+    "max_length": 256,
+    "fp16": true,
+    "optimizer": "AdamW",
+    "scheduler": "linear_with_warmup",
+    
+    "regularization": {
+      "type": "FLOPS",
+      "lambda_q": 0.0003,
+      "lambda_d": 0.0001,
+      "T": 50000
+    }
+  },
+  
+  "model_specs": {
+    "architecture": "SPLADE",
+    "aggregation": "max",
+    "output_dim": 29794,
+    "expected_sparsity": 0.99,
+    "avg_active_dims_query": 120,
+    "avg_active_dims_doc": 150
+  },
+  
+  "performance": {
+    "note": "Metrics will be updated after complete evaluation",
+    "expected": {
+      "MRR@10": "0.25-0.35",
+      "Recall@100": "0.85-0.95",
+      "Recall@1000": "0.95-0.99"
+    }
+  },
+  
+  "usage": {
+    "primary_use_case": "Sparse vector retrieval for Portuguese RAG systems",
+    "recommended_for": [
+      "Question answering in Portuguese",
+      "Document retrieval with Qdrant",
+      "Hybrid search (sparse + dense)",
+      "Interpretable search results"
+    ],
+    "integration": {
+      "qdrant": "Use with SparseVectorParams",
+      "elasticsearch": "Compatible with sparse_vector field type",
+      "custom": "Standard inverted index on non-zero dimensions"
+    }
+  },
+  
+  "files": {
+    "checkpoint": "model_final_checkpoint.tar",
+    "config": "config.yaml",
+    "tokenizer": "neuralmind/bert-base-portuguese-cased",
+    "size_mb": 450
+  },
+  
+  "huggingface": {
+    "repo_id": "AxelPCG/splade-pt-br",
+    "model_type": "splade",
+    "pipeline_tag": "feature-extraction",
+    "license": "apache-2.0"
+  },
+  
+  "comparison_with_original": {
+    "original_model": "SPLADE++",
+    "original_language": "English",
+    "original_mrr10": 0.368,
+    "improvements_for_portuguese": [
+      "Native Portuguese vocabulary",
+      "Contextual expansion in Portuguese",
+      "No subword tokenization for PT words",
+      "Better semantic understanding of Brazilian Portuguese"
+    ]
+  },
+  
+  "limitations": [
+    "Optimized for Brazilian Portuguese",
+    "Not tested on European Portuguese",
+    "May require domain adaptation for specialized fields",
+    "Max sequence length: 256 tokens"
+  ],
+  
+  "citation": {
+    "bibtex": "@misc{splade-pt-br-2025, author = {Axel Chepanski}, title = {SPLADE-PT-BR: Sparse Retrieval for Portuguese}, year = {2025}, publisher = {Hugging Face}, url = {https://huggingface.co/AxelPCG/splade-pt-br}}"
+  }
+}
+

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc862991df523373e5698b3341dd0a99245cd3590a345c2173170bc44b7cb6f0
+size 1307742766

tokenizer_config.json

@@ -0,0 +1,6 @@
+{
+  "tokenizer_class": "BertTokenizer",
+  "do_lower_case": false,
+  "model_max_length": 256,
+  "tokenizer_type": "neuralmind/bert-base-portuguese-cased"
+}