smaller batch_size, higher temperature

generate.py

@@ -30,14 +30,14 @@ if torch.backends.mps.is_available():
 else:
     device = "cuda"
     model_id = "google/gemma-2b-it"
-    batch_size = 10
+    batch_size = 4
 
 model = models.transformers(model_id, device=device)
 
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 sampler = PenalizedMultinomialSampler()
 low_temperature_sampler = PenalizedMultinomialSampler(temperature=0.3)
-high_temperature_sampler = PenalizedMultinomialSampler(temperature=1.1)
+high_temperature_sampler = PenalizedMultinomialSampler(temperature=1.5)
 empty_tokens = [token_id for token_id in range(tokenizer.vocab_size) if not tokenizer.decode([token_id], skip_special_tokens=True).strip()]
 sampler.set_max_repeats(empty_tokens, 1)
 disallowed_patterns = [regex.compile(r"\p{Han}")]  # focus on english for now