Update: Tried again on the latest Master branch (transformers-4.11.0.dev0) and this time I was able to get it to generate a different output by changing the seed. However, changing temperature, top_k and top_p still does not seem to have an influence on the outcome.
import jax
from transformers import FlaxGPTNeoForCausalLM, AutoTokenizer

model_name = 'EleutherAI/gpt-neo-125M'
model = FlaxGPTNeoForCausalLM.from_pretrained(model_name)
tokenizer = AutoTokenizer.from_pretrained(model_name)

prompt_text = "Hello there, my name is"
generated_max_length = 50

#Changing the seed and thus the prng_key value below, does seem to change the outcome.
seed = 1000
model.seed = seed

inputs = tokenizer(prompt_text, return_tensors="np")

#Changing temperature, top_k and top_p does not seem to change the outcome
outputs = model.generate(
    input_ids = inputs["input_ids"], 
    max_length=generated_max_length, 
    pad_token_id = model.config.eos_token_id,    
    prng_key=jax.random.PRNGKey(seed),
    temperature=1.0,
    early_stopping=True,
    top_k=50,
    top_p=0.95,
    do_sample=True,
    no_repeat_ngram_size=4)

output_sequence = outputs['sequences'].squeeze(0)
text = tokenizer.decode(output_sequence, clean_up_tokenization_spaces=True)

print(text)