diff --git a/transformer_lens/HookedEncoder.py b/transformer_lens/HookedEncoder.py
index e470731d3..254a72a10 100644
--- a/transformer_lens/HookedEncoder.py
+++ b/transformer_lens/HookedEncoder.py
@@ -229,7 +229,7 @@ def from_pretrained(
             official_model_name, cfg, hf_model, **from_pretrained_kwargs
         )

-        model = cls(cfg, tokenizer, move_to_device)
+        model = cls(cfg, tokenizer, move_to_device=False)

         dtype = from_pretrained_kwargs.get("torch_dtype", None)
         if dtype is not None:
@@ -237,6 +237,9 @@ def from_pretrained(

         model.load_state_dict(state_dict, strict=False)

+        if move_to_device:
+            model.to(cfg.device)
+
         print(f"Loaded pretrained model {model_name} into HookedTransformer")

         return model
diff --git a/transformer_lens/HookedTransformer.py b/transformer_lens/HookedTransformer.py
index fb9540853..581068fc3 100644
--- a/transformer_lens/HookedTransformer.py
+++ b/transformer_lens/HookedTransformer.py
@@ -84,10 +84,6 @@ def __init__(
             )
         self.cfg = cfg

-        assert (
-            self.cfg.n_devices == 1 or move_to_device
-        ), "If n_devices > 1, must move_to_device"
-
         if tokenizer is not None:
             self.set_tokenizer(tokenizer)
         elif self.cfg.tokenizer_name is not None:
@@ -157,7 +153,7 @@ def __init__(
             # the second gets the next n_layers // n_devices blocks ... the last gets the last n_layers // n_devices
             # blocks, the final
             # normalization layer (if it exists) and the unembed layer
-            HookedTransformer.move_model_modules_to_device(self)
+            self.move_model_modules_to_device()

         # Helper variable to store a small (10K-20K) dataset of training data. Empty by default, can be loaded with
         # load_sample_training_dataset
@@ -748,22 +744,21 @@ def cpu(self):
         # Wrapper around cuda that also changes self.cfg.device
         return self.to("cpu")

-    @classmethod
-    def move_model_modules_to_device(cls, model: "HookedTransformer"):
-        model.embed.to(devices.get_device_for_block_index(0, model.cfg))
-        model.hook_embed.to(devices.get_device_for_block_index(0, model.cfg))
-        if model.cfg.positional_embedding_type != "rotary":
-            model.pos_embed.to(devices.get_device_for_block_index(0, model.cfg))
-            model.hook_pos_embed.to(devices.get_device_for_block_index(0, model.cfg))
-        if hasattr(model, "ln_final"):
-            model.ln_final.to(
-                devices.get_device_for_block_index(model.cfg.n_layers - 1, model.cfg)
+    def move_model_modules_to_device(self):
+        self.embed.to(devices.get_device_for_block_index(0, self.cfg))
+        self.hook_embed.to(devices.get_device_for_block_index(0, self.cfg))
+        if self.cfg.positional_embedding_type != "rotary":
+            self.pos_embed.to(devices.get_device_for_block_index(0, self.cfg))
+            self.hook_pos_embed.to(devices.get_device_for_block_index(0, self.cfg))
+        if hasattr(self, "ln_final"):
+            self.ln_final.to(
+                devices.get_device_for_block_index(self.cfg.n_layers - 1, self.cfg)
             )
-        model.unembed.to(
-            devices.get_device_for_block_index(model.cfg.n_layers - 1, model.cfg)
+        self.unembed.to(
+            devices.get_device_for_block_index(self.cfg.n_layers - 1, self.cfg)
         )
-        for i, block in enumerate(model.blocks):
-            block.to(devices.get_device_for_block_index(i, model.cfg))
+        for i, block in enumerate(self.blocks):
+            block.to(devices.get_device_for_block_index(i, self.cfg))

     @classmethod
     def from_pretrained(
@@ -778,7 +773,6 @@ def from_pretrained(
         hf_model=None,
         device=None,
         n_devices=1,
-        move_state_dict_to_device=True,
         tokenizer=None,
         move_to_device=True,
         **from_pretrained_kwargs,
@@ -824,9 +818,6 @@ def from_pretrained(
                 default will load to CUDA if available, else CPU.
             n_devices (int, optional): The number of devices to split the model
                 across. Defaults to 1. If greater than 1, `device` must be cuda.
-            move_state_dict_to_device (bool): Whether to move the state dict to the
-                relevant device before processing and loading in the weights.
-                Defaults to True.
             tokenizer (*optional): The tokenizer to use for the model. If not
                 provided, it is inferred from cfg.tokenizer_name or initialized to None.
                 If None, then the model cannot be passed strings, and d_vocab must be explicitly set.
@@ -882,7 +873,7 @@ def from_pretrained(
         )

         # Create the HookedTransformer object
-        model = cls(cfg, tokenizer, move_to_device)
+        model = cls(cfg, tokenizer, move_to_device=False)

         dtype = from_pretrained_kwargs.get("torch_dtype", None)
         if dtype is not None:
@@ -894,9 +885,11 @@ def from_pretrained(
             center_writing_weights=center_writing_weights,
             center_unembed=center_unembed,
             refactor_factored_attn_matrices=refactor_factored_attn_matrices,
-            move_state_dict_to_device=move_state_dict_to_device,
         )

+        if move_to_device:
+            model.move_model_modules_to_device()
+
         print(f"Loaded pretrained model {model_name} into HookedTransformer")

         return model
@@ -961,7 +954,6 @@ def load_and_process_state_dict(
         center_unembed: bool = True,
         fold_value_biases: bool = True,
         refactor_factored_attn_matrices: bool = False,
-        move_state_dict_to_device: bool = True,
     ):
         """Method to load a state dict into the model, and to apply processing to simplify it. The state dict is assumed
         to be in the HookedTransformer format.
@@ -984,39 +976,10 @@ def load_and_process_state_dict(
                 output bias to the layer, and make it easier to interpret the head's output.
             refactor_factored_attn_matrices (bool, optional): Whether to convert the factored
                 matrices (W_Q & W_K, and W_O & W_V) to be "even". Defaults to False
-            move_state_dict_to_device (bool, optional): Whether to move the state dict to the device of the model.
-                Defaults to True.
             model_name (str, optional): checks the model name for special cases of state dict loading. Only used for
                 Redwood 2L model currently
         """

-        assert (
-            self.cfg.n_devices == 1 or move_state_dict_to_device
-        ), "If n_devices > 1, move_state_dict_to_device must be True"
-
-        if move_state_dict_to_device:
-            for k, v in state_dict.items():
-                if k.startswith("embed") or k.startswith("pos_embed"):
-                    state_dict[k] = v.to(
-                        devices.get_device_for_block_index(0, self.cfg)
-                    )
-                elif k.startswith("ln_final") or k.startswith("unembed"):
-                    state_dict[k] = v.to(
-                        devices.get_device_for_block_index(
-                            self.cfg.n_layers - 1, self.cfg
-                        )
-                    )
-                elif k.startswith("blocks"):
-                    state_dict[k] = v.to(
-                        devices.get_device_for_block_index(
-                            int(k.split(".")[1]), self.cfg
-                        )
-                    )
-                else:
-                    raise KeyError(
-                        f"State Dict contains a key not in the HookedTransformer format: {k}"
-                    )
-
         state_dict = self.fill_missing_keys(state_dict)
         if fold_ln:
             if self.cfg.normalization_type not in ["LN", "LNPre"]:
@@ -1356,7 +1319,6 @@ def process_weights_(
         center_writing_weights: bool = True,
         center_unembed: bool = True,
         refactor_factored_attn_matrices: bool = False,
-        move_state_dict_to_device: bool = True,
     ):
         """
         Wrapper around load_and_process_state_dict to allow for in-place processing of the weights. This is useful if
@@ -1380,7 +1342,6 @@ def process_weights_(
             center_writing_weights=center_writing_weights,
             center_unembed=center_unembed,
             refactor_factored_attn_matrices=refactor_factored_attn_matrices,
-            move_state_dict_to_device=move_state_dict_to_device,
         )

     @torch.inference_mode()
diff --git a/transformer_lens/utils.py b/transformer_lens/utils.py
index 8c805ccc2..d1d2ffb03 100644
--- a/transformer_lens/utils.py
+++ b/transformer_lens/utils.py
@@ -55,11 +55,8 @@ def download_file_from_hf(
         **select_compatible_kwargs(kwargs, hf_hub_download),
     )

-    # Load to the CPU device if CUDA is not available
-    map_location = None if torch.cuda.is_available() else torch.device("cpu")
-
     if file_path.endswith(".pth") or force_is_torch:
-        return torch.load(file_path, map_location=map_location)
+        return torch.load(file_path, map_location="cpu")
     elif file_path.endswith(".json"):
         return json.load(open(file_path, "r"))
     else:

diff --git a/tests/acceptance/test_hooked_transformer.py b/tests/acceptance/test_hooked_transformer.py
index 85d4002b5..f708b0301 100644
--- a/tests/acceptance/test_hooked_transformer.py
+++ b/tests/acceptance/test_hooked_transformer.py
@@ -6,6 +6,7 @@
 from transformers import AutoConfig

 from transformer_lens import HookedTransformer
+from transformer_lens.components import LayerNormPre
 from transformer_lens.loading_from_pretrained import OFFICIAL_MODEL_NAMES
 from transformer_lens.utils import clear_huggingface_cache

@@ -141,6 +142,15 @@ def test_from_pretrained_dtype():
     assert model.W_K.dtype == torch.bfloat16

+def test_process_weights_inplace():
+    """Check that process_weights_ works"""
+    model = HookedTransformer.from_pretrained_no_processing("gpt2-small")
+    model.process_weights_()
+    loss = model.forward(text, return_type="loss")
+    assert (loss.item() - loss_store["gpt2-small"]) < 4e-5
+    assert isinstance(model.ln_final, LayerNormPre)
+
+
 def test_from_pretrained_revision():
     """
     Check that the from_pretrained parameter `revision` (= git version) works
diff --git a/tests/unit/test_svd_interpreter.py b/tests/unit/test_svd_interpreter.py
index 6aba89d32..0f64f7ee2 100644
--- a/tests/unit/test_svd_interpreter.py
+++ b/tests/unit/test_svd_interpreter.py
@@ -6,8 +6,7 @@

 MODEL = "solu-2l"
 VECTOR_TYPES = ["OV", "w_in", "w_out"]
-ATOL = 1e-4  # Absolute tolerance - how far does a float have to be before we consider it no longer equal?
-# ATOL is set to 1e-4 because the tensors we check on are also to 4 decimal places.
+ATOL = 2e-4  # Absolute tolerance - how far does a float have to be before we consider it no longer equal?
 model = HookedTransformer.from_pretrained(MODEL)
 unfolded_model = HookedTransformer.from_pretrained(MODEL, fold_ln=False)
 second_model = HookedTransformer.from_pretrained("solu-3l")

[Bug Report] `refactor_factored_attn_matrices` uses lots of GPU memory
**Describe the bug**
When loading models with `refactor_factored_attn_matrices=True` and `device="cuda"`, they take up a lot of space on GPU. Thanks to @callummcdougall for noticing this. See example code at bottom for snippets and statistics.

**System Info**
vast.ai 2*3090 devbox, on branch main.

**Additional context**
Doing `import gc; gc.collect(); import torch; torch.cuda.empty_cache()` after the refactor stuff would help but if someone could look into further whether we can keep the permanent refactor objects on GPU (or not) that would be great and save some more trouble! If nothing else I can add a warning if we enter the refactor function with device equal to GPU.

### Checklist

- [ :white_check_mark:  ] I have checked that there is no similar [issue](https://github.com/neelnanda-io/TransformerLens/issues) in the repo (**required**)

Example code

On my machine with 3090s, this uses 46% of GPU memory:

```
model = HookedTransformer.from_pretrained(
    "solu-10l", # "NeelNanda/SoLU_10L1280W_C4_Code"
    device="cuda",
    refactor_factored_attn_matrices=True,
)
```

I check memory with

```
_r = torch.cuda.memory_reserved(0)
_a = torch.cuda.memory_allocated(0)
_p = 100*(_r-_a)/_r
print(f"Percentage of reserved memory that is allocated: {_p:.2f}%")
```

After `import gc; gc.collect(); import torch; torch.cuda.empty_cache()` this is down to 6%

However, with this code

```
model = HookedTransformer.from_pretrained(
    "solu-10l", # "NeelNanda/SoLU_10L1280W_C4_Code"
    device="cpu",
    refactor_factored_attn_matrices=True,
)
model.to("cuda")
```

just 0.16% of my memory is used

Is `refactor_factored_attn_matrices` really the issue here? As far as I can tell, using `from_pretrained` to load models to the GPU directly always takes up much more GPU memory than first loading them to the CPU and then moving them to the GPU. When I use your snippet, setting `refactor_factored_attn_matrices=True` only increases the total memory usage by a small amount.


diff --git a/annif/project.py b/annif/project.py
index 13f470fcb..05fa7353f 100644
--- a/annif/project.py
+++ b/annif/project.py
@@ -9,7 +9,6 @@
 import annif.corpus
 import annif.suggestion
 import annif.backend
-import annif.vocab
 from annif.datadir import DatadirMixin
 from annif.exception import AnnifException, ConfigurationException, \
     NotSupportedException, NotInitializedException
@@ -155,9 +154,8 @@ def vocab(self):
             if self.vocab_spec is None:
                 raise ConfigurationException("vocab setting is missing",
                                              project_id=self.project_id)
-            self._vocab = annif.vocab.get_vocab(self.vocab_spec,
-                                                self._base_datadir,
-                                                self.language)
+            self._vocab = self.registry.get_vocab(self.vocab_spec,
+                                                  self.language)

         return self._vocab

diff --git a/annif/registry.py b/annif/registry.py
index 878ccfbdb..a56340ab0 100644
--- a/annif/registry.py
+++ b/annif/registry.py
@@ -1,35 +1,41 @@
 """Registry that keeps track of Annif projects"""

 import collections
+import re
 from flask import current_app
 import annif
 from annif.config import parse_config
 from annif.project import Access, AnnifProject
+from annif.vocab import AnnifVocabulary
+from annif.util import parse_args

 logger = annif.logger

 class AnnifRegistry:
-    """Class that keeps track of the Annif projects"""
-
-    # Note: The individual projects are stored in a shared static variable,
-    # keyed by the "registry ID" which is unique to the registry instance.
-    # This is done to make it possible to serialize AnnifRegistry instances
-    # without including the potentially huge project objects (which contain
-    # backends with large models, vocabularies with lots of concepts etc).
-    # Serialized AnnifRegistry instances can then be passed between
-    # processes when using the multiprocessing module.
+    """Class that keeps track of the Annif projects and vocabularies"""
+
+    # Note: The individual projects and vocabularies are stored in shared
+    # static variables, keyed by the "registry ID" which is unique to the
+    # registry instance. This is done to make it possible to serialize
+    # AnnifRegistry instances without including the potentially huge objects
+    # (which contain backends with large models, vocabularies with lots of
+    # concepts etc). Serialized AnnifRegistry instances can then be passed
+    # between processes when using the multiprocessing module.
     _projects = {}
+    _vocabs = {}

     def __init__(self, projects_config_path, datadir, init_projects):
         self._rid = id(self)
+        self._datadir = datadir
         self._projects[self._rid] = \
-            self._create_projects(projects_config_path, datadir)
+            self._create_projects(projects_config_path)
+        self._vocabs[self._rid] = {}
         if init_projects:
             for project in self._projects[self._rid].values():
                 project.initialize()

-    def _create_projects(self, projects_config_path, datadir):
+    def _create_projects(self, projects_config_path):
         # parse the configuration
         config = parse_config(projects_config_path)

@@ -42,7 +48,7 @@ def _create_projects(self, projects_config_path, datadir):
         for project_id in config.project_ids:
             projects[project_id] = AnnifProject(project_id,
                                                 config[project_id],
-                                                datadir,
+                                                self._datadir,
                                                 self)
         return projects

@@ -64,6 +70,23 @@ def get_project(self, project_id, min_access=Access.private):
         except KeyError:
             raise ValueError("No such project {}".format(project_id))

+    def get_vocab(self, vocab_spec, default_language):
+        """Return an AnnifVocabulary corresponding to the vocab_spec. If no
+        language information is specified, use the given default language."""
+        match = re.match(r'(\w+)(\((.*)\))?', vocab_spec)
+        if match is None:
+            raise ValueError(
+                f"Invalid vocabulary specification: {vocab_spec}")
+        vocab_id = match.group(1)
+        posargs, kwargs = parse_args(match.group(3))
+        language = posargs[0] if posargs else default_language
+        vocab_key = (vocab_id, language)
+
+        if vocab_key not in self._vocabs[self._rid]:
+            self._vocabs[self._rid][vocab_key] = AnnifVocabulary(
+                vocab_id, self._datadir, language)
+        return self._vocabs[self._rid][vocab_key]
+

 def initialize_projects(app):
     projects_config_path = app.config['PROJECTS_CONFIG_PATH']
diff --git a/annif/vocab.py b/annif/vocab.py
index da57ec6cb..9605bdf9b 100644
--- a/annif/vocab.py
+++ b/annif/vocab.py
@@ -1,28 +1,15 @@
 """Vocabulary management functionality for Annif"""

 import os.path
-import re
 import annif
 import annif.corpus
 import annif.util
 from annif.datadir import DatadirMixin
 from annif.exception import NotInitializedException
-from annif.util import parse_args

 logger = annif.logger

-def get_vocab(vocab_spec, datadir, default_language):
-    match = re.match(r'(\w+)(\((.*)\))?', vocab_spec)
-    if match is None:
-        raise ValueError(f"Invalid vocabulary specification: {vocab_spec}")
-    vocab_id = match.group(1)
-    posargs, kwargs = parse_args(match.group(3))
-    language = posargs[0] if posargs else default_language
-
-    return AnnifVocabulary(vocab_id, datadir, language)
-
-
 class AnnifVocabulary(DatadirMixin):
     """Class representing a subject vocabulary which can be used by multiple
     Annif projects."""

diff --git a/tests/test_vocab.py b/tests/test_vocab.py
index 68f0cb983..02225ce16 100644
--- a/tests/test_vocab.py
+++ b/tests/test_vocab.py
@@ -19,9 +19,9 @@ def load_dummy_vocab(tmpdir):
     return vocab

-def test_get_vocab_invalid():
+def test_get_vocab_invalid(registry):
     with pytest.raises(ValueError) as excinfo:
-        annif.vocab.get_vocab('', None, None)
+        registry.get_vocab('', None)
     assert 'Invalid vocabulary specification' in str(excinfo.value)

Share vocabulary objects between projects
Currently, each AnnifProject creates its own vocabulary (AnnifVocabulary object, which wraps a SubjectIndex) even though in reality many projects may use the same vocabulary - which is now even more likely since vocabularies are multilingual (#600). This leads to unnecessary use of RAM and also CPU when the subject index has to be recreated many times.

Instead, the vocabularies could be moved under AnnifRegistry which currently only stores projects. The vocabularies could be loaded lazily as needed using a get_vocab method, similar to the current get_vocab method in annif.vocab.
Share vocabulary objects between projects
Currently, each AnnifProject creates its own vocabulary (AnnifVocabulary object, which wraps a SubjectIndex) even though in reality many projects may use the same vocabulary - which is now even more likely since vocabularies are multilingual (#600). This leads to unnecessary use of RAM and also CPU when the subject index has to be recreated many times.

Instead, the vocabularies could be moved under AnnifRegistry which currently only stores projects. The vocabularies could be loaded lazily as needed using a get_vocab method, similar to the current get_vocab method in annif.vocab.


diff --git a/CHANGELOG.md b/CHANGELOG.md
index 80fd89a6..7410c952 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/).

 ### Changed
 - On Debian-based systems, the `python-pip` package gets uninstalled and `pip` is installed via `easy_install` instead to get the latest version.
+- Recorded audio is now streamed to AVS instead of recording the whole thing and then sending it at once.
+    - Brings huge speed improvement (response latency).
+    - This means that when your recording LED (or whatever your device equivalent) is on, data gets sent to Amazon already.
+    - Used code from @respeaker (thank you!).

 ### Fixed
 - Updated old versions of requirements in `requirements.txt`. Also fixes `ImportError: No module named cheroot.server`.
diff --git a/src/alexapi/capture.py b/src/alexapi/capture.py
index edad3e02..57e3c3c1 100644
--- a/src/alexapi/capture.py
+++ b/src/alexapi/capture.py
@@ -4,7 +4,7 @@
 import alsaaudio
 import webrtcvad

-from alexapi.exceptions import ConfigurationException
+from .exceptions import ConfigurationException

 logger = logging.getLogger(__name__)

@@ -55,62 +55,76 @@ def silence_listener(self, throwaway_frames=None, force_record=None):
 		inp.setrate(self.VAD_SAMPLERATE)
 		inp.setformat(alsaaudio.PCM_FORMAT_S16_LE)
 		inp.setperiodsize(self.VAD_PERIOD)
-		audio = b''

-		start = time.time()
-
-		do_VAD = True
-		if force_record and not force_record[1]:
-			do_VAD = False
-
-		# Buffer as long as we haven't heard enough silence or the total size is within max size
-		thresholdSilenceMet = False
-		frames = 0
-		numSilenceRuns = 0
-		silenceRun = 0
+		debug = logging.getLogger('alexapi').getEffectiveLevel() == logging.DEBUG

 		logger.debug("Start recording")

 		if self._state_callback:
 			self._state_callback()

-		if do_VAD:
-			# do not count first 10 frames when doing VAD
-			while frames < throwaway_frames:
+		def _listen():
+			start = time.time()
+
+			do_VAD = True
+			if force_record and not force_record[1]:
+				do_VAD = False
+
+			# Buffer as long as we haven't heard enough silence or the total size is within max size
+			thresholdSilenceMet = False
+			frames = 0
+			numSilenceRuns = 0
+			silenceRun = 0
+
+			if debug:
+				audio = b''
+
+			if do_VAD:
+				# do not count first 10 frames when doing VAD
+				while frames < throwaway_frames:
+					length, data = inp.read()
+					frames += 1
+					if length:
+						yield data
+
+						if debug:
+							audio += data
+
+			# now do VAD
+			while (force_record and force_record[0]()) \
+					or (do_VAD and (thresholdSilenceMet is False) and ((time.time() - start) < self.MAX_RECORDING_LENGTH)):
+
 				length, data = inp.read()
-				frames += 1
 				if length:
-					audio += data
+					yield data

-		# now do VAD
-		while (force_record and force_record[0]()) \
-				or (do_VAD and (thresholdSilenceMet is False) and ((time.time() - start) < self.MAX_RECORDING_LENGTH)):
+					if debug:
+						audio += data

-			length, data = inp.read()
-			if length:
-				audio += data
+					if do_VAD and (length == self.VAD_PERIOD):
+						isSpeech = self._vad.is_speech(data, self.VAD_SAMPLERATE)

-				if do_VAD and (length == self.VAD_PERIOD):
-					isSpeech = self._vad.is_speech(data, self.VAD_SAMPLERATE)
+						if not isSpeech:
+							silenceRun += 1
+						else:
+							silenceRun = 0
+							numSilenceRuns += 1

-					if not isSpeech:
-						silenceRun += 1
-					else:
-						silenceRun = 0
-						numSilenceRuns += 1
+				if do_VAD:
+					# only count silence runs after the first one
+					# (allow user to speak for total of max recording length if they haven't said anything yet)
+					if (numSilenceRuns != 0) and ((silenceRun * self.VAD_FRAME_MS) > self.VAD_SILENCE_TIMEOUT):
+						thresholdSilenceMet = True

-			if do_VAD:
-				# only count silence runs after the first one
-				# (allow user to speak for total of max recording length if they haven't said anything yet)
-				if (numSilenceRuns != 0) and ((silenceRun * self.VAD_FRAME_MS) > self.VAD_SILENCE_TIMEOUT):
-					thresholdSilenceMet = True
+			logger.debug("End recording")

-		logger.debug("End recording")
+			inp.close()

-		inp.close()
+			if self._state_callback:
+				self._state_callback(False)

-		if self._state_callback:
-			self._state_callback(False)
+			if debug:
+				with open(self._tmp_path + 'recording.wav', 'wb') as rf:
+					rf.write(audio)

-		with open(self._tmp_path + 'recording.wav', 'wb') as rf:
-			rf.write(audio)
+		return _listen()
diff --git a/src/main.py b/src/main.py
index 7b6ade20..a8c1cd8a 100755
--- a/src/main.py
+++ b/src/main.py
@@ -13,6 +13,7 @@
 import optparse
 import email
 import subprocess
+from future.builtins import bytes

 import yaml
 import requests
@@ -277,27 +278,33 @@ def renew(self):
 			logger.critical("AVS token: Failed to obtain a token: " + str(exp))

-def alexa_speech_recognizer():
-	# https://developer.amazon.com/public/solutions/alexa/alexa-voice-service/rest/speechrecognizer-requests
-	logger.debug("Sending Speech Request...")
-
-	platform.indicate_processing()
+# from https://github.com/respeaker/Alexa/blob/master/alexa.py
+def alexa_speech_recognizer_generate_data(audio, boundary):
+	"""
+	Generate a iterator for chunked transfer-encoding request of Alexa Voice Service
+	Args:
+		audio: raw 16 bit LSB audio data
+		boundary: boundary of multipart content
+	Returns:
+	"""
+	logger.debug('Start sending speech to Alexa Voice Service')
+	chunk = '--%s\r\n' % boundary
+	chunk += (
+		'Content-Disposition: form-data; name="request"\r\n'
+		'Content-Type: application/json; charset=UTF-8\r\n\r\n'
+	)

-	url = 'https://access-alexa-na.amazon.com/v1/avs/speechrecognizer/recognize'
-	headers = {'Authorization': 'Bearer %s' % token}
 	data = {
 		"messageHeader": {
-			"deviceContext": [
-				{
-					"name": "playbackState",
-					"namespace": "AudioPlayer",
-					"payload": {
-						"streamId": "",
-						"offsetInMilliseconds": "0",
-						"playerActivity": "IDLE"
-					}
+			"deviceContext": [{
+				"name": "playbackState",
+				"namespace": "AudioPlayer",
+				"payload": {
+					"streamId": "",
+					"offsetInMilliseconds": "0",
+					"playerActivity": "IDLE"
 				}
-			]
+			}]
 		},
 		"messageBody": {
 			"profile": "alexa-close-talk",
@@ -306,12 +313,38 @@ def alexa_speech_recognizer():
 		}
 	}

-	with open(tmp_path + 'recording.wav', 'rb') as inf:
-		files = [
-			('file', ('request', json.dumps(data), 'application/json; charset=UTF-8')),
-			('file', ('audio', inf, 'audio/L16; rate=16000; channels=1'))
-		]
-		resp = requests.post(url, headers=headers, files=files)
+	yield bytes(chunk + json.dumps(data) + '\r\n', 'utf8')
+
+	chunk = '--%s\r\n' % boundary
+	chunk += (
+		'Content-Disposition: form-data; name="audio"\r\n'
+		'Content-Type: audio/L16; rate=16000; channels=1\r\n\r\n'
+	)
+
+	yield bytes(chunk, 'utf8')
+
+	for audio_chunk in audio:
+		yield audio_chunk
+
+	yield bytes('--%s--\r\n' % boundary, 'utf8')
+	logger.debug('Finished sending speech to Alexa Voice Service')
+
+	platform.indicate_processing()
+
+
+def alexa_speech_recognizer(audio_stream):
+	# https://developer.amazon.com/public/solutions/alexa/alexa-voice-service/rest/speechrecognizer-requests
+
+	url = 'https://access-alexa-na.amazon.com/v1/avs/speechrecognizer/recognize'
+	boundary = 'this-is-a-boundary'
+	headers = {
+		'Authorization': 'Bearer %s' % token,
+		'Content-Type': 'multipart/form-data; boundary=%s' % boundary,
+		'Transfer-Encoding': 'chunked',
+	}
+
+	data = alexa_speech_recognizer_generate_data(audio_stream, boundary)
+	resp = requests.post(url, headers=headers, data=data)

 	platform.indicate_processing(False)

@@ -396,7 +429,7 @@ def process_response(response):
 		try:
 			data = bytes("Content-Type: ", 'utf-8') + bytes(response.headers['content-type'], 'utf-8') + bytes('\r\n\r\n', 'utf-8') + response.content
 			msg = email.message_from_bytes(data) # pylint: disable=no-member
-		except TypeError:
+		except AttributeError:
 			data = "Content-Type: " + response.headers['content-type'] + '\r\n\r\n' + response.content
 			msg = email.message_from_string(data)

@@ -427,10 +460,10 @@ def process_response(response):

 						player.play_speech(resources_path + 'beep.wav')
 						timeout = directive['payload']['timeoutIntervalInMillis'] / 116
-						capture.silence_listener(timeout)
+						audio_stream = capture.silence_listener(timeout)

 						# now process the response
-						alexa_speech_recognizer()
+						alexa_speech_recognizer(audio_stream)

 				elif directive['namespace'] == 'AudioPlayer':
 					if directive['name'] == 'play':
@@ -481,15 +514,10 @@ def trigger_callback(trigger):

 def trigger_process(trigger):
-	if event_commands['pre_interaction']:
-		subprocess.Popen(event_commands['pre_interaction'], shell=True, stdout=subprocess.PIPE)

 	if player.is_playing():
 		player.stop()

-	if trigger.voice_confirm:
-		player.play_speech(resources_path + 'alexayes.mp3')
-
 	# clean up the temp directory
 	if not debug:
 		for some_file in os.listdir(tmp_path):
@@ -500,12 +528,18 @@ def trigger_process(trigger):
 			except Exception as exp: # pylint: disable=broad-except
 				logger.warning(exp)

+	if event_commands['pre_interaction']:
+		subprocess.Popen(event_commands['pre_interaction'], shell=True, stdout=subprocess.PIPE)
+
 	force_record = None
 	if trigger.event_type in triggers.types_continuous:
 		force_record = (trigger.continuous_callback, trigger.event_type in triggers.types_vad)

-	capture.silence_listener(force_record=force_record)
-	alexa_speech_recognizer()
+	if trigger.voice_confirm:
+		player.play_speech(resources_path + 'alexayes.mp3')
+
+	audio_stream = capture.silence_listener(force_record=force_record)
+	alexa_speech_recognizer(audio_stream)

 	triggers.enable()

diff --git a/src/requirements.txt b/src/requirements.txt
index c073524d..531394f1 100644
--- a/src/requirements.txt
+++ b/src/requirements.txt
@@ -5,4 +5,5 @@ pyalsaaudio>=0.8.4
 webrtcvad>=2.0.10
 pyyaml
 pocketsphinx>=0.1.3
-coloredlogs
\ No newline at end of file
+coloredlogs
+future>=0.16.0
\ No newline at end of file

Streaming audio to and from AVS
While waiting for Echo push notifications for a year now, I did a small extension of https://github.com/sammachin/AlexaPi to essentially enable push by submitting pre-recorded questions to AVS when triggered by an event. It worked pretty well, but the problem I faced is that response time is slow, typically 5 or even 10 seconds. I believe the approach used in https://github.com/alexa/alexa-avs-sample-app address those performance issues by streaming the audio to AVS (and maybe also streaming the response back, I asked that question, hoping to hear back from them). The approach is explained here - https://forums.developer.amazon.com/questions/10982/avs-takes-too-much-time-to-respond.html.

Is streaming audio to and from AVS in the plan for the new evolving AlexaPi?

Sounds interesting. We're not there yet to do these _future_ things - we need to refactor the AVS code first, but we ain't that far either. Let's keep it here for when we get there.

Also, I'm not sure that streaming **from** AVS is gonna have significant performance impact, because Alexa's answers are usually very small filesize-wise. Streaming **to** Alexa is probably gonna have noticeable impact, because from my experience, upload speed is often crap.

Here is a terrific example of implementing streaming to AVS using the v1 API - https://github.com/respeaker/Alexa/blob/master/alexa.py

If anyone implemented this, that would be awesome.
I thought someone was working on the new API ?
Yep, but we might do this for the old also, since it might not be such a big change (I dunno, just guessing) and the new API integration + testing + setting it as default is gonna take some time.
@alexa-pi/team

Okay, I got this working (streaming to Amazon) with our AlexaPi - basically just copied a small portion of code from @respeaker and holy shit! If you don't have an Echo / Dot or haven't tried ReSpeaker you will not believe how fast it can get! I'll open a PR today and I'd like to release it this weekend as a new version (and bump all those planned versions by one), because this is definitely worth it. Stay tuned!
Awesome. Count me in if you need someone to test it :-)


diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py
index 86a96239bb..c161a5b955 100644
--- a/sdk/python/feast/infra/online_stores/dynamodb.py
+++ b/sdk/python/feast/infra/online_stores/dynamodb.py
@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import itertools
 import logging
 from datetime import datetime
 from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
@@ -50,10 +51,16 @@ class DynamoDBOnlineStoreConfig(FeastConfigBaseModel):
     """Online store type selector"""

     region: StrictStr
-    """ AWS Region Name """
+    """AWS Region Name"""

     table_name_template: StrictStr = "{project}.{table_name}"
-    """ DynamoDB table name template """
+    """DynamoDB table name template"""
+
+    sort_response: bool = True
+    """Whether or not to sort BatchGetItem response."""
+
+    batch_size: int = 40
+    """Number of items to retrieve in a DynamoDB BatchGetItem call."""

 class DynamoDBOnlineStore(OnlineStore):
@@ -211,26 +218,46 @@ def online_read(
         online_config = config.online_store
         assert isinstance(online_config, DynamoDBOnlineStoreConfig)
         dynamodb_resource = self._get_dynamodb_resource(online_config.region)
+        table_instance = dynamodb_resource.Table(
+            _get_table_name(online_config, config, table)
+        )

         result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = []
-        for entity_key in entity_keys:
-            table_instance = dynamodb_resource.Table(
-                _get_table_name(online_config, config, table)
-            )
-            entity_id = compute_entity_id(entity_key)
+        entity_ids = [compute_entity_id(entity_key) for entity_key in entity_keys]
+        batch_size = online_config.batch_size
+        sort_response = online_config.sort_response
+        entity_ids_iter = iter(entity_ids)
+        while True:
+            batch = list(itertools.islice(entity_ids_iter, batch_size))
+            # No more items to insert
+            if len(batch) == 0:
+                break
+            batch_entity_ids = {
+                table_instance.name: {
+                    "Keys": [{"entity_id": entity_id} for entity_id in batch]
+                }
+            }
             with tracing_span(name="remote_call"):
-                response = table_instance.get_item(Key={"entity_id": entity_id})
-            value = response.get("Item")
-
-            if value is not None:
-                res = {}
-                for feature_name, value_bin in value["values"].items():
-                    val = ValueProto()
-                    val.ParseFromString(value_bin.value)
-                    res[feature_name] = val
-                result.append((datetime.fromisoformat(value["event_ts"]), res))
+                response = dynamodb_resource.batch_get_item(
+                    RequestItems=batch_entity_ids
+                )
+            response = response.get("Responses")
+            table_responses = response.get(table_instance.name)
+            if table_responses:
+                if sort_response:
+                    table_responses = self._sort_dynamodb_response(
+                        table_responses, entity_ids
+                    )
+                for tbl_res in table_responses:
+                    res = {}
+                    for feature_name, value_bin in tbl_res["values"].items():
+                        val = ValueProto()
+                        val.ParseFromString(value_bin.value)
+                        res[feature_name] = val
+                    result.append((datetime.fromisoformat(tbl_res["event_ts"]), res))
             else:
-                result.append((None, None))
+                batch_size_nones = ((None, None),) * len(batch)
+                result.extend(batch_size_nones)
         return result

     def _get_dynamodb_client(self, region: str):
@@ -243,6 +270,20 @@ def _get_dynamodb_resource(self, region: str):
             self._dynamodb_resource = _initialize_dynamodb_resource(region)
         return self._dynamodb_resource

+    def _sort_dynamodb_response(self, responses: list, order: list):
+        """DynamoDB Batch Get Item doesn't return items in a particular order."""
+        # Assign an index to order
+        order_with_index = {value: idx for idx, value in enumerate(order)}
+        # Sort table responses by index
+        table_responses_ordered = [
+            (order_with_index[tbl_res["entity_id"]], tbl_res) for tbl_res in responses
+        ]
+        table_responses_ordered = sorted(
+            table_responses_ordered, key=lambda tup: tup[0]
+        )
+        _, table_responses_ordered = zip(*table_responses_ordered)
+        return table_responses_ordered
+

 def _initialize_dynamodb_client(region: str):
     return boto3.client("dynamodb", region_name=region)

diff --git a/sdk/python/tests/unit/online_store/test_dynamodb_online_store.py b/sdk/python/tests/unit/online_store/test_dynamodb_online_store.py
new file mode 100644
index 0000000000..0f42230ef5
--- /dev/null
+++ b/sdk/python/tests/unit/online_store/test_dynamodb_online_store.py
@@ -0,0 +1,57 @@
+from dataclasses import dataclass
+
+import pytest
+from moto import mock_dynamodb2
+
+from feast.infra.offline_stores.file import FileOfflineStoreConfig
+from feast.infra.online_stores.dynamodb import (
+    DynamoDBOnlineStore,
+    DynamoDBOnlineStoreConfig,
+)
+from feast.repo_config import RepoConfig
+from tests.utils.online_store_utils import (
+    _create_n_customer_test_samples,
+    _create_test_table,
+    _insert_data_test_table,
+)
+
+REGISTRY = "s3://test_registry/registry.db"
+PROJECT = "test_aws"
+PROVIDER = "aws"
+TABLE_NAME = "dynamodb_online_store"
+REGION = "us-west-2"
+
+
+@dataclass
+class MockFeatureView:
+    name: str
+
+
+@pytest.fixture
+def repo_config():
+    return RepoConfig(
+        registry=REGISTRY,
+        project=PROJECT,
+        provider=PROVIDER,
+        online_store=DynamoDBOnlineStoreConfig(region=REGION),
+        offline_store=FileOfflineStoreConfig(),
+    )
+
+
+@mock_dynamodb2
+@pytest.mark.parametrize("n_samples", [5, 50, 100])
+def test_online_read(repo_config, n_samples):
+    """Test DynamoDBOnlineStore online_read method."""
+    _create_test_table(PROJECT, f"{TABLE_NAME}_{n_samples}", REGION)
+    data = _create_n_customer_test_samples(n=n_samples)
+    _insert_data_test_table(data, PROJECT, f"{TABLE_NAME}_{n_samples}", REGION)
+
+    entity_keys, features = zip(*data)
+    dynamodb_store = DynamoDBOnlineStore()
+    returned_items = dynamodb_store.online_read(
+        config=repo_config,
+        table=MockFeatureView(name=f"{TABLE_NAME}_{n_samples}"),
+        entity_keys=entity_keys,
+    )
+    assert len(returned_items) == len(data)
+    assert [item[1] for item in returned_items] == list(features)
diff --git a/sdk/python/tests/utils/online_store_utils.py b/sdk/python/tests/utils/online_store_utils.py
new file mode 100644
index 0000000000..ee90c2a542
--- /dev/null
+++ b/sdk/python/tests/utils/online_store_utils.py
@@ -0,0 +1,54 @@
+from datetime import datetime
+
+import boto3
+
+from feast import utils
+from feast.infra.online_stores.helpers import compute_entity_id
+from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
+from feast.protos.feast.types.Value_pb2 import Value as ValueProto
+
+
+def _create_n_customer_test_samples(n=10):
+    return [
+        (
+            EntityKeyProto(
+                join_keys=["customer"], entity_values=[ValueProto(string_val=str(i))]
+            ),
+            {
+                "avg_orders_day": ValueProto(float_val=1.0),
+                "name": ValueProto(string_val="John"),
+                "age": ValueProto(int64_val=3),
+            },
+        )
+        for i in range(n)
+    ]
+
+
+def _create_test_table(project, tbl_name, region):
+    client = boto3.client("dynamodb", region_name=region)
+    client.create_table(
+        TableName=f"{project}.{tbl_name}",
+        KeySchema=[{"AttributeName": "entity_id", "KeyType": "HASH"}],
+        AttributeDefinitions=[{"AttributeName": "entity_id", "AttributeType": "S"}],
+        BillingMode="PAY_PER_REQUEST",
+    )
+
+
+def _delete_test_table(project, tbl_name, region):
+    client = boto3.client("dynamodb", region_name=region)
+    client.delete_table(TableName=f"{project}.{tbl_name}")
+
+
+def _insert_data_test_table(data, project, tbl_name, region):
+    dynamodb_resource = boto3.resource("dynamodb", region_name=region)
+    table_instance = dynamodb_resource.Table(f"{project}.{tbl_name}")
+    for entity_key, features in data:
+        entity_id = compute_entity_id(entity_key)
+        with table_instance.batch_writer() as batch:
+            batch.put_item(
+                Item={
+                    "entity_id": entity_id,
+                    "event_ts": str(utils.make_tzaware(datetime.utcnow())),
+                    "values": {k: v.SerializeToString() for k, v in features.items()},
+                }
+            )

DynamoDB batch retrieval does slow sequential gets (online_read)
The current DynamoDB implementation does sequential gets (https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/online_stores/dynamodb.py#L163)

## Possible Solution
A better approach is to do some multi-get operation or at least run these queries in parallel and collect the results.

cc @vlin-lgtm
@adchia, May I know what is needed to be done. I would like to contribute to this issue.
> @adchia, May I know what is needed to be done. I would like to contribute to this issue.

It appears we have the list of primary keys (entity_ids), so we can switch to [BatchGetItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchGetItem.html), instead.
^^ @Vandinimodi1595
@adchia @vlin-lgtm @Vandinimodi1595   switching to `BatchGetItem` could throw an `ValidationException` [because](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchGetItem.html)

> A single operation can retrieve up to 16 MB of data, which can contain as many as 100 items. BatchGetItem returns a partial result if the response size limit is exceeded

One alternative could be to write a custom `BatchReader` to automatically handle batch writes to DynamoDB, boto3 [BatchWriter](https://github.com/boto/boto3/blob/develop/boto3/dynamodb/table.py#L63) could be taken as example for doing this, it handles [unprocessed items](https://github.com/boto/boto3/blob/develop/boto3/dynamodb/table.py#L149).

Another alternative is using threads to read items in parallel. SageMaker SDK [Ingestion Manager](https://github.com/aws/sagemaker-python-sdk/blob/26b984a2d693f95957e8555f19fa47aa36d5d5ea/src/sagemaker/feature_store/feature_group.py#L153) could be taken as example for doing it, it works similar than the previous option calling [put_record](https://github.com/aws/sagemaker-python-sdk/blob/26b984a2d693f95957e8555f19fa47aa36d5d5ea/src/sagemaker/feature_store/feature_group.py#L223) on a [set of indexes](https://github.com/aws/sagemaker-python-sdk/blob/26b984a2d693f95957e8555f19fa47aa36d5d5ea/src/sagemaker/feature_store/feature_group.py#L345).

If you're open, I'm happy to collaborate with this.
DynamoDB's record size limit is 400KB. Since we have a list of primary keys, we know the maximum number of records we would get back, and we can assume each record is at most 400KB. We can call `BatchGetItem` in micro batches so `ValidationException` wouldn't be thrown.
@vlin-lgtm  I was also considering the case of a secondary index, this might be useful when you want to access to attributes other than pk. For example
```
user_group    location                        value
100                usa                                value_1.com
100                mexico                             value_2.com
100                brazil                             value_3.com
```
@TremaMiguel, I am not sure if secondary indices are relevant. But I don't know Feast' codebase well enough to be certain.
The code snippet in the description is to get features given a set of entity ids. Entity ids are primary keys. I don't know where Feast uses secondary indices or if it even uses it at all.

We are not looking to create a generic wrapper for the BatchQuery API for DynamoDB.

A for-loop to `BatchGet` 40 entities at a time is sufficient for this.
@vlin-lgtm @adchia apologies I've misunderstood that this issue is related to sequential calls in [online_write_batch](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/online_stores/dynamodb.py#L139), a similar issue could be raise for [online_read](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/online_stores/dynamodb.py#L185) as it iteratively process the `entity_keys` #2351

I still believe the approach mentioned above could help

> Another alternative is using threads to read items in parallel. SageMaker SDK [Ingestion Manager](https://github.com/aws/sagemaker-python-sdk/blob/26b984a2d693f95957e8555f19fa47aa36d5d5ea/src/sagemaker/feature_store/feature_group.py#L153) could be taken as example for doing it, it works similar than the previous option calling [put_record](https://github.com/aws/sagemaker-python-sdk/blob/26b984a2d693f95957e8555f19fa47aa36d5d5ea/src/sagemaker/feature_store/feature_group.py#L223) on a [set of indexes](https://github.com/aws/sagemaker-python-sdk/blob/26b984a2d693f95957e8555f19fa47aa36d5d5ea/src/sagemaker/feature_store/feature_group.py#L345).

what are your thoughts?
Thanks @TremaMiguel,

Sounds good. https://github.com/feast-dev/feast/issues/2351 appears to be a dup of this issue.

I believe doing a BatchGet or a for-loop of BatchGet is good enough. What is your use case? This is used for online; I don't think there will be a case where we will need to fetch features for a lot of entities online in bulk. Multi-threading and multi-processing can add unnecessary complexity.

In any case, thanks a lot for wanting to contribute. Please feel free to open a PR. 🙏
Agreed with @vlin-lgtm on all points! BatchGet is the largest win here and simplest.

Threads are fine too though after you first use batch gets, though I think that'll be much less of an impact. You can see how we expose multithreading in https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/online_stores/datastore.py via write_concurrency.
Thanks @adchia and @vlin-lgtm, I'll start working on this


diff --git a/CHANGES.rst b/CHANGES.rst
index 64bd28811f..134e3ff27f 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,11 @@
+.. scm-version-title:: v8.5.0
+
+- :issue:`305` via :pr:`311`: In
+  :py:class:`~cheroot.connections.ConnectionManager`,
+  process connections as they become active rather than
+  waiting for a ``tick`` event, addressing performance
+  degradation introduced in v8.1.0.
+
 .. scm-version-title:: v8.4.8

 - :issue:`317` via :pr:`337`: Fixed a regression in
diff --git a/cheroot/connections.py b/cheroot/connections.py
index 8d3bc1f1ab..c38cfd1f19 100644
--- a/cheroot/connections.py
+++ b/cheroot/connections.py
@@ -3,7 +3,6 @@
 from __future__ import absolute_import, division, print_function
 __metaclass__ = type

-import collections
 import io
 import os
 import socket
@@ -126,8 +125,10 @@ def __init__(self, server):
             server (cheroot.server.HTTPServer): web server object
                 that uses this ConnectionManager instance.
         """
+        self._serving = False
+        self._stop_requested = False
+
         self.server = server
-        self._readable_conns = collections.deque()
         self._selector = _ThreadsafeSelector()

         self._selector.register(
@@ -145,13 +146,13 @@ def put(self, conn):
         # if this conn doesn't have any more data waiting to be read,
         # register it with the selector.
         if conn.rfile.has_data():
-            self._readable_conns.append(conn)
+            self.server.process_conn(conn)
         else:
             self._selector.register(
                 conn.socket.fileno(), selectors.EVENT_READ, data=conn,
             )

-    def expire(self):
+    def _expire(self):
         """Expire least recently used connections.

         This happens if there are either too many open connections, or if the
@@ -171,48 +172,57 @@ def expire(self):
             self._selector.unregister(sock_fd)
             conn.close()

-    def get_conn(self):
-        """Return a HTTPConnection object which is ready to be handled.
+    def stop(self):
+        """Stop the selector loop in run() synchronously.
+
+        May take up to half a second.
+        """
+        self._stop_requested = True
+        while self._serving:
+            time.sleep(0.01)
+
+    def run(self, expiration_interval):
+        """Run the connections selector indefinitely.

-        A connection returned by this method should be ready for a worker
-        to handle it. If there are no connections ready, None will be
-        returned.
+        Args:
+            expiration_interval (float): Interval, in seconds, at which
+                connections will be checked for expiration.
+
+        Connections that are ready to process are submitted via
+        self.server.process_conn()

-        Any connection returned by this method will need to be `put`
-        back if it should be examined again for another request.
+        Connections submitted for processing must be `put()`
+        back if they should be examined again for another request.

-        :returns: HTTPConnection instance, or None
-        :rtype: cheroot.server.HTTPConnection
+        Can be shut down by calling `stop()`.
         """
-        # return a readable connection if any exist
-        with suppress(IndexError):
-            return self._readable_conns.popleft()
+        self._serving = True
+        last_expiration_check = time.time()

-        # Will require a select call.
-        try:
-            # The timeout value impacts performance and should be carefully
-            # chosen. Ref:
-            # github.com/cherrypy/cheroot/issues/305#issuecomment-663985165
-            active_list = self._selector.select(timeout=0.01)
-        except OSError:
-            self._remove_invalid_sockets()
-            # Wait for the next tick to occur.
-            return None
-
-        for (sock_fd, conn) in active_list:
-            if conn is self.server:
-                # New connection
-                return self._from_server_socket(self.server.socket)
+        while not self._stop_requested:
+            try:
+                active_list = self._selector.select(timeout=0.01)
+            except OSError:
+                self._remove_invalid_sockets()
+                continue

-            # unregister connection from the selector until the server
-            # has read from it and returned it via put()
-            self._selector.unregister(sock_fd)
-            self._readable_conns.append(conn)
+            for (sock_fd, conn) in active_list:
+                if conn is self.server:
+                    # New connection
+                    new_conn = self._from_server_socket(self.server.socket)
+                    self.server.process_conn(new_conn)
+                else:
+                    # unregister connection from the selector until the server
+                    # has read from it and returned it via put()
+                    self._selector.unregister(sock_fd)
+                    self.server.process_conn(conn)

-        try:
-            return self._readable_conns.popleft()
-        except IndexError:
-            return None
+            now = time.time()
+            if (now - last_expiration_check) > expiration_interval:
+                self._expire()
+                last_expiration_check = now
+
+        self._serving = False

     def _remove_invalid_sockets(self):
         """Clean up the resources of any broken connections.
@@ -332,10 +342,6 @@ def _from_server_socket(self, server_socket):  # noqa: C901  # FIXME

     def close(self):
         """Close all monitored connections."""
-        for conn in self._readable_conns:
-            conn.close()
-        self._readable_conns.clear()
-
         for (_, conn) in self._selector.connections:
             if conn is not self.server:  # server closes its own socket
                 conn.close()
@@ -345,11 +351,11 @@ def close(self):
     def _num_connections(self):
         """Return the current number of connections.

-        Includes any in the readable list or registered with the selector,
+        Includes all conns registered with the selector,
         minus one for the server socket, which is always registered
         with the selector.
         """
-        return len(self._readable_conns) + len(self._selector) - 1
+        return len(self._selector) - 1

     @property
     def can_add_keepalive_connection(self):
diff --git a/cheroot/server.py b/cheroot/server.py
index a5c31b2e65..fb3bc421a2 100644
--- a/cheroot/server.py
+++ b/cheroot/server.py
@@ -7,12 +7,13 @@

     server = HTTPServer(...)
     server.start()
-    ->  while True:
-            tick()
-            # This blocks until a request comes in:
-            child = socket.accept()
-            conn = HTTPConnection(child, ...)
-            server.requests.put(conn)
+    ->  serve()
+        while ready:
+            _connections.run()
+                while not stop_requested:
+                    child = socket.accept()  # blocks until a request comes in
+                    conn = HTTPConnection(child, ...)
+                    server.process_conn(conn)  # adds conn to threadpool

 Worker threads are kept in a pool and poll the Queue, popping off and then
 handling each connection in turn. Each connection can consist of an arbitrary
@@ -1523,6 +1524,11 @@ class HTTPServer:
     timeout = 10
     """The timeout in seconds for accepted connections (default 10)."""

+    expiration_interval = 0.5
+    """The interval, in seconds, at which the server checks for
+    expired connections (default 0.5).
+    """
+
     version = 'Cheroot/{version!s}'.format(version=__version__)
     """A version string for the HTTPServer."""

@@ -1592,7 +1598,6 @@ def __init__(
         self.requests = threadpool.ThreadPool(
             self, min=minthreads or 1, max=maxthreads,
         )
-        self.serving = False

         if not server_name:
             server_name = self.version
@@ -1797,21 +1802,17 @@ def prepare(self):  # noqa: C901  # FIXME

     def serve(self):
         """Serve requests, after invoking :func:`prepare()`."""
-        self.serving = True
         while self.ready:
             try:
-                self.tick()
+                self._connections.run(self.expiration_interval)
             except (KeyboardInterrupt, SystemExit):
-                self.serving = False
                 raise
             except Exception:
                 self.error_log(
-                    'Error in HTTPServer.tick', level=logging.ERROR,
+                    'Error in HTTPServer.serve', level=logging.ERROR,
                     traceback=True,
                 )

-        self.serving = False
-
     def start(self):
         """Run the server forever.

@@ -2039,17 +2040,13 @@ def resolve_real_bind_addr(socket_):

         return bind_addr

-    def tick(self):
-        """Accept a new connection and put it on the Queue."""
-        conn = self._connections.get_conn()
-        if conn:
-            try:
-                self.requests.put(conn)
-            except queue.Full:
-                # Just drop the conn. TODO: write 503 back?
-                conn.close()
-
-        self._connections.expire()
+    def process_conn(self, conn):
+        """Process an incoming HTTPConnection."""
+        try:
+            self.requests.put(conn)
+        except queue.Full:
+            # Just drop the conn. TODO: write 503 back?
+            conn.close()

     @property
     def interrupt(self):
@@ -2067,14 +2064,15 @@ def interrupt(self, interrupt):

     def stop(self):  # noqa: C901  # FIXME
         """Gracefully shutdown a server that is serving forever."""
+        if not self.ready:
+            return  # already stopped
+
         self.ready = False
         if self._start_time is not None:
             self._run_time += (time.time() - self._start_time)
         self._start_time = None

-        # ensure serve is no longer accessing socket, connections
-        while self.serving:
-            time.sleep(0.1)
+        self._connections.stop()

         sock = getattr(self, 'socket', None)
         if sock:
diff --git a/docs/conf.py b/docs/conf.py
index 175b9cfe66..f1da660b65 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -108,6 +108,7 @@
     ('py:class', '_pyio.BufferedWriter'),
     ('py:class', '_pyio.BufferedReader'),
     ('py:class', 'unittest.case.TestCase'),
+    ('py:meth', 'cheroot.connections.ConnectionManager.get_conn'),
 ]

 # Ref:

diff --git a/cheroot/test/test_conn.py b/cheroot/test/test_conn.py
index b8660cfb15..c33fb2b44b 100644
--- a/cheroot/test/test_conn.py
+++ b/cheroot/test/test_conn.py
@@ -1145,6 +1145,6 @@ def test_invalid_selected_connection(test_client, monkeypatch):
     # trigger the internal errors
     faux_get_map.sabotage_conn = faux_select.request_served = True
     # give time to make sure the error gets handled
-    time.sleep(0.2)
+    time.sleep(test_client.server_instance.expiration_interval * 2)
     assert faux_select.os_error_triggered
     assert faux_get_map.conn_closed
diff --git a/cheroot/test/test_server.py b/cheroot/test/test_server.py
index f71ce61dd5..52b8b1b978 100644
--- a/cheroot/test/test_server.py
+++ b/cheroot/test/test_server.py
@@ -10,6 +10,7 @@
 import socket
 import tempfile
 import threading
+import time
 import uuid

 import pytest
@@ -125,20 +126,20 @@ def test_serving_is_false_and_stop_returns_after_ctrlc():

     httpserver.prepare()

-    # Simulate a Ctrl-C on the first call to `get_conn`.
+    # Simulate a Ctrl-C on the first call to `run`.
     def raise_keyboard_interrupt(*args):
         raise KeyboardInterrupt()

-    httpserver._connections.get_conn = raise_keyboard_interrupt
+    httpserver._connections.run = raise_keyboard_interrupt

     serve_thread = threading.Thread(target=httpserver.serve)
     serve_thread.start()

     # The thread should exit right away due to the interrupt.
-    serve_thread.join(0.5)
+    serve_thread.join(httpserver.expiration_interval * 2)
     assert not serve_thread.is_alive()

-    assert not httpserver.serving
+    assert not httpserver._connections._serving
     httpserver.stop()

@@ -295,11 +296,12 @@ def test_high_number_of_file_descriptors(resource_limit):
     httpserver = HTTPServer(
         bind_addr=(ANY_INTERFACE_IPV4, EPHEMERAL_PORT), gateway=Gateway,
     )
-    httpserver.prepare()

     try:
         # This will trigger a crash if select() is used in the implementation
-        httpserver.tick()
+        with httpserver._run_in_thread():
+            # allow server to run long enough to invoke select()
+            time.sleep(1.0)
     except:  # noqa: E722
         raise  # only needed for `else` to work
     else:

[cheroot==8.1.0 regression] Significant performance drop
❓ **I'm submitting a ...**
- [X] 🐞 bug report
- [ ] 🐣 feature request
- [ ] ❓ question about the decisions made in the repository

🐞 **Describe the bug. What is the current behavior?**

This more a note than a bug report:
I run some benchmark against my project ([WsgiDAV](https://github.com/mar10/wsgidav)] and found that there is a drastical drop in performance between cheroot versions 8.0.0 and 8.1.0.

The test uses a homegrown tool ([stressor](https://github.com/mar10/stressor)) that runs a test script ([see here](https://github.com/mar10/wsgidav/blob/master/tests/stressor/test_rw.yaml)):

- 10 users in parallel
  - GET a common file
  - PUT another file (one per user)
  - GET that file
- Running for 30 seconds

Benchmarks show 11k activities per 30 seconds until v8.0.
Starting with cheroot 8.1 this drops to 2.9k activities per 30 sec.

I also tried with gevent as backend, and it delivered 12.4k activities.

Please note that these results may be due to poor implementation on my tools, but all I changed was WSGI server behind WsgiDAV, so I thought I let you know.

💡 **To Reproduce**

See https://github.com/mar10/wsgidav/blob/master/tests/stressor/results.md

If you are interested, I can provide more details.

Cheers
Martin

Hey @mar10, thanks for the heads up! v8.1.0 introduced a number of functional regressions that are got some fixes in v8.3.0 and v8.4.0 so please also test with these. It was a result of refactoring the connection management to decouple keep-alive TCP connections from HTTP requests. So I'd say that you should also check whether your tool reuses the connections or creates new ones of each HTTP request.
Hi Sviatoslav,

I already [tested with 8.3.1 and 8.4.0](https://github.com/mar10/wsgidav/blob/master/tests/stressor/results.md) with same results.

[stressor](https://github.com/mar10/stressor/) uses the requests library with sessions, which *should* [support keep-alive](https://requests.readthedocs.io/en/master/user/advanced/#keep-alive), but I can double check.

However, even if it did *not* reuse connections, would the refactoring explain that cheroot now slows down to 25% (for my specific benchmark)?
I did not change anything in the scenario (test-tool, WSGI app), except for swapping out the WSGI servers.
Hm... I think you use the same session for all the requests meaning that they all are processed sequentially because they come one after another over the wire in the same TCP connection. HTTP/1.1 doesn't have built-in parallelism, unlike HTTP/2 that uses streams within the same connection.

Try checking that it does use just one connection (using tcpdump/wireshark, I guess). Also try make stressor creat a number of sessions and load-balance the queries. Plus try stress-testing with some other tool like ab/wrk2/locust.
Stressor spawns one thread for every virtual user.
Each 'user' is holding its own `requests/urllib3` session, so every user should re-use its connection.
,
I modified the test. This time a sequence is only one GET request to a single static file:

### Cheroot 8.0

Executed 10,163 activities in 10,163 sequences, using 1 parallel sessions.  => 338.8 requests per sec/user
Executed 11,584 activities in 11,584 sequences, using 2 parallel sessions.  => 193.1 requests per sec/user
Executed 14,545 activities in 14,545 sequences, using 10 parallel sessions. =>  48.5 requests per sec/user
=> 0.003 ..  0.02 seconds per request

### Cheroot 8.4

Executed 292 activities in 292 sequences, using 1 parallel sessions.      => 9.7 requests per sec/user
Executed 582 activities in 582 sequences, using 2 parallel sessions.      => 9.7 requests per sec/user
Executed 2,943 activities in 2,933 sequences, using 10 parallel sessions. => 9.8 requests per sec/user
=> 0.1 seconds per request

It may sound silly, but it looks like someone introduced a `sleep(0.1)` in v8.1+ ;-)
Weird. This clearly needs profiling. What are the thread num settings in your setup?
@mar10 mind record a flame graph using https://github.com/benfred/py-spy?
numthreads is set to 50.

py-spy looks interesting, I will try it out if I find time.

However, I searched for "0.1" and found that `connections.py` contains this code:
```py
        try:
            for fno in socket_dict:
                self._selector.register(fno, selectors.EVENT_READ)
            rlist = [
                key.fd for key, _event
                in self._selector.select(timeout=0.1)
            ]
        except OSError:
          ...
```

I patched the timeout value and ran the benchmarks again

timeout=0.1 (original)
Executed 2,907 activities in 2,907 sequences, using 10 parallel sessions.

timeout=0.05
Executed 5,766 activities in 5,766 sequences, using 10 parallel sessions.

timeout=0.02
Executed 12,199 activities in 12,199 sequences, using 10 parallel sessions.

timeout=0.01
Executed 13,933 activities in 13,933 sequences, using 10 parallel sessions.

Smaller values don't seem to improve the performance.
Extracting what @liamstask posted in the PR:
>it looks to me that this could be related to the fact that, currently in `server.tick()`, only one request can be added per tick even if several are available to be processed in `ConnectionManager.connections`.
>
>in this case, since `ConnectionManager` does not register connections previously marked as read-ready with the selector, it's possible that _none_ of the remaining connections in the selector are active, meaning the server is forced to wait for the full timeout duration each time `tick()` is called.
>
>fundamentally, the value of this timeout should not really be particularly sensitive, since the selector should wake up once connections are ready to be processed, regardless of the timeout.

_(Originally posted by @liamstask in https://github.com/cherrypy/cheroot/pull/308#issuecomment-664040481)_
8.4.2 fixed it for me, thanks

8.4.1:
Run time 0:00:30.109496, net: 4:59.91 min.
Executed 2,920 activities in 2,920 sequences, using 10 parallel sessions.

8.4.2:
Run time 0:00:30.099942, net: 4:57.40 min.
Executed 14,131 activities in 14,131 sequences, using 10 parallel sessions.


diff --git a/src/transformers/models/whisper/feature_extraction_whisper.py b/src/transformers/models/whisper/feature_extraction_whisper.py
index 42104c329346..508e85b91ffd 100644
--- a/src/transformers/models/whisper/feature_extraction_whisper.py
+++ b/src/transformers/models/whisper/feature_extraction_whisper.py
@@ -94,41 +94,63 @@ def __init__(
             mel_scale="slaney",
         )

-    def _np_extract_fbank_features(self, waveform: np.array) -> np.ndarray:
+    def _np_extract_fbank_features(self, waveform_batch: np.array, device: str) -> np.ndarray:
         """
         Compute the log-mel spectrogram of the provided audio, gives similar results to Whisper's original torch
         implementation with 1e-5 tolerance.
         """
-        log_spec = spectrogram(
-            waveform,
-            window_function(self.n_fft, "hann"),
-            frame_length=self.n_fft,
-            hop_length=self.hop_length,
-            power=2.0,
-            mel_filters=self.mel_filters,
-            log_mel="log10",
-        )
-        log_spec = log_spec[:, :-1]
-        log_spec = np.maximum(log_spec, log_spec.max() - 8.0)
-        log_spec = (log_spec + 4.0) / 4.0
-        return log_spec
-
-    def _torch_extract_fbank_features(self, waveform: np.array) -> np.ndarray:
+        if device != "cpu":
+            raise ValueError(
+                f"Got device `{device}` for feature extraction, but feature extraction on CUDA accelerator "
+                "devices requires torch, which is not installed. Either set `device='cpu'`, or "
+                "install torch according to the official instructions: https://pytorch.org/get-started/locally/"
+            )
+        log_spec_batch = []
+        for waveform in waveform_batch:
+            log_spec = spectrogram(
+                waveform,
+                window_function(self.n_fft, "hann"),
+                frame_length=self.n_fft,
+                hop_length=self.hop_length,
+                power=2.0,
+                mel_filters=self.mel_filters,
+                log_mel="log10",
+            )
+            log_spec = log_spec[:, :-1]
+            log_spec = np.maximum(log_spec, log_spec.max() - 8.0)
+            log_spec = (log_spec + 4.0) / 4.0
+            log_spec_batch.append(log_spec)
+        log_spec_batch = np.array(log_spec_batch)
+        return log_spec_batch
+
+    def _torch_extract_fbank_features(self, waveform: np.array, device: str = "cpu") -> np.ndarray:
         """
-        Compute the log-mel spectrogram of the provided audio using the PyTorch STFT implementation.
+        Compute the log-mel spectrogram of the audio using PyTorch's GPU-accelerated STFT implementation with batching,
+        yielding results similar to cpu computing with 1e-5 tolerance.
         """
         waveform = torch.from_numpy(waveform).type(torch.float32)

         window = torch.hann_window(self.n_fft)
+        if device != "cpu":
+            waveform = waveform.to(device)
+            window = window.to(device)
         stft = torch.stft(waveform, self.n_fft, self.hop_length, window=window, return_complex=True)
         magnitudes = stft[..., :-1].abs() ** 2

         mel_filters = torch.from_numpy(self.mel_filters).type(torch.float32)
+        if device != "cpu":
+            mel_filters = mel_filters.to(device)
         mel_spec = mel_filters.T @ magnitudes

         log_spec = torch.clamp(mel_spec, min=1e-10).log10()
-        log_spec = torch.maximum(log_spec, log_spec.max() - 8.0)
+        if waveform.dim() == 2:
+            max_val = log_spec.max(dim=2, keepdim=True)[0].max(dim=1, keepdim=True)[0]
+            log_spec = torch.maximum(log_spec, max_val - 8.0)
+        else:
+            log_spec = torch.maximum(log_spec, log_spec.max() - 8.0)
         log_spec = (log_spec + 4.0) / 4.0
+        if device != "cpu":
+            log_spec = log_spec.detach().cpu()
         return log_spec.numpy()

     @staticmethod
@@ -165,6 +187,7 @@ def __call__(
         max_length: Optional[int] = None,
         sampling_rate: Optional[int] = None,
         do_normalize: Optional[bool] = None,
+        device: Optional[str] = "cpu",
         **kwargs,
     ) -> BatchFeature:
         """
@@ -211,6 +234,9 @@ def __call__(
             do_normalize (`bool`, *optional*, defaults to `False`):
                 Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly
                 improve the performance of the model.
+            device (`str`, *optional*, defaults to `'cpu'`):
+                Specifies the device for computation of the log-mel spectrogram of audio signals in the
+                `_torch_extract_fbank_features` method. (e.g., "cpu", "cuda")
         """

         if sampling_rate is not None:
@@ -272,7 +298,7 @@ def __call__(
         extract_fbank_features = (
             self._torch_extract_fbank_features if is_torch_available() else self._np_extract_fbank_features
         )
-        input_features = [extract_fbank_features(waveform) for waveform in input_features[0]]
+        input_features = extract_fbank_features(input_features[0], device)

         if isinstance(input_features[0], List):
             padded_inputs["input_features"] = [np.asarray(feature, dtype=np.float32) for feature in input_features]

diff --git a/tests/models/whisper/test_feature_extraction_whisper.py b/tests/models/whisper/test_feature_extraction_whisper.py
index 77c7a9be3de6..8b1e25927e33 100644
--- a/tests/models/whisper/test_feature_extraction_whisper.py
+++ b/tests/models/whisper/test_feature_extraction_whisper.py
@@ -24,7 +24,7 @@
 from datasets import load_dataset

 from transformers import WhisperFeatureExtractor
-from transformers.testing_utils import check_json_file_has_correct_format, require_torch
+from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_torch_gpu
 from transformers.utils.import_utils import is_torch_available

 from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
@@ -207,6 +207,7 @@ def _load_datasamples(self, num_samples):

         return [x["array"] for x in speech_samples]

+    @require_torch_gpu
     @require_torch
     def test_torch_integration(self):
         # fmt: off
@@ -223,6 +224,7 @@ def test_torch_integration(self):
         input_speech = self._load_datasamples(1)
         feature_extractor = WhisperFeatureExtractor()
         input_features = feature_extractor(input_speech, return_tensors="pt").input_features
+
         self.assertEqual(input_features.shape, (1, 80, 3000))
         self.assertTrue(torch.allclose(input_features[0, 0, :30], EXPECTED_INPUT_FEATURES, atol=1e-4))

@@ -253,3 +255,37 @@ def test_zero_mean_unit_variance_normalization_trunc_np_longest(self):

         self.assertTrue(np.all(np.mean(audio) < 1e-3))
         self.assertTrue(np.all(np.abs(np.var(audio) - 1) < 1e-3))
+
+    @require_torch_gpu
+    @require_torch
+    def test_torch_integration_batch(self):
+        # fmt: off
+        EXPECTED_INPUT_FEATURES = torch.tensor(
+            [
+                [
+                    0.1193, -0.0946, -0.1098, -0.0196, 0.0225, -0.0690, -0.1736, 0.0951,
+                    0.0971, -0.0817, -0.0702, 0.0162, 0.0260, 0.0017, -0.0192, -0.1678,
+                    0.0709, -0.1867, -0.0655, -0.0274, -0.0234, -0.1884, -0.0516, -0.0554,
+                    -0.0274, -0.1425, -0.1423, 0.0837, 0.0377, -0.0854
+                ],
+                [
+                    -0.4696, -0.0751, 0.0276, -0.0312, -0.0540, -0.0383, 0.1295, 0.0568,
+                    -0.2071, -0.0548, 0.0389, -0.0316, -0.2346, -0.1068, -0.0322, 0.0475,
+                    -0.1709, -0.0041, 0.0872, 0.0537, 0.0075, -0.0392, 0.0371, 0.0189,
+                    -0.1522, -0.0270, 0.0744, 0.0738, -0.0245, -0.0667
+                ],
+                [
+                    -0.2337, -0.0060, -0.0063, -0.2353, -0.0431, 0.1102, -0.1492, -0.0292,
+                     0.0787, -0.0608, 0.0143, 0.0582, 0.0072, 0.0101, -0.0444, -0.1701,
+                     -0.0064, -0.0027, -0.0826, -0.0730, -0.0099, -0.0762, -0.0170, 0.0446,
+                     -0.1153, 0.0960, -0.0361, 0.0652, 0.1207, 0.0277
+                ]
+            ]
+        )
+        # fmt: on
+
+        input_speech = self._load_datasamples(3)
+        feature_extractor = WhisperFeatureExtractor()
+        input_features = feature_extractor(input_speech, return_tensors="pt").input_features
+        self.assertEqual(input_features.shape, (3, 80, 3000))
+        self.assertTrue(torch.allclose(input_features[:, 0, :30], EXPECTED_INPUT_FEATURES, atol=1e-4))

[Whisper] Computing features on CPU slows down WhisperFeatureExtractor
### Feature request

In the WhisperFeatureExtractor class, to reduce the time it takes to compute audio features, an option should be added to compute the features on GPU in batch mode.

### Motivation

Currently, the Whisper Feature Extractor processes audio features on the CPU one by one for each batch, which slows down the process. Additionally, it doesn't utilize GPU operations despite using torch functions.

We ( + @vaibhavagg303 ) modified the code to enable batch feature extraction on CPU, cutting the average time for batches of 8 from 1.5 seconds to 0.25 seconds — depending on available CPU resources. Furthermore, we transferred all variables — mel filters and hanning window — in the function __torch_extract_fbank_features_  to GPU, further reducing **the average time to 0.02 seconds — a significant reduction from 1.5 seconds.**

Whisper Models, already with high latency, experience even greater latency due to processing input features on the CPU without batching. By doing batch-wise inference on GPU, the total inference time for Whisper-Small is reduced significantly, **bringing it down by whopping 25%.**

@sanchit-gandhi

The code used to measure the time taken is as follows.
```
from transformers import WhisperProcessor, WhisperForConditionalGeneration
feature_extractor = WhisperFeatureExtractor.from_pretrained("openai/whisper-small")
import librosa
import time

processor = WhisperProcessor.from_pretrained("openai/whisper-small", language='english')
model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-small")
model.config.forced_decoder_ids = None
model = model.to('cuda')

total_time = 0
batch_size= 8

for i in range(0, len(df), batch_size):
  sample = df[i:i+ batch_size]
  files = sample['file'].tolist()
  sample_array = []
  for file in files:
    data, sample_rate = librosa.load(file, sr = 16000)
    sample_array.append(data)

 st = time.time()
  input_features = feature_extractor(sample_array, sampling_rate=16000).input_features
  input_features = torch.tensor(input_features).to('cuda')

  with torch.no_grad():
    predicted_ids = model.generate(input_features, language='english')
  transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)

  total_time += time.time() - st
```

### Your contribution

This is the PR by @vaibhavagg303.
https://github.com/huggingface/transformers/pull/29900


diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py
index 755c9b98a..c082552eb 100644
--- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py
+++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py
@@ -182,8 +182,6 @@ def update_transition_domain_models(self):
                 # STEP 5: Parse creation and expiration data
                 updated_transition_domain = self.parse_creation_expiration_data(domain_name, transition_domain)

-                # Check if the instance has changed before saving
-                updated_transition_domain.save()
                 updated_transition_domains.append(updated_transition_domain)
                 logger.info(f"{TerminalColors.OKCYAN}" f"Successfully updated {domain_name}" f"{TerminalColors.ENDC}")

@@ -199,6 +197,28 @@ def update_transition_domain_models(self):
                 )
                 failed_transition_domains.append(domain_name)

+        updated_fields = [
+            "organization_name",
+            "organization_type",
+            "federal_type",
+            "federal_agency",
+            "first_name",
+            "middle_name",
+            "last_name",
+            "email",
+            "phone",
+            "epp_creation_date",
+            "epp_expiration_date",
+        ]
+
+        batch_size = 1000
+        # Create a Paginator object. Bulk_update on the full dataset
+        # is too memory intensive for our current app config, so we can chunk this data instead.
+        paginator = Paginator(updated_transition_domains, batch_size)
+        for page_num in paginator.page_range:
+            page = paginator.page(page_num)
+            TransitionDomain.objects.bulk_update(page.object_list, updated_fields)
+
         failed_count = len(failed_transition_domains)
         if failed_count == 0:
             if self.debug:
diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py
index 1a581a4ec..27a8364bc 100644
--- a/src/registrar/models/domain.py
+++ b/src/registrar/models/domain.py
@@ -910,10 +910,15 @@ def renew(self):
         raise NotImplementedError()

     def get_security_email(self):
-        logger.info("get_security_email-> getting the contact ")
-        secContact = self.security_contact
-        if secContact is not None:
-            return secContact.email
+        logger.info("get_security_email-> getting the contact")
+
+        security = PublicContact.ContactTypeChoices.SECURITY
+        security_contact = self.generic_contact_getter(security)
+
+        # If we get a valid value for security_contact, pull its email
+        # Otherwise, just return nothing
+        if security_contact is not None and isinstance(security_contact, PublicContact):
+            return security_contact.email
         else:
             return None

@@ -1121,7 +1126,6 @@ def generic_contact_getter(self, contact_type_choice: PublicContact.ContactTypeC
         If you wanted to setup getter logic for Security, you would call:
         cache_contact_helper(PublicContact.ContactTypeChoices.SECURITY),
         or cache_contact_helper("security").
-
         """
         # registrant_contact(s) are an edge case. They exist on
         # the "registrant" property as opposed to contacts.
diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py
index 3924c03c4..f9608f553 100644
--- a/src/registrar/utility/csv_export.py
+++ b/src/registrar/utility/csv_export.py
@@ -3,10 +3,13 @@
 from datetime import datetime
 from registrar.models.domain import Domain
 from registrar.models.domain_information import DomainInformation
-from registrar.models.public_contact import PublicContact
-from django.db.models import Value
-from django.db.models.functions import Coalesce
 from django.utils import timezone
+from django.core.paginator import Paginator
+from django.db.models import F, Value, CharField
+from django.db.models.functions import Concat, Coalesce
+
+from registrar.models.public_contact import PublicContact
+

 logger = logging.getLogger(__name__)

@@ -20,50 +23,77 @@ def write_header(writer, columns):

 def get_domain_infos(filter_condition, sort_fields):
-    domain_infos = DomainInformation.objects.filter(**filter_condition).order_by(*sort_fields)
-    return domain_infos
+    domain_infos = (
+        DomainInformation.objects.select_related("domain", "authorizing_official")
+        .filter(**filter_condition)
+        .order_by(*sort_fields)
+    )

+    # Do a mass concat of the first and last name fields for authorizing_official.
+    # The old operation was computationally heavy for some reason, so if we precompute
+    # this here, it is vastly more efficient.
+    domain_infos_cleaned = domain_infos.annotate(
+        ao=Concat(
+            Coalesce(F("authorizing_official__first_name"), Value("")),
+            Value(" "),
+            Coalesce(F("authorizing_official__last_name"), Value("")),
+            output_field=CharField(),
+        )
+    )
+    return domain_infos_cleaned

-def write_row(writer, columns, domain_info):
-    security_contacts = domain_info.domain.contacts.filter(contact_type=PublicContact.ContactTypeChoices.SECURITY)

-    # For linter
-    ao = " "
-    if domain_info.authorizing_official:
-        first_name = domain_info.authorizing_official.first_name or ""
-        last_name = domain_info.authorizing_official.last_name or ""
-        ao = first_name + " " + last_name
+def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None):
+    """Given a set of columns, generate a new row from cleaned column data"""

-    security_email = " "
-    if security_contacts:
-        security_email = security_contacts[0].email
+    # Domain should never be none when parsing this information
+    if domain_info.domain is None:
+        raise ValueError("Domain is none")
+
+    domain = domain_info.domain  # type: ignore
+
+    # Grab the security email from a preset dictionary.
+    # If nothing exists in the dictionary, grab from .contacts.
+    if security_emails_dict is not None and domain.name in security_emails_dict:
+        _email = security_emails_dict.get(domain.name)
+        security_email = _email if _email is not None else " "
+    else:
+        # If the dictionary doesn't contain that data, lets filter for it manually.
+        # This is a last resort as this is a more expensive operation.
+        security_contacts = domain.contacts.filter(contact_type=PublicContact.ContactTypeChoices.SECURITY)
+        _email = security_contacts[0].email if security_contacts else None
+        security_email = _email if _email is not None else " "

-    invalid_emails = {"registrar@dotgov.gov", "dotgov@cisa.dhs.gov"}
     # These are default emails that should not be displayed in the csv report
-    if security_email is not None and security_email.lower() in invalid_emails:
+    invalid_emails = {"registrar@dotgov.gov", "dotgov@cisa.dhs.gov"}
+    if security_email.lower() in invalid_emails:
         security_email = "(blank)"

+    if domain_info.federal_type:
+        domain_type = f"{domain_info.get_organization_type_display()} - {domain_info.get_federal_type_display()}"
+    else:
+        domain_type = domain_info.get_organization_type_display()
+
     # create a dictionary of fields which can be included in output
     FIELDS = {
-        "Domain name": domain_info.domain.name,
-        "Domain type": domain_info.get_organization_type_display() + " - " + domain_info.get_federal_type_display()
-        if domain_info.federal_type
-        else domain_info.get_organization_type_display(),
+        "Domain name": domain.name,
+        "Domain type": domain_type,
         "Agency": domain_info.federal_agency,
         "Organization name": domain_info.organization_name,
         "City": domain_info.city,
         "State": domain_info.state_territory,
-        "AO": ao,
+        "AO": domain_info.ao,  # type: ignore
         "AO email": domain_info.authorizing_official.email if domain_info.authorizing_official else " ",
         "Security contact email": security_email,
-        "Status": domain_info.domain.get_state_display(),
-        "Expiration date": domain_info.domain.expiration_date,
-        "Created at": domain_info.domain.created_at,
-        "First ready": domain_info.domain.first_ready,
-        "Deleted": domain_info.domain.deleted,
+        "Status": domain.get_state_display(),
+        "Expiration date": domain.expiration_date,
+        "Created at": domain.created_at,
+        "First ready": domain.first_ready,
+        "Deleted": domain.deleted,
     }

-    writer.writerow([FIELDS.get(column, "") for column in columns])
+    row = [FIELDS.get(column, "") for column in columns]
+    return row

 def write_body(
@@ -78,13 +108,41 @@ def write_body(
     """

     # Get the domainInfos
-    domain_infos = get_domain_infos(filter_condition, sort_fields)
-
-    all_domain_infos = list(domain_infos)
+    all_domain_infos = get_domain_infos(filter_condition, sort_fields)
+
+    # Store all security emails to avoid epp calls or excessive filters
+    sec_contact_ids = all_domain_infos.values_list("domain__security_contact_registry_id", flat=True)
+    security_emails_dict = {}
+    public_contacts = (
+        PublicContact.objects.only("email", "domain__name")
+        .select_related("domain")
+        .filter(registry_id__in=sec_contact_ids)
+    )

-    # Write rows to CSV
-    for domain_info in all_domain_infos:
-        write_row(writer, columns, domain_info)
+    # Populate a dictionary of domain names and their security contacts
+    for contact in public_contacts:
+        domain: Domain = contact.domain
+        if domain is not None and domain.name not in security_emails_dict:
+            security_emails_dict[domain.name] = contact.email
+        else:
+            logger.warning("csv_export -> Domain was none for PublicContact")
+
+    # Reduce the memory overhead when performing the write operation
+    paginator = Paginator(all_domain_infos, 1000)
+    for page_num in paginator.page_range:
+        page = paginator.page(page_num)
+        rows = []
+        for domain_info in page.object_list:
+            try:
+                row = parse_row(columns, domain_info, security_emails_dict)
+                rows.append(row)
+            except ValueError:
+                # This should not happen. If it does, just skip this row.
+                # It indicates that DomainInformation.domain is None.
+                logger.error("csv_export -> Error when parsing row, domain was None")
+                continue
+
+        writer.writerows(rows)

 def export_data_type_to_csv(csv_file):

Exporting any domain report (full metadata, current-full, -federal) takes a long time, 401s, and only generates on the second attempt
### Current Behavior

See title.

### Expected Behavior

Generating one of these reports shouldn't take multiple minutes, throw an error, or require a retry.

### Steps to Reproduce

1. In the domains table on manage.get.gov/admin, click on "Export all domain metadata", "Export current-full.csv". or "Export current-federal.csv"
2. Wait ~3 mins for the browser to throw ERR_TOO_MANY_REDIRECTS, redirect to a 401 page
3. Navigate back to the page to re-request the report, which should arrive in roughly a minute

### Environment

Stable

### Additional Context

* The [current-full.csv I was able to retrieve](https://docs.google.com/spreadsheets/d/1STyDpPj3kx1kJFJ9gQgmLe9OUamNNs1h7c_n71Q3x_s/edit#gid=787357430) the second time. Much of the data is off.
* See [Slack thread](https://cisa-corp.slack.com/archives/C05BGB4L5NF/p1702609627618349).

### Issue Links

_No response_


diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py
index 541462c3b0..de7c6dabfb 100644
--- a/agenta-backend/agenta_backend/models/converters.py
+++ b/agenta-backend/agenta_backend/models/converters.py
@@ -359,6 +359,7 @@ async def environment_db_to_output(
         deployed_variant_name = None
         revision = None

+    await environment_db.fetch_link(AppEnvironmentDB.deployed_app_variant_revision)
     environment_output = EnvironmentOutput(
         name=environment_db.name,
         app_id=str(environment_db.app.id),
diff --git a/agenta-backend/agenta_backend/routers/configs_router.py b/agenta-backend/agenta_backend/routers/configs_router.py
index 0670ded7f1..f05531d828 100644
--- a/agenta-backend/agenta_backend/routers/configs_router.py
+++ b/agenta-backend/agenta_backend/routers/configs_router.py
@@ -93,7 +93,7 @@ async def get_config(
     try:
         base_db = await db_manager.fetch_base_by_id(base_id)

-        # detemine whether the user has access to the base
+        # determine whether the user has access to the base
         if isCloudEE():
             has_permission = await check_action_access(
                 user_uid=request.state.user_id,
@@ -111,15 +111,16 @@ async def get_config(
         # in case environment_name is provided, find the variant deployed
         if environment_name:
             app_environments = await db_manager.list_environments(
-                app_id=str(base_db.app.id)
+                app_id=str(base_db.app.ref.id)
+            )
+            found_variant_revision = next(
+                (
+                    app_environment.deployed_app_variant_revision
+                    for app_environment in app_environments
+                    if app_environment.name == environment_name
+                ),
+                None,
             )
-            found_variant = None
-            for app_environment in app_environments:
-                if app_environment.name == environment_name:
-                    found_variant_revision = (
-                        app_environment.deployed_app_variant_revision
-                    )
-                    break
             if not found_variant_revision:
                 raise HTTPException(
                     status_code=400,
@@ -134,11 +135,14 @@ async def get_config(
             config = found_variant_revision.config
         elif config_name:
             variants_db = await db_manager.list_variants_for_base(base_db)
-            found_variant = None
-            for variant_db in variants_db:
-                if variant_db.config_name == config_name:
-                    found_variant = variant_db
-                    break
+            found_variant = next(
+                (
+                    variant_db
+                    for variant_db in variants_db
+                    if variant_db.config_name == config_name
+                ),
+                None,
+            )
             if not found_variant:
                 raise HTTPException(
                     status_code=400,
diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py
index ac1234cfa9..acf8c358cb 100644
--- a/agenta-backend/agenta_backend/services/db_manager.py
+++ b/agenta-backend/agenta_backend/services/db_manager.py
@@ -257,14 +257,15 @@ async def fetch_base_by_id(base_id: str) -> Optional[VariantBaseDB]:
     Returns:
         VariantBaseDB: The fetched base, or None if no base was found.
     """
+
     if base_id is None:
         raise Exception("No base_id provided")
-    base = await VariantBaseDB.find_one(
-        VariantBaseDB.id == ObjectId(base_id), fetch_links=True
-    )
-    if base is None:
+
+    base = await VariantBaseDB.find_one(VariantBaseDB.id == ObjectId(base_id))
+    if not base:
         logger.error("Base not found")
-        return False
+        return None
+
     return base

@@ -651,9 +652,7 @@ async def list_variants_for_base(base: VariantBaseDB) -> List[AppVariantDB]:
     """
     assert base is not None, "base cannot be None"
     app_variants_db = (
-        await AppVariantDB.find(
-            AppVariantDB.base.id == ObjectId(base.id), fetch_links=True
-        )
+        await AppVariantDB.find(AppVariantDB.base.id == ObjectId(base.id))
         .sort("variant_name")
         .to_list()
     )
@@ -864,6 +863,10 @@ async def add_variant_from_base_and_config(
         config_name=new_config_name,
         parameters=parameters,
     )
+
+    # Prefetch image in base_db
+    await base_db.fetch_link(VariantBaseDB.image)
+
     db_app_variant = AppVariantDB(
         app=previous_app_variant_db.app,
         variant_name=new_variant_name,

[AGE-164] [Improvement] Speed up the endpoint /get_config
The call to retrieve the configuration takes 1.20 seconds, indicating that 'get_config' is highly unoptimised.

Here's the current process:

* It fetches the base_id.
* It checks permissions.
* It lists the environment for the app_id, which is obtained from the base_id.
* It searches one by one until it finds the correct name.
* It gets the variant_revision_id from the found environment.
* It returns the configuration from that variant.

This process involves a significant number of queries!

From [SyncLinear.com](https://synclinear.com) | [AGE-164](https://linear.app/agenta/issue/AGE-164/[improvement]-speed-up-the-endpoint-get-config)


diff --git a/changelog.d/8744.bugfix b/changelog.d/8744.bugfix
new file mode 100644
index 000000000000..f8f9630bd6af
--- /dev/null
+++ b/changelog.d/8744.bugfix
@@ -0,0 +1,1 @@
+Fix a bug where appservices may be sent an excessive amount of read receipts and presence. Broke in v1.22.0.
diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index 9fc844422862..5c6458eb52fc 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -226,7 +226,7 @@ async def _notify_interested_services_ephemeral(
         new_token: Optional[int],
         users: Collection[Union[str, UserID]],
     ):
-        logger.info("Checking interested services for %s" % (stream_key))
+        logger.debug("Checking interested services for %s" % (stream_key))
         with Measure(self.clock, "notify_interested_services_ephemeral"):
             for service in services:
                 # Only handle typing if we have the latest token
diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py
index c242c409cf26..153cbae7b912 100644
--- a/synapse/handlers/receipts.py
+++ b/synapse/handlers/receipts.py
@@ -158,7 +158,8 @@ async def get_new_events_as(
         if from_key == to_key:
             return [], to_key

-        # We first need to fetch all new receipts
+        # Fetch all read receipts for all rooms, up to a limit of 100. This is ordered
+        # by most recent.
         rooms_to_events = await self.store.get_linearized_receipts_for_all_rooms(
             from_key=from_key, to_key=to_key
         )
diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py
index ca7917c9895b..1e7949a3233a 100644
--- a/synapse/storage/databases/main/receipts.py
+++ b/synapse/storage/databases/main/receipts.py
@@ -278,7 +278,8 @@ def f(txn):
     async def get_linearized_receipts_for_all_rooms(
         self, to_key: int, from_key: Optional[int] = None
     ) -> Dict[str, JsonDict]:
-        """Get receipts for all rooms between two stream_ids.
+        """Get receipts for all rooms between two stream_ids, up
+        to a limit of the latest 100 read receipts.

         Args:
             to_key: Max stream id to fetch receipts upto.
@@ -294,12 +295,16 @@ def f(txn):
                 sql = """
                     SELECT * FROM receipts_linearized WHERE
                     stream_id > ? AND stream_id <= ?
+                    ORDER BY stream_id DESC
+                    LIMIT 100
                 """
                 txn.execute(sql, [from_key, to_key])
             else:
                 sql = """
                     SELECT * FROM receipts_linearized WHERE
                     stream_id <= ?
+                    ORDER BY stream_id DESC
+                    LIMIT 100
                 """

                 txn.execute(sql, [to_key])

When enabling EDU support for appservices, synapse will start from stream position 0 when fetching presence/receipts
Which is very expensive if you were say, matrix.org. If the appservice hasn't got a stream position in the database, we should go from a reasonable point in time.