diff --git a/src/prefect/_internal/concurrency/calls.py b/src/prefect/_internal/concurrency/calls.py
index 13e9e9bbfc61..6ca4dc45567e 100644
--- a/src/prefect/_internal/concurrency/calls.py
+++ b/src/prefect/_internal/concurrency/calls.py
@@ -10,6 +10,7 @@
 import dataclasses
 import inspect
 import threading
+import weakref
 from concurrent.futures._base import (
     CANCELLED,
     CANCELLED_AND_NOTIFIED,
@@ -32,21 +33,30 @@
 T = TypeVar("T")
 P = ParamSpec("P")

-
-# Tracks the current call being executed
-current_call: contextvars.ContextVar["Call"] = contextvars.ContextVar("current_call")
+# Tracks the current call being executed. Note that storing the `Call`
+# object for an async call directly in the contextvar appears to create a
+# memory leak, despite the fact that we `reset` when leaving the context
+# that sets this contextvar. A weakref avoids the leak and works because a)
+# we already have strong references to the `Call` objects in other places
+# and b) this is used for performance optimizations where we have fallback
+# behavior if this weakref is garbage collected. A fix for issue #10952.
+current_call: contextvars.ContextVar["weakref.ref[Call]"] = contextvars.ContextVar(
+    "current_call"
+)

 # Create a strong reference to tasks to prevent destruction during execution errors
 _ASYNC_TASK_REFS = set()

 def get_current_call() -> Optional["Call"]:
-    return current_call.get(None)
+    call_ref = current_call.get(None)
+    if call_ref:
+        return call_ref()

 @contextlib.contextmanager
 def set_current_call(call: "Call"):
-    token = current_call.set(call)
+    token = current_call.set(weakref.ref(call))
     try:
         yield
     finally:
@@ -181,6 +191,29 @@ def result(self, timeout=None):
             # Break a reference cycle with the exception in self._exception
             self = None

+    def _invoke_callbacks(self):
+        """
+        Invoke our done callbacks and clean up cancel scopes and cancel
+        callbacks. Fixes a memory leak that hung on to Call objects,
+        preventing garbage collection of Futures.
+
+        A fix for #10952.
+        """
+        if self._done_callbacks:
+            done_callbacks = self._done_callbacks[:]
+            self._done_callbacks[:] = []
+
+            for callback in done_callbacks:
+                try:
+                    callback(self)
+                except Exception:
+                    logger.exception("exception calling callback for %r", self)
+
+        self._cancel_callbacks = []
+        if self._cancel_scope:
+            self._cancel_scope._callbacks = []
+            self._cancel_scope = None
+

 @dataclasses.dataclass
 class Call(Generic[T]):
diff --git a/src/prefect/_internal/concurrency/cancellation.py b/src/prefect/_internal/concurrency/cancellation.py
index 8dcb4e6519de..25c2c2b5ad9b 100644
--- a/src/prefect/_internal/concurrency/cancellation.py
+++ b/src/prefect/_internal/concurrency/cancellation.py
@@ -270,6 +270,11 @@ def __exit__(self, exc_type, exc_val, exc_tb):
             # Mark as cancelled
             self.cancel(throw=False)

+        # TODO: Can we also delete the scope?
+        # We have to exit this scope to prevent leaking memory. A fix for
+        # issue #10952.
+        self._anyio_scope.__exit__(exc_type, exc_val, exc_tb)
+
         super().__exit__(exc_type, exc_val, exc_tb)

         if self.cancelled() and exc_type is not CancelledError:

Task input persisted leading to memory not being released (same for output).
### First check

- [X] I added a descriptive title to this issue.
- [X] I used the GitHub search to find a similar request and didn't find it.
- [X] I searched the Prefect documentation for this feature.

### Prefect Version

2.x

### Describe the current behavior

Hi,

I really like the design of Prefect. To me Prefect is close to perfect. There is just the issue that **passing large sets of data to and from tasks quickly eats up all memory**. This eaten up memory cannot be released, or at least, I could not find any option to do so. Actually, being able to pass data into tasks is one of the advertised advantages of Prefect over Airflow:

```Python
from prefect import flow, task
import os
import psutil
import sys
import gc

@task(persist_result=False, cache_result_in_memory=False)  # <----- Remove this line, and the memory is released -----
def my_task(df):
    pass

@flow
def my_sub_flow_1():

    print(f"Memory before: {psutil.Process(os.getpid()).memory_info()[0] / float(1024*1024)}MiB")
    df = bytearray(1024*1024*1024)  # 1024MiB of memory

    my_task(df)

    print(f'{sys.getrefcount(df)} references to df')
    del df  # release memory
    gc.collect()  # garbage collection not needed, just be certain
    print(f"Memory after: {psutil.Process(os.getpid()).memory_info()[0] / float(1024*1024)}MiB")

my_sub_flow_1()
```

output is:

```
Memory before: 163MiB
4 references to df
Memory after: 1187MiB
```

then removing the `@task` decorator, all memory gets released by the `del` and everything works fine. Of course, this is a minimal example. In real live, data engineers like me want to run flows with lots of sub_flows passing large pandas data frames around without running out of memory.

### Describe the proposed behavior

It would be a really great enhancement if we could run sub_flows like the one below with large dataframes without running out of memory quickly once having more subflows or tasks.

```Python
@flow(empty_cache_after_flowrun=True):  # <----- something like this here
def my_sub_flow_1():
    df=task_load_data1()
    df=task_modify_data(df)
    task_upload_data(df, release_)

    prefect.release_task_cache  # <----- or something like this here

    df=task_load_data2()
    df=task_modify_data(df)
    task_upload_data(df)
```

### Example Use

see above. Currently, all big objects passed from one task to another need to go via disk or any other store to be able to release memory. This somehow defeats the great idea of Prefect.

### Additional context

Debugging showed that the class `class UnpersistedResult(BaseResult)` persists the _outputs_ of task '_forever_' by design in `self._cache`. Surprisingly _inputs_ are also persisted somewhere. I spend quite some time debugging all the options, reading documentation and googling. In fact, the documentation is to me misleading as it suggests that things can be non-persistent. persist_result=False, cache_result_in_memory=False are not of help either, especially not to the problem of input persistence.

I also tried to modify the source to release the `self._cache` after it has been read to contribute, but sadly, I did not find a non destructive way to solve the issue to help also others that might be affected. Actually, I am wondering that I am the only one having this issue.

Ps.: Other options I found are not using tasks at all so that no data is passed around. I also experimented with using a wrapper classes to encapsulate each object passed around and then deleting the large object from the class (leaving the class itself in memory), but that somehow defeats the simplicity of prefect.

Hi @sibbiii, thanks for the well written issue and investigation. I can reproduce the behavior with your MRE.

I think it is unlikely that the references are maintained as part of results as you're not returning anything from your function.

I think this likely has to do with the dependency tracking or as your said _input persistence_. `prefect` keeps tracks of inputs as a way to track/resolve dependencies between tasks. We already provide some mechanisms like `quote()` to opt out of dependency tracking. This is possibly a bug where a reference is maintained when it shouldn't be OR a an enhancement. Either way it seems like it makes sense to at least have an opt out.
Hi,

> I think it is unlikely that the references are maintained as part of results as you're not returning anything from your function.

You are right, I checked, they are not maintained in UnpersistedResult.

Thereby invested some time to create another minimal example how to fill up memory by calling a task with some input data.
When executing the code below, memory usage (e.g. open the task manager) increases with time.

<img src="https://github.com/PrefectHQ/prefect/assets/115080722/89e99b95-a7c2-4bf0-bbc4-604335c2aa5d" width="400">

```Python
from prefect import flow, task
import gc

@task  # <----- Remove this line, and the memory is released -----
def my_task(some_large_object):
    pass

@flow
def my_sub_flow():
    my_task(bytearray(1024 * 1024 * 300))  # 300MiB

for i in range(100):
    my_sub_flow()
    gc.collect()
```
I did not find out yet why the memory cannot be released if the task decorator is added.
Interestingly, calling a task in a loop (similar to the example below) does not fill up the memory.

For the output of a task, filling up the memory is easy to reproduce:
```Python
from prefect import flow, task

@task  # <----- Remove this line, and the memory is released -----
def my_task():
    return bytearray(1024*1024*300)  # 300MiB of memory

@flow
def my_sub_flow():
    for i in range(100):
        my_task()

my_sub_flow()
```
Here a reference to the data is kept in self._cache of UnpersistedResult.

Unfortunately, not being able to release the memory for inputs and outputs does result in memory issues when running flows with many tasks or lots of data being transferred to and from tasks. I hope there are other ways to keeping track of dependencies between tasks rather than "storing" all input data by keeping the reference, same for the outputs.
I think these issues highly impact long running flows/tasks. Where memory is slowly crippling in and got OutOfMemory.

My use case is http calls to a website with hundreds of thousands of sitemaps.

I've been investigating this memory leak and one issue I've found with the parameter case is that after garbage collection, the chain of referrers looks like:
* the task parameters
  * the `kwargs` dict on a `Call` object to `get_task_call_return_value`
    * an intermediate `hamt` object (a `frozenset` somewhere?)
      * a `contextvars.Context` object
        * the `kwargs` dict on another another `Call` for `QueueService.drain` of the `APILogWorker` service

I think what's happening here is that even when the flow and tasks have both finished running, there are lingering references to them in the service that streams logs back to the Prefect API, so I'm going to try to figure out how we can break those references.

One possible solution is that the `Call` object can probably drop its `args`/`kwargs` immediately after it has finished running, as they shouldn't be used after that.  Need to experiment some more....
For a task that returns a large value, it's also the `Call`, but this time it's the `Call.future` that's holding a long-term reference to the result of the call.  It's a bit trickier to know when we're done with the `future` of a `Call` since a caller may theoretically call for the `result` multiple times.  Looking to see if there's a way we can just drop the `Call` object itself when we're done with it...
In the case of task return values, at the end of a flow run there are three references:

1: a `Call` object's `.future` <-- this is something we may be able to address
2: an `UnpersistedResult` in the `SequentialTaskRunner._results` cache of `State` results
3: another copy of an `UnpersistedResult` referenced by the list of `State` results in the  `Future._result` of the future for `create_then_begin_flow_run`

That first one seems like something we may be able to address, but the second/third start to interfere with how our result processing works, which might need to be opened as a separate issue.  I do believe I can address the _parameter_ issue by breaking the `Call`'s reference to the task's `args`/`kwargs`, which should help with this issue about task _inputs_
Hi @sibbiii, I just merged a fix that will be in the next release, but if you could give it a try on `prefect@main`, that would be amazing!  I was able to address the _parameter_ issue as you described in the original writeup, but the _result_ issue is a bit more invasive of an issue.  I'll bring this back to the team to talk about how we might improve the performance/memory use around result caching between flow runs.
> Hi @sibbiii, I just merged a fix that will be in the next release, but if you could give it a try on prefect@main, that would be amazing!

Hi @chrisguidry,

I imported main head (2.14.4+12.g6d7acc78b9) and reran the example at the beginning of this issue. The output is still the same, that is, memory is _not_ released if the `@task` decorator is in place and is released if the decorator is removed.

Sebastian
Ah yes, @sibbiii, that is likely to still be the case during the flow run.  My fix corrected an issue where memory wasn't freed _between_ flow runs, but it wouldn't have addressed freeing memory _during_ a flow run.  Check out this example for what my change fixed:

```python
import gc
import os
import sys

import psutil
from prefect import flow, task

@task(
    persist_result=False, cache_result_in_memory=False
)  # <----- Remove this line, and the memory is released -----
def my_task(df):
    pass

@flow
def my_sub_flow_1():
    print(
        f"Memory before task: {psutil.Process(os.getpid()).memory_info()[0] / float(1024*1024)}MiB"
    )
    df = bytearray(1024 * 1024 * 1024)  # 1024MiB of memory

    my_task(df)

    print(f"{sys.getrefcount(df)} references to df")
    del df  # release memory
    gc.collect()  # garbage collection not needed, just be certain
    print(
        f"Memory after task: {psutil.Process(os.getpid()).memory_info()[0] / float(1024*1024)}MiB"
    )

if __name__ == "__main__":
    print(
        f"Memory before flow: {psutil.Process(os.getpid()).memory_info()[0] / float(1024*1024)}MiB"
    )

    my_sub_flow_1()

    gc.collect()  # garbage collection not needed, just be certain
    print(
        f"Memory after flow: {psutil.Process(os.getpid()).memory_info()[0] / float(1024*1024)}MiB"
    )
```

Before the fix:

```
Memory before flow: 80.4453125MiB
...
Memory before task: 136.1328125MiB
09:26:15.670 | INFO    | Flow run 'spiked-ostrich' - Created task run 'my_task-0' for task 'my_task'
09:26:15.671 | INFO    | Flow run 'spiked-ostrich' - Executing 'my_task-0' immediately...
09:26:16.353 | INFO    | Task run 'my_task-0' - Finished in state Completed()
4 references to df
Memory after task: 1162.3828125MiB
09:26:16.566 | INFO    | Flow run 'spiked-ostrich' - Finished in state Completed('All states completed.')
Memory after flow: 1163.3203125MiB

```

After the fix:

```
Memory before flow: 84.30078125MiB
...
Memory before task: 99.55078125MiB
09:21:21.617 | INFO    | Flow run 'uncovered-trogon' - Created task run 'my_task-0' for task 'my_task'
09:21:21.618 | INFO    | Flow run 'uncovered-trogon' - Executing 'my_task-0' immediately...
09:21:22.285 | INFO    | Task run 'my_task-0' - Finished in state Completed()
3 references to df
Memory after task: 1165.80078125MiB <---- what you're observing
09:21:22.531 | INFO    | Flow run 'uncovered-trogon' - Finished in state Completed('All states completed.')
Memory after flow: 141.91796875MiB <---- what I was able to fix in the first pass
```

I'm going to re-open this for further investigation.
Hi @chrisguidry,

I checked: you are right. It works when the flow is finished.

By the way, now that the input issue is fixed,
there is a _hack_ to release memory using something like this:

```
class SelfDestruct:

    class AlreadyDestructedException(Exception):
        pass

    class NoObjectStored:
        pass  # Singleton for defining that there is no object in the stored

    def __init__(self, obj):
        self._obj_store = obj

    def get_destruct(self, destruct=True):
        if self._obj_store is self.NoObjectStored:
            raise self.AlreadyDestructedException

        result = self._obj_store
        if destruct:
            self._obj_store = self.NoObjectStored
        return result
```

and then

```
@task()
def my_task(df):
    return myHelper.SelfDestruct(df)

@flow
def my_sub_flow():
    df = ...
    result = my_task(df).get_destruct()
```

which actually works fine, that is, the memory is released. It just does not look like "clean code". Still I could actually add this logic to the `@task` and `@flow` decorators ...

Other than this, I had no smart Idea so far how to fix the issue.
We've been discussing internally how we may need to take a fresh look at how results are handled across the board, with memory efficiency in mind.  Thanks for your help diagnosing these, we'll keep working on this and keep you posted.
Hi, I am running on the newest prefect version (2.14.15). I am still experiencing the same memory leak even after upgrading the prefect version. I turned storage persistence and memory cache off for the task. Here is a reproducible piece of code that causes the memory leak.
```python
@flow
async def periodic_batch_flow(until: Optional[str] = None):
    for i in range(3):
        print(f"Memory before flow: {psutil.Process(os.getpid()).memory_info()[0] / float(1024 * 1024)}MiB")
        await subflow()
        print(f"Memory after flow: {psutil.Process(os.getpid()).memory_info()[0] / float(1024 * 1024)}MiB")

@task(cache_result_in_memory=False, persist_result=False)
async def some_heavy_task(df):
    # some processing and writing to db
    await asyncio.sleep(10)
    return

@flow
async def subflow():
    print(
        f"Memory before task: {psutil.Process(os.getpid()).memory_info()[0] / float(1024*1024)}MiB"
    )
    df = pd.DataFrame(np.random.randint(0, 100, size=(int(2.5e7), 4)), columns=list("ABCD"))
    await some_heavy_task(df)

    del df  # doesn't actually do anything
    gc.collect()  # doesn't do anything
    print(
        f"Memory after task: {psutil.Process(os.getpid()).memory_info()[0] / float(1024*1024)}MiB"
    )
    return
```

And the output after one iteration of the loop:
"""
14:29:43.012 | INFO    | prefect.engine - Created flow run 'pastel-perch' for flow 'periodic-batch-flow'
Memory before flow: 287.171875MiB
14:29:43.151 | INFO    | Flow run 'pastel-perch' - Created subflow run 'radiant-loon' for flow 'subflow'
Memory before task: 287.9375MiB
14:29:43.843 | INFO    | Flow run 'radiant-loon' - Created task run 'some_heavy_task-0' for task 'some_heavy_task'
14:29:43.844 | INFO    | Flow run 'radiant-loon' - Executing 'some_heavy_task-0' immediately...
14:29:53.909 | INFO    | Task run 'some_heavy_task-0' - Finished in state Completed()
Memory after task: 1051.625MiB
14:29:54.012 | INFO    | Flow run 'radiant-loon' - Finished in state Completed('All states completed.')
Memory after flow: 1051.890625MiB
"""

As you can see, the memory doesn't go down even after the subflow finishes. If I continue with the loop to more iterations, the memory just grows and grows.
If I try to run @chrisguidry 's flow and task, I see the same results that he posted after the bugfix got merged, the only difference I see between his flow and mine are that mine is async.
Just wanted to drop a note here to say we are actively investigating this. We've narrowed down the area of interest to our use of an `anyio` cancellation scope using a slightly simplified version of the example code in Itay's comment.

Here's the code I'm using with all of my debugging annotations:

```python
import asyncio
import gc
import os
import time
from typing import Optional

import numpy as np
import objgraph
import pandas as pd
import prefect._internal.concurrency.calls
import psutil
from anyio._backends._asyncio import _task_states
from prefect import flow, task

@task(cache_result_in_memory=False, persist_result=False)
async def some_heavy_task(df):
    # some processing and writing to db
    await asyncio.sleep(10)
    return

@task(cache_result_in_memory=False, persist_result=False)
def some_heavy_task_sync(df):
    # some processing and writing to db
    time.sleep(10)
    return

@flow
async def periodic_batch_flow_tasks(until: Optional[str] = None):
    for i in range(4):
        print(
            "task refs before iteration ",
            prefect._internal.concurrency.calls._ASYNC_TASK_REFS,
        )
        print(
            f"Memory before task: {psutil.Process(os.getpid()).memory_info()[0] / float(1024 * 1024)}MiB"
        )
        df = pd.DataFrame(
            np.random.randint(0, 100, size=(int(2.5e7), 4)), columns=list("ABCD")
        )
        some_heavy_task_sync(df)
        del df  # doesn't actually do anything
        gc.collect()  # doesn't do anything

        print(
            f"Memory after task: {psutil.Process(os.getpid()).memory_info()[0] / float(1024 * 1024)}MiB"
        )
        print(
            f"task refs after iteration {i}",
            prefect._internal.concurrency.calls._ASYNC_TASK_REFS,
        )
        print(
            f"task states after iteration {i}", {k: v for k, v in _task_states.items()}
        )

    await asyncio.sleep(5)

    print(
        "task refs at end of loop ",
        prefect._internal.concurrency.calls._ASYNC_TASK_REFS,
    )
    print("task states at end of loop ", {k: v for k, v in _task_states.items()})

    print("Printing dataframes within flow run")
    for i, obj in enumerate(objgraph.by_type("DataFrame")):
        objgraph.show_chain(
            objgraph.find_backref_chain(obj, objgraph.is_proper_module),
            filename=f"chain{i}.png",
        )

if __name__ == "__main__":
    asyncio.run(periodic_batch_flow_tasks())

```

Profiling with `memray` shows that if an async task is used, the Python process keeps references to all `DataFrames` that the flow creates. If a sync task is used, only one reference exists when we exit the loop.

Here's the profiling command I'm using:

```
$ memray run -o output.bin memory_leak.py
$ memray flamegraph output.bin
$ open memray-flamegraph-output.html
```


diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml
index a66fed7a5..07d2b502a 100644
--- a/tdrs-backend/docker-compose.yml
+++ b/tdrs-backend/docker-compose.yml
@@ -101,6 +101,15 @@ services:
     command: --config /usr/share/grafana/conf/custom.ini
     depends_on:
       - grafana-pg
+
+  alertmanager:
+    restart: always
+    image: prom/alertmanager:v0.27.0
+    ports:
+      - 9093:9093
+    volumes:
+      - ./plg/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
+    command: --config.file=/etc/alertmanager/alertmanager.yml --storage.path=/alertmanager --log.level=debug --web.external-url=http://localhost:3000/alerts --web.route-prefix=/alerts --cluster.listen-address=""

   prometheus:
     restart: always
@@ -109,12 +118,14 @@ services:
       - 9090:9090
     volumes:
       - ./plg/prometheus/prometheus.local.yml:/etc/prometheus/prometheus.yml
-      - ./plg/prometheus/django_rules.yml:/etc/prometheus/django_rules.yml
+      - ./plg/prometheus/django-rules.yml:/etc/prometheus/django-rules.yml
+      - ./plg/prometheus/alerts.local.yml:/etc/prometheus/alerts.yml
       - prometheus_data:/prometheus
     depends_on:
       - web
       - celery-exporter
       - postgres-exporter
+      - alertmanager

   promtail:
     restart: always
diff --git a/tdrs-backend/plg/alertmanager/alertmanager.yml b/tdrs-backend/plg/alertmanager/alertmanager.yml
new file mode 100644
index 000000000..9414062ae
--- /dev/null
+++ b/tdrs-backend/plg/alertmanager/alertmanager.yml
@@ -0,0 +1,71 @@
+global:
+  # The smarthost and SMTP sender used for mail notifications.
+  smtp_smarthost: 'smtp.sendgrid.net:587'
+  smtp_from: 'no-reply@tanfdata.acf.hhs.gov'
+  smtp_auth_username: 'apikey'
+  smtp_auth_password: '{{ sendgrid_api_key }}'
+
+# The directory from which notification templates are read.
+templates:
+  - '/etc/alertmanager/template/*.tmpl'
+
+# The root route on which each incoming alert enters.
+route:
+  # The labels by which incoming alerts are grouped together. For example,
+  # multiple alerts coming in for cluster=A and alertname=LatencyHigh would
+  # be batched into a single group.
+  group_by: ['alertname', 'env', 'service']
+
+  # When a new group of alerts is created by an incoming alert, wait at
+  # least 'group_wait' to send the initial notification.
+  # This way ensures that you get multiple alerts for the same group that start
+  # firing shortly after another are batched together on the first
+  # notification.
+  group_wait: 30s
+
+  # When the first notification was sent, wait 'group_interval' to send a batch
+  # of new alerts that started firing for that group.
+  group_interval: 5m
+
+  # If an alert has successfully been sent, wait 'repeat_interval' to
+  # resend them.
+  repeat_interval: 5m
+
+  # A default receiver
+  receiver: admin-team-emails
+
+  # All the above attributes are inherited by all child routes and can
+  # overwritten on each.
+
+  # The child route trees.
+  routes:
+    # This routes performs a regular expression match on alert labels to
+    # catch alerts that are related to a list of services.
+    - matchers:
+        - alertname=~"UpTime"
+      receiver: dev-team-emails
+      group_wait: 30m
+
+# Inhibition rules allow to mute a set of alerts given that another alert is
+# firing.
+# We use this to mute any warning-level notifications if the same alert is
+# already critical.
+inhibit_rules:
+  - source_matchers: [severity="CRITICAL"]
+    target_matchers: [severity="WARNING"]
+    # Apply inhibition if the alertname is the same.
+    # CAUTION:
+    #   If all label names listed in `equal` are missing
+    #   from both the source and target alerts,
+    #   the inhibition rule will apply!
+    equal: [alertname, env, service]
+
+
+receivers:
+  - name: 'admin-team-emails'
+    email_configs:
+      - to: '{{ admin_team_emails }}'
+
+  - name: 'dev-team-emails'
+    email_configs:
+      - to: '{{ dev_team_emails }}'
diff --git a/tdrs-backend/plg/alertmanager/manifest.yml b/tdrs-backend/plg/alertmanager/manifest.yml
new file mode 100644
index 000000000..80067f717
--- /dev/null
+++ b/tdrs-backend/plg/alertmanager/manifest.yml
@@ -0,0 +1,10 @@
+version: 1
+applications:
+  - name: alertmanager
+    memory: 512M
+    disk_quota: 1G
+    instances: 1
+    command: |
+      mkdir /tmp
+    buildpacks:
+    - https://github.com/cloudfoundry/binary-buildpack
diff --git a/tdrs-backend/plg/grafana/dashboards/logs_dashboard.json b/tdrs-backend/plg/grafana/dashboards/logs_dashboard.json
index 6843e5a85..5b34ecf3a 100644
--- a/tdrs-backend/plg/grafana/dashboards/logs_dashboard.json
+++ b/tdrs-backend/plg/grafana/dashboards/logs_dashboard.json
@@ -19,9 +19,95 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 4,
   "links": [],
   "panels": [
+    {
+      "datasource": {
+        "default": true,
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "dark-red",
+                "value": 80
+              },
+              {
+                "color": "light-red",
+                "value": 85
+              },
+              {
+                "color": "#EAB839",
+                "value": 90
+              },
+              {
+                "color": "semi-dark-green",
+                "value": 95
+              },
+              {
+                "color": "dark-green",
+                "value": 100
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 14,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "100 * avg_over_time(up{job=~\"$app\"}[$__range])",
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "App Uptime",
+      "type": "stat"
+    },
     {
       "datasource": {
         "type": "loki",
@@ -31,7 +117,7 @@
         "h": 28,
         "w": 24,
         "x": 0,
-        "y": 0
+        "y": 14
       },
       "id": 1,
       "options": {
@@ -56,7 +142,7 @@
           "refId": "A"
         }
       ],
-      "title": "Logs",
+      "title": "Job Logs",
       "type": "logs"
     }
   ],
@@ -71,7 +157,7 @@
     "list": [
       {
         "current": {
-          "selected": false,
+          "selected": true,
           "text": "All",
           "value": "$__all"
         },
@@ -98,11 +184,35 @@
         "skipUrlSync": false,
         "sort": 0,
         "type": "query"
+      },
+      {
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "definition": "query_result(up)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "App",
+        "multi": false,
+        "name": "app",
+        "options": [],
+        "query": {
+          "qryType": 3,
+          "query": "query_result(up)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "/.*job=\"([^\"]+).*/",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
       }
     ]
   },
   "time": {
-    "from": "now-3h",
+    "from": "now-24h",
     "to": "now"
   },
   "timepicker": {},
diff --git a/tdrs-backend/plg/prometheus/alerts.local.yml b/tdrs-backend/plg/prometheus/alerts.local.yml
new file mode 100644
index 000000000..99183c544
--- /dev/null
+++ b/tdrs-backend/plg/prometheus/alerts.local.yml
@@ -0,0 +1,39 @@
+groups:
+  - name: database.alerts
+    rules:
+      - alert: LocalDatabaseDown
+        expr: last_over_time(pg_up{job="postgres"}[1m]) == 0
+        for: 1m
+        labels:
+          severity: CRITICAL
+        annotations:
+          summary: "The {{ $labels.service }} service is down."
+          description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute."
+  - name: backend.alerts
+    rules:
+    - alert: LocalBackendDown
+      expr: last_over_time(up{job=~"tdp-backend.*"}[1m]) == 0
+      for: 1m
+      labels:
+        severity: ERROR
+      annotations:
+          summary: "The {{ $labels.service }} service is down."
+          description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute."
+  - name: plg.alerts
+    rules:
+    - alert: LocalLokiDown
+      expr: last_over_time(up{job="loki"}[1m]) == 0
+      labels:
+        severity: ERROR
+      annotations:
+          summary: "The {{ $labels.service }} service is down."
+          description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute."
+  - name: app.alerts
+    rules:
+    - alert: UpTime
+      expr: avg_over_time(up[1m]) < 0.95
+      labels:
+        severity: WARNING
+      annotations:
+          summary: "The {{ $labels.service }} service has a uptime warning."
+          description: "The {{ $labels.service }} service in the {{ $labels.env }} environment is not maintaining 95% uptime."
diff --git a/tdrs-backend/plg/prometheus/alerts.yml b/tdrs-backend/plg/prometheus/alerts.yml
new file mode 100644
index 000000000..affe54498
--- /dev/null
+++ b/tdrs-backend/plg/prometheus/alerts.yml
@@ -0,0 +1,73 @@
+groups:
+  - name: database.alerts
+    rules:
+      - alert: DevDatabaseDown
+        expr: last_over_time(pg_up{job="postgres-dev"}[1m]) == 0
+        labels:
+          severity: ERROR
+        annotations:
+          summary: "The {{ $labels.service }} service is down."
+          description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute."
+      - alert: StagingDatabaseDown
+        expr: last_over_time(pg_up{job="postgres-staging"}[1m]) == 0
+        labels:
+          severity: ERROR
+        annotations:
+          summary: "The {{ $labels.service }} service is down."
+          description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute."
+      - alert: ProductionDatabaseDown
+        expr: last_over_time(pg_up{job="postgres-production"}[1m]) == 0
+        labels:
+          severity: CRITICAL
+        annotations:
+          summary: "The {{ $labels.service }} service is down."
+          description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute."
+  - name: backend.alerts
+    rules:
+    - alert: DevEnvironmentBackendDown
+      expr: last_over_time(up{job=~"tdp-backend.*", job!~".*prod", job!~".*staging"}[5m]) == 0
+      labels:
+        severity: ERROR
+      annotations:
+          summary: "The {{ $labels.service }} service is down."
+          description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 5 minutes."
+    - alert: StagingBackendDown
+      expr: last_over_time(up{job=~"tdp-backend-staging""}[1m]) == 0
+      labels:
+        severity: ERROR
+      annotations:
+          summary: "The {{ $labels.service }} service is down."
+          description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute."
+    - alert: ProductionBackendDown
+      expr: last_over_time(up{job=~"tdp-backend-prod"}[1m]) == 0
+      labels:
+        severity: CRITICAL
+      annotations:
+          summary: "The {{ $labels.service }} service is down."
+          description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute."
+  - name: plg.alerts
+    rules:
+    - alert: LokiDown
+      expr: last_over_time(up{job="loki"}[1m]) == 0
+      labels:
+        severity: ERROR
+      annotations:
+          summary: "The {{ $labels.service }} service is down."
+          description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute."
+    - alert: GrafanaDown
+      expr: last_over_time(up{job="grafana"}[1m]) == 0
+      labels:
+        severity: ERROR
+      annotations:
+          summary: "The {{ $labels.service }} service is down."
+          description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute."
+  - name: app.alerts
+    rules:
+    - alert: UpTime
+      expr: avg_over_time(up[1d]) < 0.95
+      for: 30m
+      labels:
+        severity: WARNING
+      annotations:
+          summary: "The {{ $labels.service }} service has a uptime warning."
+          description: "The {{ $labels.service }} service in the {{ $labels.env }} environment is not maintaining 95% uptime."
diff --git a/tdrs-backend/plg/prometheus/django_rules.yml b/tdrs-backend/plg/prometheus/django-rules.yml
similarity index 100%
rename from tdrs-backend/plg/prometheus/django_rules.yml
rename to tdrs-backend/plg/prometheus/django-rules.yml
diff --git a/tdrs-backend/plg/prometheus/prometheus.local.yml b/tdrs-backend/plg/prometheus/prometheus.local.yml
index b9d8256b1..8b0a4517d 100644
--- a/tdrs-backend/plg/prometheus/prometheus.local.yml
+++ b/tdrs-backend/plg/prometheus/prometheus.local.yml
@@ -7,18 +7,19 @@ global:
 # Alertmanager configuration
 alerting:
   alertmanagers:
-    - static_configs:
+    - path_prefix: /alerts
+      static_configs:
         - targets:
-          # - alertmanager:9093
+          - alertmanager:9093

 # Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
 rule_files:
-  - "django_rules.yml"
+  - "django-rules.yml"
+  - "alerts.yml"

 # A scrape configuration containing exactly one endpoint to scrape:
 # Here it's Prometheus itself.
 scrape_configs:
-  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
   - job_name: "prometheus"
     static_configs:
       - targets: ["localhost:9090"]
@@ -27,11 +28,35 @@ scrape_configs:
     metrics_path: "/prometheus/metrics"
     static_configs:
       - targets: ["web:8080"]
+        labels:
+          service: "tdp-backend"
+          env: "local"

   - job_name: "celery"
     static_configs:
       - targets: ["celery-exporter:9540"]
+        labels:
+          service: "celery"
+          env: "local"

   - job_name: postgres
     static_configs:
       - targets: ["postgres-exporter:9187"]
+        labels:
+          service: "postgres"
+          env: "local"
+
+  - job_name: loki
+    static_configs:
+      - targets: ["loki:3100"]
+        labels:
+          service: "loki"
+          env: "local"
+
+  - job_name: grafana
+    metrics_path: /grafana/metrics
+    static_configs:
+      - targets: ["grafana:9400"]
+        labels:
+          service: "grafana"
+          env: "local"
diff --git a/tdrs-backend/plg/prometheus/prometheus.yml b/tdrs-backend/plg/prometheus/prometheus.yml
index a8afaaa38..007422b26 100644
--- a/tdrs-backend/plg/prometheus/prometheus.yml
+++ b/tdrs-backend/plg/prometheus/prometheus.yml
@@ -6,24 +6,31 @@ global:
 # Alertmanager configuration
 alerting:
   alertmanagers:
-    - static_configs:
+    - path_prefix: /alerts
+      static_configs:
         - targets:
-          # - alertmanager:9093
+          # - alertmanager.apps.internal:8080

 # Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
 rule_files:
-  - "django_rules.yml"
+  - "django-rules.yml"

 scrape_configs:
   # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
   - job_name: "prometheus"
     static_configs:
       - targets: ["localhost:8080"]
+        labels:
+          service: "prometheus"
+          env: "production"

   - job_name: "tdp-backend-raft"
     metrics_path: "/prometheus/metrics"
     static_configs:
       - targets: ["tdp-backend-raft.apps.internal:8080"]
+        labels:
+          service: "tdp-backend"
+          env: "dev"

   - job_name: "tdp-backend-qasp"
     metrics_path: "/prometheus/metrics"
@@ -34,29 +41,65 @@ scrape_configs:
     metrics_path: "/prometheus/metrics"
     static_configs:
       - targets: ["tdp-backend-develop.apps.internal:8080"]
+        labels:
+          service: "tdp-backend"
+          env: "dev"

   - job_name: "tdp-backend-staging"
     metrics_path: "/prometheus/metrics"
     static_configs:
       - targets: ["tdp-backend-staging.apps.internal:8080"]
+        labels:
+          service: "tdp-backend"
+          env: "staging"

   - job_name: "tdp-backend-prod"
     metrics_path: "/prometheus/metrics"
     static_configs:
       - targets: ["tdp-backend-prod.apps.internal:8080"]
+        labels:
+          service: "tdp-backend"
+          env: "production"

   - job_name: "celery-exporter-raft"
     static_configs:
       - targets: ["celery-exporter-raft.apps.internal:9540"]
+        labels:
+          service: "celery"
+          env: "dev"

   - job_name: postgres-dev
     static_configs:
       - targets: ["pg-exporter-dev.apps.internal:9187"]
+        labels:
+          service: "postgres"
+          env: "dev"

   - job_name: postgres-staging
     static_configs:
       - targets: ["pg-exporter-staging.apps.internal:9187"]
+        labels:
+          service: "postgres"
+          env: "staging"

   - job_name: postgres-production
     static_configs:
       - targets: ["pg-exporter-production.apps.internal:9187"]
+        labels:
+          service: "postgres"
+          env: "production"
+
+  - job_name: loki
+    static_configs:
+      - targets: ["loki.apps.internal:3100"]
+        labels:
+          service: "loki"
+          env: "production"
+
+  - job_name: grafana
+    metrics_path: /grafana/metrics
+    static_configs:
+      - targets: ["grafana.app.cloud.gov:9400"]
+        labels:
+          service: "grafana"
+          env: "production"
diff --git a/tdrs-backend/tdpservice/urls.py b/tdrs-backend/tdpservice/urls.py
index eb91ffe48..e6b22e876 100755
--- a/tdrs-backend/tdpservice/urls.py
+++ b/tdrs-backend/tdpservice/urls.py
@@ -11,7 +11,7 @@
 from rest_framework.permissions import AllowAny

-from .users.api.authorization_check import AuthorizationCheck, KibanaAuthorizationCheck, GrafanaAuthorizationCheck
+from .users.api.authorization_check import AuthorizationCheck, KibanaAuthorizationCheck, PlgAuthorizationCheck
 from .users.api.login import TokenAuthorizationLoginDotGov, TokenAuthorizationAMS
 from .users.api.login import CypressLoginDotGovAuthenticationOverride
 from .users.api.login_redirect_oidc import LoginRedirectAMS, LoginRedirectLoginDotGov
@@ -54,7 +54,7 @@
     path("admin/", admin.site.urls, name="admin"),
     path("prometheus/", include("django_prometheus.urls")),
     path("kibana_auth_check/", KibanaAuthorizationCheck.as_view(), name="kibana-authorization-check"),
-    path("grafana_auth_check/", GrafanaAuthorizationCheck.as_view(), name="grafana-authorization-check"),
+    path("plg_auth_check/", PlgAuthorizationCheck.as_view(), name="plg-authorization-check"),
 ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

 # TODO: Supply `terms_of_service` argument in OpenAPI Info once implemented
diff --git a/tdrs-backend/tdpservice/users/api/authorization_check.py b/tdrs-backend/tdpservice/users/api/authorization_check.py
index 60da3a17b..1d7bed218 100644
--- a/tdrs-backend/tdpservice/users/api/authorization_check.py
+++ b/tdrs-backend/tdpservice/users/api/authorization_check.py
@@ -72,22 +72,24 @@ def get(self, request, *args, **kwargs):
             logger.debug(f"User: {user} has incorrect authentication credentials. Not allowing access to Kibana.")
             return HttpResponse(status=401)

-class GrafanaAuthorizationCheck(APIView):
+class PlgAuthorizationCheck(APIView):
     """Check if user is authorized to view Grafana."""

     query_string = False
-    pattern_name = "grafana-authorization-check"
+    pattern_name = "plg-authorization-check"
     permission_classes = [IsAuthenticated]

     def get(self, request, *args, **kwargs):
-        """Handle get request and verify user is authorized to access grafana."""
+        """Handle get request and verify user is authorized to access plg apps."""
         user = request.user

         user_in_valid_group = user.is_ofa_sys_admin or user.is_developer

+        print("\n\nINSIDE AUTH CHECK\n\n")
+
         if user_in_valid_group:
-            logger.debug(f"User: {user} has correct authentication credentials. Allowing access to Grafana.")
+            logger.debug(f"User: {user} has correct authentication credentials. Allowing access to plg.")
             return HttpResponse(status=200)
         else:
-            logger.debug(f"User: {user} has incorrect authentication credentials. Not allowing access to Grafana.")
+            logger.debug(f"User: {user} has incorrect authentication credentials. Not allowing access to plg.")
             return HttpResponse(status=401)
diff --git a/tdrs-frontend/docker-compose.yml b/tdrs-frontend/docker-compose.yml
index 13094148b..4a1a41fac 100644
--- a/tdrs-frontend/docker-compose.yml
+++ b/tdrs-frontend/docker-compose.yml
@@ -32,11 +32,12 @@ services:
       - LOCAL_DEV=true
       - KIBANA=kibana
       - GRAFANA=grafana
+      - ALERTS=alertmanager
       - REACT_APP_DEVAUTH=${REACT_APP_DEVAUTH}
     command: >
       /bin/sh -c
       "echo 'starting nginx' &&
-        envsubst '$${BACK_END} $${KIBANA} $${GRAFANA}' < /etc/nginx/locations.conf > /etc/nginx/locations_.conf &&
+        envsubst '$${BACK_END} $${KIBANA} $${GRAFANA} $${ALERTS}' < /etc/nginx/locations.conf > /etc/nginx/locations_.conf &&
         rm /etc/nginx/locations.conf &&
         cp /etc/nginx/locations_.conf /etc/nginx/locations.conf &&
        envsubst '
diff --git a/tdrs-frontend/nginx/local/locations.conf b/tdrs-frontend/nginx/local/locations.conf
index e25dad318..29ec9dec3 100644
--- a/tdrs-frontend/nginx/local/locations.conf
+++ b/tdrs-frontend/nginx/local/locations.conf
@@ -62,7 +62,7 @@ location = /kibana_auth_check {
 }

 location /grafana/ {
-        auth_request /grafana_auth_check;
+        auth_request /plg_auth_check;
         auth_request_set $auth_status $upstream_status;

         set $grafana http://${GRAFANA}:9400$request_uri;
@@ -79,9 +79,27 @@ location /grafana/ {
         proxy_buffer_size     4k;
 }

-location = /grafana_auth_check {
+location /alerts/ {
+        auth_request /plg_auth_check;
+        auth_request_set $auth_status $upstream_status;
+
+        set $alerts http://${ALERTS}:9093$request_uri;
+        proxy_pass $alerts;
+        proxy_set_header   Host $host:3000;
+        proxy_set_header   X-Real-IP         $remote_addr;
+        proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+        proxy_set_header   X-Forwarded-Proto https;
+
+        proxy_connect_timeout 300;
+        proxy_read_timeout    300;
+        proxy_send_timeout    300;
+        send_timeout          900;
+        proxy_buffer_size     4k;
+}
+
+location = /plg_auth_check {
     internal;
-    set $endpoint http://${BACK_END}:8080/grafana_auth_check/;
+    set $endpoint http://${BACK_END}:8080/plg_auth_check/;
     proxy_pass $endpoint$1$is_args$args;
     proxy_set_header   Host $host:3000;
     proxy_set_header   X-Real-IP         $remote_addr;
diff --git a/tdrs-frontend/src/components/Header/Header.jsx b/tdrs-frontend/src/components/Header/Header.jsx
index cd25c3930..602a2a613 100644
--- a/tdrs-frontend/src/components/Header/Header.jsx
+++ b/tdrs-frontend/src/components/Header/Header.jsx
@@ -8,7 +8,7 @@ import {
   accountIsInReview,
   accountCanViewAdmin,
   accountCanViewKibana,
-  accountCanViewGrafana,
+  accountCanViewPlg,
 } from '../../selectors/auth'

 import NavItem from '../NavItem/NavItem'
@@ -32,7 +32,7 @@ function Header() {
   const userAccessRequestApproved = useSelector(accountStatusIsApproved)
   const userIsAdmin = useSelector(accountCanViewAdmin)
   const userViewKibana = useSelector(accountCanViewKibana)
-  const userViewGrafana = useSelector(accountCanViewGrafana)
+  const userViewPlg = useSelector(accountCanViewPlg)

   const menuRef = useRef()

@@ -148,12 +148,19 @@ function Header() {
                       href={`${process.env.REACT_APP_BACKEND_HOST}/kibana/`}
                     />
                   )}
-                  {userViewGrafana && (
-                    <NavItem
-                      pathname={pathname}
-                      tabTitle="Grafana"
-                      href={`${process.env.REACT_APP_BACKEND_HOST}/grafana/`}
-                    />
+                  {userViewPlg && (
+                    <>
+                      <NavItem
+                        pathname={pathname}
+                        tabTitle="Grafana"
+                        href={`${process.env.REACT_APP_BACKEND_HOST}/grafana/`}
+                      />
+                      <NavItem
+                        pathname={pathname}
+                        tabTitle="Alerts"
+                        href={`${process.env.REACT_APP_BACKEND_HOST}/alerts/`}
+                      />
+                    </>
                   )}
                 </>
               )}
diff --git a/tdrs-frontend/src/components/SiteMap/SiteMap.jsx b/tdrs-frontend/src/components/SiteMap/SiteMap.jsx
index e7355842c..84e38dda0 100644
--- a/tdrs-frontend/src/components/SiteMap/SiteMap.jsx
+++ b/tdrs-frontend/src/components/SiteMap/SiteMap.jsx
@@ -4,14 +4,14 @@ import {
   accountStatusIsApproved,
   accountCanViewAdmin,
   accountCanViewKibana,
-  accountCanViewGrafana,
+  accountCanViewPlg,
 } from '../../selectors/auth'

 const SiteMap = ({ user }) => {
   const userIsApproved = useSelector(accountStatusIsApproved)
   const userIsAdmin = useSelector(accountCanViewAdmin)
   const userViewKibana = useSelector(accountCanViewKibana)
-  const userViewGrafana = useSelector(accountCanViewGrafana)
+  const userViewPlg = useSelector(accountCanViewPlg)

   return (
     <div className="margin-top-5">
@@ -43,11 +43,17 @@ const SiteMap = ({ user }) => {
         />
       )}

-      {userViewGrafana && (
-        <SiteMap.Link
-          text="Grafana"
-          link={`${process.env.REACT_APP_BACKEND_HOST}/grafana/`}
-        />
+      {userViewPlg && (
+        <>
+          <SiteMap.Link
+            text="Grafana"
+            link={`${process.env.REACT_APP_BACKEND_HOST}/grafana/`}
+          />
+          <SiteMap.Link
+            text="Alerts"
+            link={`${process.env.REACT_APP_BACKEND_HOST}/alerts/`}
+          />
+        </>
       )}
     </div>
   )
diff --git a/tdrs-frontend/src/selectors/auth.js b/tdrs-frontend/src/selectors/auth.js
index e6143ff5e..081196160 100644
--- a/tdrs-frontend/src/selectors/auth.js
+++ b/tdrs-frontend/src/selectors/auth.js
@@ -72,6 +72,6 @@ export const accountCanViewKibana = (state) =>
     selectPrimaryUserRole(state)?.name
   )

-export const accountCanViewGrafana = (state) =>
+export const accountCanViewPlg = (state) =>
   accountStatusIsApproved(state) &&
   ['OFA System Admin', 'Developer'].includes(selectPrimaryUserRole(state)?.name)

Local Alertmanager
**Description:**
As our monitoring capabilities enhance, we need a method to alert on logs, or metrics we are tracking. In this ticket we want to integrate [Alertmanager](https://prometheus.io/docs/alerting/latest/overview/) with our PLG stack to enable fine grained alerting capabilities. As an MVP implementation, we would like to integrate SendGrid with Alertmanager to notify the appropriate parties of some issue. The goal is to implement Prometheus [recording rules ](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) and (or) [alerting rules](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) for some metric that we are currently tracking. When the said metrics breaches some threshold or value, Prometheus should fire an alert to Alertmanager who then routes it to the correct receiver.

**Acceptance Criteria:**
_Create a list of functional outcomes that must be achieved to complete this issue_

- [ ] Prometheus connects to Alertmanager
- [ ] Prometheus sends alerts to Alertmanager
- [ ] Alertmanager integrated with SendGrid
- [ ] Alert emails are received from Alertmanager/SendGrid
- [ ] Documentation updated indicating Alertmanager integration
- [ ] README is updated, if necessary

**Tasks:**
_Create a list of granular, specific work items that must be completed to deliver the desired outcomes of this issue_

- [x] Add or create Prometheus rules for a chosen metric or metrics alerting
- [x] Add Alertmanager to compose file
- [x] [Configure](https://prometheus.io/docs/alerting/latest/configuration/#email_config) Alertmanager with Sendgrid
- [x] Generate a test to fire the chosen alert(s)
- [x] Run Testing Checklist and confirm all tests pass

**Notes:**
_Add additional useful information, such as related issues and functionality that isn't covered by this specific issue, and other considerations that will be helpful for anyone reading this_

N/A

**Supporting Documentation:**
_Please include any relevant log snippets/files/screen shots_

- [Alertmanager Docs](https://prometheus.io/docs/alerting/latest/configuration/)
- [Prometheus Docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/)

**Open Questions:**
_Please include any questions or decisions that must be made before beginning work or to confidently call this issue complete_

- _Open Question 1_
- _Open Question 2_

frontend
backend
dev
devops
Ready to Merge


diff --git a/lib/streamlit/runtime/app_session.py b/lib/streamlit/runtime/app_session.py
index 6fae7653b807..36e681c344f7 100644
--- a/lib/streamlit/runtime/app_session.py
+++ b/lib/streamlit/runtime/app_session.py
@@ -225,8 +225,10 @@ def shutdown(self) -> None:
             self._uploaded_file_mgr.remove_session_files(self.id)

             if runtime.exists():
-                runtime.get_instance().media_file_mgr.clear_session_refs(self.id)
-                runtime.get_instance().media_file_mgr.remove_orphaned_files()
+                rt = runtime.get_instance()
+                rt.media_file_mgr.clear_session_refs(self.id)
+                rt.media_file_mgr.remove_orphaned_files()
+                rt.message_cache.remove_refs_for_session(self)

             # Shut down the ScriptRunner, if one is active.
             # self._state must not be set to SHUTDOWN_REQUESTED until
diff --git a/lib/streamlit/runtime/forward_msg_cache.py b/lib/streamlit/runtime/forward_msg_cache.py
index f348603ac520..13f494867569 100644
--- a/lib/streamlit/runtime/forward_msg_cache.py
+++ b/lib/streamlit/runtime/forward_msg_cache.py
@@ -217,7 +217,28 @@ def has_message_reference(
         age = entry.get_session_ref_age(session, script_run_count)
         return age <= int(config.get_option("global.maxCachedMessageAge"))

-    def remove_expired_session_entries(
+    def remove_refs_for_session(self, session: "AppSession") -> None:
+        """Remove refs for all entries for the given session.
+
+        This should be called when an AppSession is being shut down.
+
+        Parameters
+        ----------
+        session : AppSession
+        """
+
+        # Operate on a copy of our entries dict.
+        # We may be deleting from it.
+        for msg_hash, entry in self._entries.copy().items():
+            if entry.has_session_ref(session):
+                entry.remove_session_ref(session)
+
+            if not entry.has_refs():
+                # The entry has no more references. Remove it from
+                # the cache completely.
+                del self._entries[msg_hash]
+
+    def remove_expired_entries_for_session(
         self, session: "AppSession", script_run_count: int
     ) -> None:
         """Remove any cached messages that have expired from the given session.
diff --git a/lib/streamlit/runtime/runtime.py b/lib/streamlit/runtime/runtime.py
index ddb18a1656dd..b34503bcac03 100644
--- a/lib/streamlit/runtime/runtime.py
+++ b/lib/streamlit/runtime/runtime.py
@@ -687,7 +687,7 @@ def _send_message(self, session_info: ActiveSessionInfo, msg: ForwardMsg) -> Non
                 config.get_option("global.maxCachedMessageAge"),
             )
             session_info.script_run_count += 1
-            self._message_cache.remove_expired_session_entries(
+            self._message_cache.remove_expired_entries_for_session(
                 session_info.session, session_info.script_run_count
             )

diff --git a/lib/tests/streamlit/runtime/app_session_test.py b/lib/tests/streamlit/runtime/app_session_test.py
index 2ee2e5d8cd40..448118736e9a 100644
--- a/lib/tests/streamlit/runtime/app_session_test.py
+++ b/lib/tests/streamlit/runtime/app_session_test.py
@@ -101,15 +101,19 @@ def test_shutdown(self, patched_disconnect):

         mock_file_mgr = MagicMock(spec=UploadedFileManager)
         session._uploaded_file_mgr = mock_file_mgr
+        mock_message_cache = Runtime._instance.message_cache

         session.shutdown()
         self.assertEqual(AppSessionState.SHUTDOWN_REQUESTED, session._state)
         mock_file_mgr.remove_session_files.assert_called_once_with(session.id)
         patched_disconnect.assert_called_once_with(session._on_secrets_file_changed)
+        mock_message_cache.remove_refs_for_session.assert_called_once_with(session)

         # A 2nd shutdown call should have no effect.
         session.shutdown()
         self.assertEqual(AppSessionState.SHUTDOWN_REQUESTED, session._state)
+        mock_message_cache.remove_refs_for_session.assert_called_once_with(session)
+
         mock_file_mgr.remove_session_files.assert_called_once_with(session.id)

     def test_shutdown_with_running_scriptrunner(self):
diff --git a/lib/tests/streamlit/runtime/forward_msg_cache_test.py b/lib/tests/streamlit/runtime/forward_msg_cache_test.py
index 7d5122808cda..8bce96913de3 100644
--- a/lib/tests/streamlit/runtime/forward_msg_cache_test.py
+++ b/lib/tests/streamlit/runtime/forward_msg_cache_test.py
@@ -101,6 +101,42 @@ def test_clear(self):
         cache.clear()
         self.assertEqual(None, cache.get_message(msg_hash))

+    def test_remove_refs_for_session(self):
+        cache = ForwardMsgCache()
+
+        session1 = _create_mock_session()
+        session2 = _create_mock_session()
+
+        # Only session1 has a ref to msg1.
+        msg1 = _create_dataframe_msg([1, 2, 3])
+        populate_hash_if_needed(msg1)
+        cache.add_message(msg1, session1, 0)
+
+        # Only session2 has a ref to msg2.
+        msg2 = _create_dataframe_msg([1, 2, 3, 4])
+        populate_hash_if_needed(msg2)
+        cache.add_message(msg2, session2, 0)
+
+        # Both session1 and session2 have a ref to msg3.
+        msg3 = _create_dataframe_msg([1, 2, 3, 4, 5])
+        populate_hash_if_needed(msg2)
+        cache.add_message(msg3, session1, 0)
+        cache.add_message(msg3, session2, 0)
+
+        cache.remove_refs_for_session(session1)
+
+        cache_entries = list(cache._entries.values())
+
+        cached_msgs = [entry.msg for entry in cache_entries]
+        assert cached_msgs == [msg2, msg3]
+
+        sessions_with_refs = {
+            s
+            for entry in cache_entries
+            for s in entry._session_script_run_counts.keys()
+        }
+        assert sessions_with_refs == {session2}
+
     def test_message_expiration(self):
         """Test MessageCache's expiration logic"""
         config._set_option("global.maxCachedMessageAge", 1, "test")
@@ -131,14 +167,14 @@ def test_message_expiration(self):

         # Remove session1's expired entries. This should not remove the
         # entry from the cache, because session2 still has a reference to it.
-        cache.remove_expired_session_entries(session1, runcount1)
+        cache.remove_expired_entries_for_session(session1, runcount1)
         self.assertFalse(cache.has_message_reference(msg, session1, runcount1))
         self.assertTrue(cache.has_message_reference(msg, session2, runcount2))

         # Expire session2's reference. The message should no longer be
         # in the cache at all.
         runcount2 += 2
-        cache.remove_expired_session_entries(session2, runcount2)
+        cache.remove_expired_entries_for_session(session2, runcount2)
         self.assertIsNone(cache.get_message(msg_hash))

     def test_cache_stats_provider(self):

Updating images will increase memory usage
### Checklist

- [X] I have searched the [existing issues](https://github.com/streamlit/streamlit/issues) for similar issues.
- [X] I added a very descriptive title to this issue.
- [X] I have provided sufficient information below to help reproduce this issue.

### Summary

When updating images frequently the memory usage of the Streamlit app keeps increasing. A lot!

You don't need a lot of sessions to get this behavior. Running a single page and refreshing it after some time will not release the memory.

### Reproducible Code Example

```Python
import datetime
import time
import psutil
import os
pid = os.getpid()

import cv2
import numpy as np
import streamlit as st

img = np.random.rand(600,600,3)

@st.cache_resource(ttl=1)
def get_memory(pid):
    process = psutil.Process(pid)

    # Get the memory usage in RAM
    memory_usage = process.memory_info().rss

    # Convert the memory usage to MB
    memory_usage_mb = memory_usage / (1024 * 1024)
    return(memory_usage_mb)
    #print(f"Total memory usage of all running Python processes: {mem_usage} bytes")

def get_image():
    # Get current date and time
    img_with_date = img.copy()
    now = datetime.datetime.now()
    dt_string = now.strftime("%d/%m/%Y %H:%M:%S.%f")[:-3]
    cv2.putText(img_with_date, dt_string, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2)
    return img_with_date

def app():
    if "page_open" not in st.session_state:
        st.session_state.page_open = True

    if "num_img" not in st.session_state:
        st.session_state.num_img = 0

    st.title("Test memory usage of st.image...")

    text_placeholder = st.empty()
    img_placeholder = st.empty()

    while "page_open" in st.session_state:
        text_placeholder.write(f"{st.session_state.num_img} {get_memory(pid):.2f}MB")
        img_placeholder.image(get_image())

        st.session_state.num_img += 1
        time.sleep(1/10)

    print("Session ended...")
    img_placeholder.empty()
    text_placeholder.empty()
    st.stop()
if __name__ == "__main__":
    app()
```

### Steps To Reproduce

1. Run streamlit app
2. Maybe open a few tabs to speed up the memory consumption
3. Close all tabs
4. Open a new tab, memory usage will have decreased a bit but it will never go down to an acceptable size anymore

### Expected Behavior

Resources should be removed when no sessions are running anymore. In specific use cases it might even be needed to have an aggressive cleanup of resources, for example when showing live video.

### Current Behavior

Current behavior is that images are stored somewhere in RAM and there doesn't seem to be a way to control it. This was already discussed [on the forum](https://discuss.streamlit.io/t/updating-images-in-a-while-loop-keeps-increasing-ram-even-after-sessions-are-closed-and-cleaned-up/38226/5) with @vdonato

### Is this a regression?

- [ ] Yes, this used to work in a previous version.

### Debug info

Streamlit version: 1.20.0
Python version: 3.10.9
Operating System: Windows 11 (WSL2)
Browser: Chrome
Virtual environment: conda

### Additional Information

The app will show both the image id and the memory usage to make it really easy to see that the RAM is indeed increasing over time.

Dependencies:
```
pip install streamlit==1.20.0
pip install opencv-python-headless
pip install psutil
```

### Are you willing to submit a PR?

- [ ] Yes, I am willing to submit a PR!

@MathijsNL Thanks for reporting this. I was able to reproduce this issue. We do have something that is actually supposed to clean up the media file storage after a session. But this somehow might not work correctly.

I also tested with older Streamlit versions (e.g. 1.15), and this does seem to behave the same way. So it's not something that we broke recently.
In the  [media_file_manager.py](https://github.com/streamlit/streamlit/blob/1.20.0/lib/streamlit/runtime/media_file_manager.py#L213) file, I added a line of code that calls the remove_orphaned_files() function (self.remove_orphaned_files()) to maintain stable memory usage at 232 MB during one session with the provided code.

However, I noticed that there is no existing call to remove_orphaned_files() in the media_file_manager code at all right now. I'm unsure of the best location to add the function call or if the function itself is outdated and because of that it is no longer in use.

When I tried adding the function call to the MediaFileManager's add function for multiple sessions, I started receiving missing file errors, but at least if there is a single long running session this workaround would work for me.

@vdonato I am tagging you here as requested on the forum.
Hey @MathijsNL,

I finally had a bit of time to dig into this, and I think there may not actually be a true memory leak here (at this point I'm fairly certain but not 100% confident as it's been quite awhile since I've had to reason about what may be going on at the OS level).

After inspecting the streamlit server's media file storage when sessions are cleaned up + poking around with a memory profiler + seeing what objects the `gc` is keeping track of, I couldn't find any evidence that we're failing to clean up the media files that images are saved in after a session ends (which is when all of these mechanisms get kicked off -- your workaround should be sufficient for the use case where a session lives indefinitely, but the ultra-long-lived session isn't a use case we currently support very well. Supporting this use case would best be filed as an enhancement request rather than a bug report).

What I think might be going on here is that the memory used by the test script is properly garbage collected by Python when a session closes, but the OS isn't reclaiming all of this memory back from the process unless there's sufficient memory pressure on the system that it needs to do so -- you'd likely need another memory hungry process on the same system to force the memory reallocation to actually happen. This seems to be consistent with information that I've been able to find online about how RSS is actually calculated (see https://utcc.utoronto.ca/~cks/space/blog/linux/LinuxMemoryStats).

Let me know if this seems inconsistent with any findings that you may have.
Going to change the status of this one to `unlikely` because it seems like there may not be a bug here, but I'm not 100% sure about this conclusion yet.