diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py
index 2402ff7d3..65a4cfcc4 100644
--- a/jmclient/jmclient/blockchaininterface.py
+++ b/jmclient/jmclient/blockchaininterface.py
@@ -292,18 +292,36 @@ def import_addresses_if_needed(self, addresses, wallet_name):
             self.import_addresses(addresses - imported_addresses, wallet_name)
         return import_needed

-    def _yield_transactions(self, wallet_name):
-        batch_size = 1000
-        iteration = 0
+    def _yield_transactions(self):
+        """ Generates a lazily fetched sequence of transactions seen in the
+        wallet (under any label/account), yielded in newest-first order. Care
+        is taken to avoid yielding duplicates even when new transactions are
+        actively being added to the wallet while the iteration is ongoing.
+        """
+        num, skip = 1, 0
+        txs = self.list_transactions(num, skip)
+        if not txs:
+            return
+        yielded_tx = txs[0]
+        yield yielded_tx
         while True:
-            new = self._rpc(
-                'listtransactions',
-                ["*", batch_size, iteration * batch_size, True])
-            for tx in new:
-                yield tx
-            if len(new) < batch_size:
+            num *= 2
+            txs = self.list_transactions(num, skip)
+            if not txs:
+                return
+            try:
+                idx = [(tx['txid'], tx['vout'], tx['category']) for tx in txs
+                        ].index((yielded_tx['txid'], yielded_tx['vout'],
+                        yielded_tx['category']))
+            except ValueError:
+                skip += num
+                continue
+            for tx in reversed(txs[:idx]):
+                yielded_tx = tx  # inefficient but more obvious
+                yield yielded_tx
+            if len(txs) < num:
                 return
-            iteration += 1
+            skip += num - 1

     def get_deser_from_gettransaction(self, rpcretval):
         """Get full transaction deserialization from a call
@@ -641,7 +659,7 @@ def _get_addr_from_desc(self, desc_str):
         assert desc_str.startswith("addr(")
         return desc_str[5:desc_str.find(")")]

-    def _yield_transactions(self, wallet_name):
+    def _yield_transactions(self):
         for u in self.scan_result["unspents"]:
             tx = {"category": "receive", "address":
                 self._get_addr_from_desc(u["desc"])}
diff --git a/jmclient/jmclient/wallet_service.py b/jmclient/jmclient/wallet_service.py
index 332523ed1..0b09d35f5 100644
--- a/jmclient/jmclient/wallet_service.py
+++ b/jmclient/jmclient/wallet_service.py
@@ -1,6 +1,7 @@
 #! /usr/bin/env python

 import collections
+import itertools
 import time
 import sys
 from decimal import Decimal
@@ -74,18 +75,19 @@ def __init__(self, wallet):
         # Dicts of registered callbacks, by type
         # and then by txinfo, for events
         # on transactions.
-        self.callbacks = {}
-        self.callbacks["all"] = []
-        self.callbacks["unconfirmed"] = {}
-        self.callbacks["confirmed"] = {}
+        self.callbacks = {
+            "all": [],  # note: list, not dict
+            "unconfirmed": {},
+            "confirmed": {},
+        }

         self.restart_callback = None

         # transactions we are actively monitoring,
         # i.e. they are not new but we want to track:
-        self.active_txids = []
+        self.active_txs = {}
         # to ensure transactions are only processed once:
-        self.processed_txids = []
+        self.processed_txids = set()

         self.set_autofreeze_warning_cb()

@@ -200,9 +202,12 @@ def register_callbacks(self, callbacks, txinfo, cb_type="all"):
             # note that in this case, txid is ignored.
             self.callbacks["all"].extend(callbacks)
         elif cb_type in ["unconfirmed", "confirmed"]:
-            if txinfo not in self.callbacks[cb_type]:
-                self.callbacks[cb_type][txinfo] = []
-            self.callbacks[cb_type][txinfo].extend(callbacks)
+            if callbacks:
+                reg = self.callbacks[cb_type].setdefault(txinfo, [])
+                if isinstance(reg, str):
+                    # found a txid breadcrumb for this txinfo
+                    reg = self.callbacks[cb_type].setdefault(reg, [])
+                reg.extend(callbacks)
         else:
             assert False, "Invalid argument: " + cb_type

@@ -285,6 +290,23 @@ def check_for_reuse(self, added_utxos):
                 self.autofreeze_warning_cb(utxo)
                 self.disable_utxo(*utxo)

+    def _yield_new_transactions(self):
+        """ Constrains the sequence generated by bci._yield_transactions so
+        that it stops just before it would yield the newest transaction
+        previously yielded by _yield_new_transactions.
+        """
+        since_txid = self.last_seen_txid
+        last = True
+        for tx in self.bci._yield_transactions():
+            if 'txid' in tx:
+                txid = tx['txid']
+                if txid == since_txid:
+                    return
+                if last:
+                    self.last_seen_txid = txid
+                    last = False
+            yield tx
+
     def transaction_monitor(self):
         """Keeps track of any changes in the wallet (new transactions).
         Intended to be run as a twisted task.LoopingCall so that this
@@ -294,33 +316,19 @@ def transaction_monitor(self):
         if not self.update_blockheight():
             return

-        txlist = self.bci.list_transactions(100)
-        if not txlist:
-            return
-
-        new_txs = []
-        for x in txlist:
-            # process either (a) a completely new tx or
-            # (b) a tx that reached unconf status but we are still
-            # waiting for conf (active_txids)
-            if "txid" not in x:
-                continue
-            if x['txid'] in self.active_txids or x['txid'] not in self.old_txs:
-                new_txs.append(x)
-        # reset for next polling event:
-        self.old_txs = set(x['txid'] for x in txlist if "txid" in x)
         # for this invocation of transaction_monitor, we *don't* want
         # to call `gettransaction` more than once per txid, even if the
         # `listtransactions` result has multiple instances for different
         # wallet labels; so we use a temporary variable to cache.
-        gettx_results = {}
-        for tx in new_txs:
-            txid = tx["txid"]
-            if txid not in gettx_results:
-                res = self.bci.get_transaction(hextobin(txid))
-                gettx_results[txid] = res
-            else:
-                res = gettx_results[txid]
+        seen_txids = set()
+        wallet_name = self.get_wallet_name()
+        for txid in itertools.chain(list(self.active_txs), reversed(
+                [tx['txid'] for tx in self._yield_new_transactions()
+                    if 'txid' in tx])):
+            if txid in seen_txids:
+                continue
+            seen_txids.add(txid)
+            res = self.bci.get_transaction(hextobin(txid))
             if not res:
                 continue
             confs = res["confirmations"]
@@ -330,13 +338,18 @@ def transaction_monitor(self):
             if confs < 0:
                 jlog.info(
                     "Transaction: " + txid + " has a conflict, abandoning.")
+                self.active_txs.pop(txid, None)
                 continue
             if confs == 0:
+                if txid in self.active_txs:
+                    # avoid processing an unconfirmed tx we've already seen
+                    continue
                 height = None
             else:
                 height = self.current_blockheight - confs + 1

-            txd = self.bci.get_deser_from_gettransaction(res)
+            txd = self.active_txs.get(txid) or \
+                    self.bci.get_deser_from_gettransaction(res)
             if txd is None:
                 continue
             removed_utxos, added_utxos = self.wallet.process_new_tx(txd, height)
@@ -347,23 +360,31 @@ def transaction_monitor(self):
                 # is absurdly fast, this is considered acceptable compared with
                 # additional complexity.
                 self.log_new_tx(removed_utxos, added_utxos, txid)
-                self.processed_txids.append(txid)
+                self.processed_txids.add(txid)

             # first fire 'all' type callbacks, irrespective of if the
             # transaction pertains to anything known (but must
             # have correct label per above); filter on this Joinmarket wallet label,
             # or the external monitoring label:
-            if (self.bci.is_address_labeled(tx, self.get_wallet_name()) or
-                self.bci.is_address_labeled(tx, self.EXTERNAL_WALLET_LABEL)):
+            if (any(self.bci.is_address_labeled(addr, wallet_name) or
+                    self.bci.is_address_labeled(addr,
+                        self.EXTERNAL_WALLET_LABEL)
+                    for addr in res["details"])):
                 for f in self.callbacks["all"]:
                     # note we need no return value as we will never
                     # remove these from the list
                     f(txd, txid)

-            # The tuple given as the second possible key for the dict
-            # is such because txid is not always available
-            # at the time of callback registration).
-            possible_keys = [txid, tuple((x.scriptPubKey, x.nValue) for x in txd.vout)]
+            # txid is not always available at the time of callback registration.
+            # Migrate any callbacks registered under the provisional key, and
+            # leave a txid breadcrumb so check_callback_called can find it.
+            txos = tuple((x.scriptPubKey, x.nValue) for x in txd.vout)
+            for cb_type in ["unconfirmed", "confirmed"]:
+                callbacks = self.callbacks[cb_type]
+                reg = callbacks.get(txos)
+                if isinstance(reg, list):
+                    callbacks.setdefault(txid, [])[:0] = reg
+                    callbacks[txos] = txid

             # note that len(added_utxos) > 0 is not a sufficient condition for
             # the tx being new, since wallet.add_new_utxos will happily re-add
@@ -373,27 +394,31 @@ def transaction_monitor(self):
             # Note also that it's entirely possible that there are only removals,
             # not additions, to the utxo set, specifically in sweeps to external
             # addresses. In this case, since removal can by definition only
-            # happen once, we must allow entries in self.active_txids through the
+            # happen once, we must allow txids in self.active_txs through the
             # filter.
             if len(added_utxos) > 0 or len(removed_utxos) > 0 \
-               or txid in self.active_txids:
+                    or txid in self.active_txs:
                 if confs == 0:
-                    for k in possible_keys:
-                        if k in self.callbacks["unconfirmed"]:
-                            for f in self.callbacks["unconfirmed"][k]:
-                                # True implies success, implies removal:
-                                if f(txd, txid):
-                                    self.callbacks["unconfirmed"][k].remove(f)
-                                    # keep monitoring for conf > 0:
-                                    self.active_txids.append(txid)
+                    callbacks = [f for f in
+                            self.callbacks["unconfirmed"].pop(txid, [])
+                            if not f(txd, txid)]
+                    if callbacks:
+                        self.callbacks["unconfirmed"][txid] = callbacks
+                    else:
+                        self.callbacks["unconfirmed"].pop(txos, None)
+                    if self.callbacks["confirmed"].get(txid):
+                        # keep monitoring for conf > 0:
+                        self.active_txs[txid] = txd
                 elif confs > 0:
-                    for k in possible_keys:
-                        if k in self.callbacks["confirmed"]:
-                            for f in self.callbacks["confirmed"][k]:
-                                if f(txd, txid, confs):
-                                    self.callbacks["confirmed"][k].remove(f)
-                                    if txid in self.active_txids:
-                                        self.active_txids.remove(txid)
+                    callbacks = [f for f in
+                            self.callbacks["confirmed"].pop(txid, [])
+                            if not f(txd, txid, confs)]
+                    if callbacks:
+                        self.callbacks["confirmed"][txid] = callbacks
+                    else:
+                        self.callbacks["confirmed"].pop(txos, None)
+                        # no more callbacks registered; stop monitoring tx
+                        self.active_txs.pop(txid, None)

     def check_callback_called(self, txinfo, callback, cbtype, msg):
         """ Intended to be a deferred Task to be scheduled some
@@ -402,15 +427,27 @@ def check_callback_called(self, txinfo, callback, cbtype, msg):
         If the callback was previously called, return True, otherwise False.
         """
         assert cbtype in ["unconfirmed", "confirmed"]
-        if txinfo in self.callbacks[cbtype]:
-            if callback in self.callbacks[cbtype][txinfo]:
+        callbacks = self.callbacks[cbtype]
+        if isinstance(txinfo, str):
+            txid = txinfo
+            reg = callbacks.get(txid)
+        else:
+            txid = None
+            reg = callbacks.get(txinfo)
+            if isinstance(reg, str):
+                # found a txid breadcrumb for this txinfo
+                txid = reg
+                reg = callbacks.get(txid)
+        if reg:
+            if callback in reg:
                 # the callback was not called, drop it and warn
-                self.callbacks[cbtype][txinfo].remove(callback)
-                # TODO - dangling txids in self.active_txids will
-                # be caused by this, but could also happen for
-                # other reasons; possibly add logic to ensure that
-                # this never occurs, although their presence should
-                # not cause a functional error.
+                reg.remove(callback)
+                if not reg:
+                    del callbacks[txinfo]
+                    if txid:
+                        callbacks.pop(txid, None)
+                        # no more callbacks registered; stop monitoring tx
+                        self.active_txs.pop(txid, None)
                 jlog.info("Timed out: " + msg)
                 return False
             # if callback is not in the list, it was already
@@ -454,8 +491,9 @@ def sync_wallet(self, fast=True):
             self.sync_unspent()
         # Don't attempt updates on transactions that existed
         # before startup
-        self.old_txs = [x['txid'] for x in self.bci.list_transactions(100)
-            if "txid" in x]
+        self.last_seen_txid = next(
+                (tx['txid'] for tx in self.bci._yield_transactions()
+                    if 'txid' in tx), None)
         if isinstance(self.bci, BitcoinCoreNoHistoryInterface):
             self.bci.set_wallet_no_history(self.wallet)
         return self.synced
@@ -668,14 +706,13 @@ def get_all_transactions(self):
         to this wallet, as a list.
         """
         res = []
-        processed_txids = []
-        for r in self.bci._yield_transactions(
-            self.get_wallet_name()):
+        processed_txids = set()
+        for r in self.bci._yield_transactions():
             txid = r["txid"]
             if txid not in processed_txids:
                 tx = self.bci.get_transaction(hextobin(txid))
                 res.append(self.bci.get_deser_from_gettransaction(tx))
-                processed_txids.append(txid)
+                processed_txids.add(txid)
         return res

     def get_transaction(self, txid):
@@ -720,7 +757,7 @@ def sync_addresses(self):
         if isinstance(self.wallet, FidelityBondMixin):
             tx_receive = []
             burner_txes = []
-            for tx in self.bci._yield_transactions(wallet_name):
+            for tx in self.bci._yield_transactions():
                 if tx['category'] == 'receive':
                     tx_receive.append(tx)
                 elif tx["category"] == "send":
@@ -743,7 +780,7 @@ def sync_addresses(self):
         else:
             #not fidelity bond wallet, significantly faster sync
             used_addresses_gen = set(tx['address']
-                                  for tx in self.bci._yield_transactions(wallet_name)
+                                  for tx in self.bci._yield_transactions()
                                   if tx['category'] == 'receive')
         # needed for address-reuse check:
         self.used_addresses = used_addresses_gen

yg-privacyenhanced.py: excessive CPU usage
At first startup of the Yield Generator, CPU usage seems acceptable, but as it sits running for a few days, it starts using excessive amounts of CPU time. It seems to depend on how many CoinJoins it has participated in. Case in point: I have two instances of `yg-privacyenhanced.py` running (with separate JM wallets and separate Bitcoin Core wallets), one running with native SegWit and the other with transitional SegWit. Both were started at the same time, about 154 hours ago. One has used about 25 minutes of CPU time. The other has used about 75 hours of CPU time — i.e., nearly half of the wall-clock time that it has been running!

Attaching to the well-behaved instance with `strace` shows that it is making one `getblockcount` RPC call and one `listtransactions` RPC call about every 4-5 seconds. Not great, but okay. (Polling is evil.) Attaching `strace` to the misbehaving instance shows that it is obsessively (many times per second!) calling `gettransaction`, all for the same transaction ID. The responses from Bitcoin Core show that the specified transaction has over 200 confirmations, so why is JoinMarket obsessively hammering requests for it?

If I restart the `yg-privacyenhanced.py` script, then the new instance is well behaved for a while, but eventually it will start obsessively hammering requests for some particular transaction ID again.

>  it is obsessively (many times per second!) calling `gettransaction`, all for the same transaction ID. The responses from Bitcoin Core show that the specified transaction has over 200 confirmations, so why is JoinMarket obsessively hammering requests for it?

It shouldn't, sounds like a bug somewhere.
> RPC call and one listtransactions RPC call about every 4-5 seconds. Not great, but okay. (Polling is evil.)

That's right, there is a single global 5 second polling loop (`WalletService.transaction_monitor`). (If you think this is bad, there used to be a polling loop per transaction we do!).

> Attaching strace to the misbehaving instance shows that it is obsessively (many times per second!) calling gettransaction, all for the same transaction ID.

That's pretty interesting, at least in the sense that I don't off the top of my head have any reason why a Joinmarket script (a yg specifically, would do that).
But in one sense it's a bit unclear - "obsessively many times per second" - but for how long? Continuously without end? Or for like 1-2 seconds? There might be a badly conceived loop somewhere that could result in the latter. I'm thinking, related to PoDLE verification, but I'd need to check. But the former would be really bizarre.

> But in one sense it's a bit unclear - "obsessively many times per second" - but for how long? Continuously without end? Or for like 1-2 seconds?

Continuously without end. That's how my JoinMarket process got up to 75 hours of CPU time used and was still climbing. Presumably it's only *not* using all 100% of a CPU core because it has to wait for responses from Bitcoin Core.
It has happened again. The repeating pattern of syscalls I am seeing is as follows:

1. `sendto` is called with payload:
    ```
    POST /wallet/joinmarket-sw0 HTTP/1.1
    Host: localhost:8332
    Accept-Encoding: identity
    Content-Length: 55
    User-Agent: joinmarket
    Content-Type: application/json
    Accept: application/json
    Authorization: Basic ##redacted##

    ```
1. `sendto` is called with payload:
    ```
    {"method": "getblockcount", "params": [], "id": 506225}
    ```
1. `recvfrom` returns payload:
    ```
    HTTP/1.1 200 OK
    Content-Type: application/json
    Date: Mon, 25 Jan 2021 20:11:45 GMT
    Content-Length: 43

    {"result":667641,"error":null,"id":506225}
    ```
1. `sendto` is called with payload:
    ```
    POST /wallet/joinmarket-sw0 HTTP/1.1
    Host: localhost:8332
    Accept-Encoding: identity
    Content-Length: 75
    User-Agent: joinmarket
    Content-Type: application/json
    Accept: application/json
    Authorization: Basic ##redacted##

    ```
1. `sendto` is called with payload:
    ```
    {"method": "listtransactions", "params": ["*", 100, 0, true], "id": 506226}
    ```
1. `recvfrom` returns payload:
    ```
    HTTP/1.1 200 OK
    Content-Type: application/json
    Date: Mon, 25 Jan 2021 20:11:45 GMT
    Content-Length: 46976

    {\"result\":[…##redacted##…
1. `sendto` is called with payload:
    ```
    POST /wallet/joinmarket-sw0 HTTP/1.1
    Host: localhost:8332
    Accept-Encoding: identity
    Content-Length: 128
    User-Agent: joinmarket
    Content-Type: application/json
    Accept: application/json
    Authorization: Basic ##redacted##

    ```
1. `sendto` is called with payload:
    ```
    {"method": "gettransaction", "params": ["e##…##c", true], "id": 506227}
    ```
1. `recvfrom` returns payload:
    ```
    HTTP/1.1 200 OK
    Content-Type: application/json
    Date: Mon, 25 Jan 2021 20:11:45 GMT
    Content-Length: 10618

    {"result":{"amount":-##redacted##,"fee":##redacted##,"confirmations":0,"trusted":false,"txid":"e##…##c","walletconflicts":[],"time":1611564203,"timereceived":1611564203,"bip125-replaceable":"unknown","details":[…],"hex":…},"error":null,"id":506227}
    ```
1. Previous three steps are repeated 21 times with `"id"` numbers incrementing from 506228 to 506248.
1. `epoll_wait` is called with a timeout of around 2.6 seconds — presumably the remainder of the 5-second polling interval.
1. Restart from step 1.

**TL;DR:** In each iteration of the 5-second polling loop, `yg-privacyenhanced.py` calls `getblockcount`, reads the response, calls `listtransactions`, reads the response, and then calls `gettransaction` with the same TxID 22 times, reading the response after each call.
My other instance of `yg-privacyenhanced.py` is not currently exhibiting the excessive CPU utilization problem, and its polling loop is only calling `getblockcount` and `listtransactions` once every 5 seconds. It is not calling `gettransaction` at all. It has completed a CoinJoin in its current run, so the difference isn't that the bad one has done a CoinJoin while the good one hasn't yet; no, both instances have done CoinJoins in their current runs.
Thanks for this. I will have to take a look at what might be going on, but basically if I read you right, you're saying that `gettransaction` is being called 22 times per polling loop, on one txid - but also, you're saying that's taking around 2.5 seconds, is that right?
I think there is unnecessary use of list in `transaction_monitor` when it should be a set. While there's some accounting for duplication, it seems very likely that there are redundant calls to gettransaction because of it (though the exact circumstances I'd need to look at). I'll spend some time testing it out tomorrow.
> you're saying that `gettransaction` is being called 22 times per polling loop, on one txid - but also, you're saying that's taking around 2.5 seconds, is that right?

Correct on all points. Note that the `listtransactions` call is pretty heavy since it always asks for 100 transactions. Seems like it should only ask for the latest 1 transaction and exponentially fetch further back until it reaches a transaction it already knows about.
I'm not sure that matters, but I could believe it. Iirc listtransactions return is not particularly large as the data it returns is relatively compact but ... in any case i need to spend a bit of time checking.

The part of the story that confuses me though is, why you specifically are getting a large *CPU* usage - I would have thought even if the listtransactions return were large it couldn't/wouldn't cause that. More plausible would be our parsing code converting the gettransaction results into deserialized objects, that could at least feasibly use a lot of CPU.
See the commit notes for #795 ... in testing I saw a 4 times or so reduction in the number of gettx calls for a full tumbler run (counting both maker and taker sides) (but meticulous accounting is tricky).

My initial thought was just caching of gettx results; but across monitoring loop invocations that makes no sense, as the main (but not only) reason we do those calls is to check confirmation status.

I could get a little more into the weeds than I did in that commit comment, but at root, after making sure at least we don't use a stupidly redundant amount of calls, I was always of the opinion that a few extra rpc calls shouldn't matter too much; I still absolutely don't understand why say 20-25 rpc calls would chew such a huge amount of CPU; I've never seen anything similar myself.

But anyway - that patch is worth running (I guess it'll get merged very soon), as it is bound to at least reduce the number of calls in pretty much any usage scenario.
@AdamISZ: Thank you for your work on this. To clarify, it is not the `listtransactions` call that is chewing up CPU time. A properly behaving YG instance (despite polling `listtransactions` every 5 seconds) uses less than 0.1% of my CPU time. It's the misbehaving instance (which calls `gettransaction` multiple times per polling cycle) that uses hours upon hours of CPU time. I currently have one that is misbehaving that has used 39 *hours* of CPU time in the past week or so since it started. My other instance, which was started at the same time, has used only 33 *minutes* of CPU time. Huge difference, obviously.

Oddly, the behaving instance isn't calling `gettransaction` in its polling loop *at all*, while the misbehaving instance is calling `gettransaction` many times. I understand that the number of confirmations changes over time, but do you really still need to be tracking it after 488 confirmations? At some point can't you just be confident that it's never going to unconfirm?

I'll apply the patch from #795 and see what impact it has. Thanks again.
> I understand that the number of confirmations changes over time, but do you really still need to be tracking it after 488 confirmations? At some point can't you just be confident that it's never going to unconfirm?

See my response to @undeath in the related PR #795 - once confirmed, we do not repeatedly call gettransaction for the txid. My reference above to not being able to cache etc. etc. is specifically that we keep doing the call once per loop, as long as it hasn't changed from 0 to 1 confs.

My best guess right now is there's a code path I'm missing, perhaps due to some unusual circumstance (e.g. what about a conflict? do you see anything about that, or anything else unusual, in the log for that bot?), but my last review didn't yet throw something up.
Apart from conflicts, the other edge case I see from the code:

Is it possible for you to get 100+ new entries in your listtransactions output in < 5 seconds?

For context, listtransactions lists by recency, all transactions relevant to your wallet. The nuance is that it can have multiple entries per transaction; e.g. if there is a send from utxo1 and a creation of utxo2, it will list it twice, once for each affected address. So while having 100 entries in 5 seconds is extremely unlikely for a "normal" user, it's not hard to find circumstances where it would happen in complete generality of use-cases.

I felt this was a reasonable trade-off when writing the code, my reasoning being: I don't want stupidly large listtransactions returns as you have to do that in every loop; and while in the extreme it's possible someone could get dangling transactions here, it'd be super-unlikely and they could just restart.
> what about a conflict? do you see anything about that, or anything else unusual, in the log for that bot?

@AdamISZ: I have an instance exhibiting the excessive CPU utilization again right now, and I don't see anything unusual in its log. Your patch has reduced the `gettransaction` calls to one per polling cycle, but the CPU usage is still crazy high. What I'm noticing now is that significant wall-clock time elapses between the call to `recvfrom` that receives the end of the `gettransaction` response and the following call to `epoll_wait`, a time during which the process would be CPU-bound in user-mode code. I am seeing about 1.2 seconds between receiving the `gettransaction` response and beginning the next wait, so something you're doing is very time-inefficient. Could it be your transaction parsing code?

> Is it possible for you to get 100+ new entries in your listtransactions output in < 5 seconds?

No, definitely not. I run each JoinMarket instance on its own dedicated Bitcoin Core wallet, so there's nothing else happening in the wallet, and I am most definitely not getting anywhere close to 100 CoinJoins per 5 seconds. (Closer to 100 CoinJoins per 5 *weeks*.)
From instrumenting the call to `BitcoinCoreInterface.get_deser_from_gettransaction` (which calls the function `stream_deserialize` in python-bitcointx's class `CTransaction` - see [here](https://github.com/Simplexum/python-bitcointx/blob/d945bf764a3f585e2bf075237a78a708bffefde7/bitcointx/core/__init__.py#L1089) i see no more than ~ 5ms per call across a bunch of simple txs on regtest, on my laptop ("i7-7700HQ CPU @ 2.80GHz × 8" so it's fairly beefy, but still ... it's not as if there's parallelisation here).

But if I look into this with more variety of transactions, I still see pretty strong evidence that there isn't really a case where the deserialization routine uses significant CPU (and tbh I would definitely not expect it to). My intuition with this case was always that there's something funky going on at the network/rpc/operating system/disk call level, i.e. specific to something about the environment in which it's running. I know your recent analysis (and thanks for doing meticulous analysis again; it's incredibly helpful) leads to you to believe otherwise, but I'd really want to delve into any delta between the cases you've had where you don't see this and those where you do ... I'm increasingly not really believing there's a JM code reason for this.

As to my belief that the deserialization routine doesn't eat CPU, here's an example of what I mean; this is the output of [this](https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/8d122fce2ab7fab965825b0583a05f5ac5c3cf37/scripts/snicker/snicker-finder.py) script with the `-j` flag:

```
2021-02-05 13:28:16,084 [INFO]  Finished processing block: 669116
2021-02-05 13:28:20,017 [INFO]  Found Joinmarket coinjoin transaction: f7e2496e956422e4320838011ec4bec5ea90664dff2db8ef8ac2f89e36017629 in block: 669117
2021-02-05 13:28:20,110 [INFO]  Finished processing block: 669117
2021-02-05 13:28:24,272 [INFO]  Finished processing block: 669118
2021-02-05 13:28:28,408 [INFO]  Finished processing block: 669119
2021-02-05 13:28:33,814 [INFO]  Finished processing block: 669120
2021-02-05 13:28:38,500 [INFO]  Finished processing block: 669121
2021-02-05 13:28:41,656 [INFO]  Found SNICKER transaction: 2e66ffce4c0406415927acfc488f7bc1d996427cc6ba90dad909505a56af5fd1 in block: 669122
2021-02-05 13:28:43,067 [INFO]  Finished processing block: 669122
2021-02-05 13:28:44,294 [INFO]  Found Joinmarket coinjoin transaction: b3044511c67fa4c8752454e0b70b592bbbf8bfaef03cc094a885bf299e647459 in block: 669123
2021-02-05 13:28:47,997 [INFO]  Finished processing block: 669123
2021-02-05 13:28:53,512 [INFO]  Finished processing block: 669124
2021-02-05 13:28:55,391 [INFO]  Found Joinmarket coinjoin transaction: 422e0aec25e667eebb1c453b446b65726440623c69b2ab381681ede2afd7c514 in block: 669125
2021-02-05 13:28:57,964 [INFO]  Finished processing block: 669125
2021-02-05 13:29:03,302 [INFO]  Finished processing block: 669126
2021-02-05 13:29:08,117 [INFO]  Finished processing block: 669127
2021-02-05 13:29:13,056 [INFO]  Finished processing block: 669128
```
4-5 seconds per block appears typical ... Python after all is *not* fast so this isn't too bad really ... and blocks are all "full-ish" in this period; we're definitely dealing with transaction numbers in the thousands, and *every* transaction is deserialized, as well as other things being done (block serialization; pattern matching checks on the individual deserialized txs, basically). Even further, we may need to call `deserialize` more than once per transaction; see the algo used here:

https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/8d122fce2ab7fab965825b0583a05f5ac5c3cf37/jmbitcoin/jmbitcoin/blocks.py#L16-L42

Given that this experiment covers *all* the types of transactions we see, I think it should be reasonably reliable.

Huh, I just had a thought; maybe checkout the SNICKER PR #768 and just run the above script and see how fast it is? (`cd scripts/snicker; python snicker-finder.py -j 669100` (or any other block height to start; just quit it before it finishes if too slow).

> Huh, I just had a thought; maybe checkout the SNICKER PR #768 and just run the above script and see how fast it is?

```
2021-02-06 14:09:36,339 [INFO]  Finished processing block: 669116
2021-02-06 14:09:52,238 [INFO]  Found Joinmarket coinjoin transaction: f7e2496e956422e4320838011ec4bec5ea90664dff2db8ef8ac2f89e36017629 in block: 669117
2021-02-06 14:09:52,533 [INFO]  Finished processing block: 669117
2021-02-06 14:10:09,018 [INFO]  Finished processing block: 669118
2021-02-06 14:10:24,557 [INFO]  Finished processing block: 669119
2021-02-06 14:10:46,027 [INFO]  Finished processing block: 669120
2021-02-06 14:11:04,486 [INFO]  Finished processing block: 669121
2021-02-06 14:11:15,397 [INFO]  Found SNICKER transaction: 2e66ffce4c0406415927acfc488f7bc1d996427cc6ba90dad909505a56af5fd1 in block: 669122
2021-02-06 14:11:21,325 [INFO]  Finished processing block: 669122
2021-02-06 14:11:23,961 [INFO]  Found Joinmarket coinjoin transaction: b3044511c67fa4c8752454e0b70b592bbbf8bfaef03cc094a885bf299e647459 in block: 669123
2021-02-06 14:11:40,018 [INFO]  Finished processing block: 669123
2021-02-06 14:12:01,805 [INFO]  Finished processing block: 669124
2021-02-06 14:12:07,471 [INFO]  Found Joinmarket coinjoin transaction: 422e0aec25e667eebb1c453b446b65726440623c69b2ab381681ede2afd7c514 in block: 669125
2021-02-06 14:12:19,141 [INFO]  Finished processing block: 669125
2021-02-06 14:12:40,288 [INFO]  Finished processing block: 669126
2021-02-06 14:12:59,261 [INFO]  Finished processing block: 669127
2021-02-06 14:13:18,015 [INFO]  Finished processing block: 669128
```

It's certainly slower than it is on your machine, by a factor of about 4, but that's not altogether surprising, given that I'm running on an Intel Core 2 Quad Q6600, a 14-year-old CPU. Still, though, that doesn't explain 1.2 seconds to deserialize a single transaction, so I'm inclined to agree with you that the deserialization is not the CPU hog in my case.

I wish I knew anything about profiling Python code. I'm completely ignorant (by choice) about Python. Is there a Python profiler that can instrument CPU time spent per function call?
I realise I never answered the last Q in this conversation, presumably because I didn't know. While I use pdb for debugging from time to time, I've never even looked for a profiler. This is just what came up from ddg-ing but you can do that as well or better than I: https://docs.python.org/3/library/profile.html

Is this close-able do you think? Apparently we have reduced the overhead but also we're not sure if it's a problem for other users (it isn't for me).
It can be closed for now. I haven't noticed either of my YG instances churning the CPU in a while now. If it happens again, I'll look into profiling and reopen this report.
This is becoming untenable again. I restarted `yg-privacyenhanced.py` 11 hours ago, and just in that time it has used over 4 hours of CPU time. I'll investigate profiling it to see where it's spending all its time.


diff --git a/tagstudio/src/qt/modals/tag_search.py b/tagstudio/src/qt/modals/tag_search.py
index d35065675..fdd986ab5 100644
--- a/tagstudio/src/qt/modals/tag_search.py
+++ b/tagstudio/src/qt/modals/tag_search.py
@@ -52,7 +52,7 @@ def __init__(self, library: Library, exclude: list[int] = None, is_tag_chooser:
         self.search_field.setObjectName("searchField")
         self.search_field.setMinimumSize(QSize(0, 32))
         Translations.translate_with_setter(self.search_field.setPlaceholderText, "home.search_tags")
-        self.search_field.textEdited.connect(lambda: self.update_tags(self.search_field.text()))
+        self.search_field.textEdited.connect(lambda text: self.update_tags(text))
         self.search_field.returnPressed.connect(lambda: self.on_return(self.search_field.text()))

         self.scroll_contents = QWidget()
@@ -119,7 +119,7 @@ def __build_row_item_widget(self, tag: Tag):
             row.addWidget(add_button)
         return container

-    def construct_tag_button(self, query: str | None):
+    def build_create_tag_button(self, query: str | None):
         """Constructs a Create Tag Button."""
         container = QWidget()
         row = QHBoxLayout(container)
@@ -208,7 +208,7 @@ def update_tags(self, query: str | None = None):
         else:
             # If query doesnt exist add create button
             self.first_tag_id = None
-            c = self.construct_tag_button(query)
+            c = self.build_create_tag_button(query)
             self.scroll_layout.addWidget(c)
         self.search_field.setFocus()

diff --git a/tagstudio/src/qt/widgets/tag.py b/tagstudio/src/qt/widgets/tag.py
index efc288bbe..1af377e3c 100644
--- a/tagstudio/src/qt/widgets/tag.py
+++ b/tagstudio/src/qt/widgets/tag.py
@@ -4,10 +4,8 @@

 import math
-from pathlib import Path
 from types import FunctionType

-from PIL import Image
 from PySide6.QtCore import QEvent, Qt, Signal
 from PySide6.QtGui import QAction, QEnterEvent, QFontMetrics
 from PySide6.QtWidgets import (
@@ -93,10 +91,6 @@ def leaveEvent(self, event: QEvent) -> None:  # noqa: N802

 class TagWidget(QWidget):
-    edit_icon_128: Image.Image = Image.open(
-        str(Path(__file__).parents[3] / "resources/qt/images/edit_icon_128.png")
-    ).resize((math.floor(14 * 1.25), math.floor(14 * 1.25)))
-    edit_icon_128.load()
     on_remove = Signal()
     on_click = Signal()
     on_edit = Signal()
@@ -126,20 +120,22 @@ def __init__(
         self.bg_button.setText(tag.name)
         if has_edit:
             edit_action = QAction(self)
-            Translations.translate_qobject(edit_action, "generic.edit")
+            edit_action.setText(Translations.translate_formatted("generic.edit"))
             edit_action.triggered.connect(on_edit_callback)
             edit_action.triggered.connect(self.on_edit.emit)
             self.bg_button.addAction(edit_action)
         # if on_click_callback:
         self.bg_button.setContextMenuPolicy(Qt.ContextMenuPolicy.ActionsContextMenu)

+        # TODO: This currently doesn't work in "Add Tag" menus. Either fix this or
+        # disable it in that context.
         search_for_tag_action = QAction(self)
-        Translations.translate_qobject(search_for_tag_action, "tag.search_for_tag")
+        search_for_tag_action.setText(Translations.translate_formatted("tag.search_for_tag"))
         search_for_tag_action.triggered.connect(self.on_click.emit)
         self.bg_button.addAction(search_for_tag_action)
-        add_to_search_action = QAction(self)
-        Translations.translate_qobject(add_to_search_action, "tag.add_to_search")
-        self.bg_button.addAction(add_to_search_action)
+        # add_to_search_action = QAction(self)
+        # add_to_search_action.setText(Translations.translate_formatted("tag.add_to_search"))
+        # self.bg_button.addAction(add_to_search_action)

         self.inner_layout = QHBoxLayout()
         self.inner_layout.setObjectName("innerLayout")

[Bug]: Adding (or searching for) tags gets increasingly slower in the same session (running process)
### Checklist

- [x] I am using an up-to-date version.
- [x] I have read the [documentation](https://github.com/TagStudioDev/TagStudio/blob/main/docs/index.md).
- [x] I have searched existing [issues](https://github.com/TagStudioDev/TagStudio/issues).

### TagStudio Version

main

### Operating System & Version

Windows 10

### Description

Updating a library, adding tags to new entries gets increasingly slower over time.

Some details in hopes they help:
Library has around 300~ tags, 1200~ existing entries
Slowness becomes pretty much unbearable (tag search takes >5 seconds) after like 5-6 new entries, which each receive maybe 8~ tags.
Closing the library (keeping the application open) and reopening the library does not help.
Even clearing the search_field input, causing it to search for *all tags* takes seconds after use, whereas this is near instant on a fresh start of the process.
Closing the process and reopening, then performing the exact same search is 10x faster

### Attempted fixes / debugging information

I've noticed search gets run for every keypress in the search_field, which I've made a local change for using QTimer as a debounce timer but that didn't make noticeable improvements (I can possibly PR this if that seems interesting to you).
Essentially it waits 225ms after a keypress to perform the search, getting canceled by every subsequent keypress, so hopefully when the name is already known by the user, we can perform a single tag search, at the end.

I suspect there's a memory leak somewhere, or an infinitely growing list that is making every subsequent iteration take increasingly longer, but I'm not  familiar enough with the project to find this.

Also the logs didn't seem to provide much useful information to debug this.

### Reproduction

I made a reproduction, all tags are obfuscated (replaced with numbers) because the data is personal.
All entries are replaced with empty `.txt` files

https://drive.google.com/file/d/1QZ86CO6JzLVMQlm8kZRk9675_-l2q98o/view?usp=sharing

Note: I think the size of the tags or the size of the search matters in reproducing the issue, the real tags are about 8-12 characters wide.

1. Open the library (I noticed the `folders` contains an absolute path, so I'm not sure the repro is sharable without some manual editing ?)
2. Add these tags to each entry separately:
```
12
23
34
45
56
78
89
90
```
*using the keyboard, pressing enter inbetween, no cursor is involved other than pressing "Add Tags"
3. By the 8th~ entry you will notice it takes increasingly longer to complete all these steps for each subsequent entry
[Bug]: Adding (or searching for) tags gets increasingly slower in the same session (running process)
### Checklist

- [x] I am using an up-to-date version.
- [x] I have read the [documentation](https://github.com/TagStudioDev/TagStudio/blob/main/docs/index.md).
- [x] I have searched existing [issues](https://github.com/TagStudioDev/TagStudio/issues).

### TagStudio Version

main

### Operating System & Version

Windows 10

### Description

Updating a library, adding tags to new entries gets increasingly slower over time.

Some details in hopes they help:
Library has around 300~ tags, 1200~ existing entries
Slowness becomes pretty much unbearable (tag search takes >5 seconds) after like 5-6 new entries, which each receive maybe 8~ tags.
Closing the library (keeping the application open) and reopening the library does not help.
Even clearing the search_field input, causing it to search for *all tags* takes seconds after use, whereas this is near instant on a fresh start of the process.
Closing the process and reopening, then performing the exact same search is 10x faster

### Attempted fixes / debugging information

I've noticed search gets run for every keypress in the search_field, which I've made a local change for using QTimer as a debounce timer but that didn't make noticeable improvements (I can possibly PR this if that seems interesting to you).
Essentially it waits 225ms after a keypress to perform the search, getting canceled by every subsequent keypress, so hopefully when the name is already known by the user, we can perform a single tag search, at the end.

I suspect there's a memory leak somewhere, or an infinitely growing list that is making every subsequent iteration take increasingly longer, but I'm not  familiar enough with the project to find this.

Also the logs didn't seem to provide much useful information to debug this.

### Reproduction

I made a reproduction, all tags are obfuscated (replaced with numbers) because the data is personal.
All entries are replaced with empty `.txt` files

https://drive.google.com/file/d/1QZ86CO6JzLVMQlm8kZRk9675_-l2q98o/view?usp=sharing

Note: I think the size of the tags or the size of the search matters in reproducing the issue, the real tags are about 8-12 characters wide.

1. Open the library (I noticed the `folders` contains an absolute path, so I'm not sure the repro is sharable without some manual editing ?)
2. Add these tags to each entry separately:
```
12
23
34
45
56
78
89
90
```
*using the keyboard, pressing enter inbetween, no cursor is involved other than pressing "Add Tags"
3. By the 8th~ entry you will notice it takes increasingly longer to complete all these steps for each subsequent entry

I've been able to reproduce this, and it does indeed look like memory usage is increasing the more searches are preformed
After *much* debugging, I believe I've narrowed this down to the `translate_qobject()` calls inside the `TagWidget` objects that are use for the context menus. When removing these translation calls or replacing them with `translate_formatted()` calls, the issue is resolved for me. I'll open up a quick PR and hope that this fixes the issue in your case as well
I've been able to reproduce this, and it does indeed look like memory usage is increasing the more searches are preformed
After *much* debugging, I believe I've narrowed this down to the `translate_qobject()` calls inside the `TagWidget` objects that are use for the context menus. When removing these translation calls or replacing them with `translate_formatted()` calls, the issue is resolved for me. I'll open up a quick PR and hope that this fixes the issue in your case as well

Priority: Critical
Type: UI/UX
TagStudio: Tags


diff --git a/python-ecosys/requests/requests/__init__.py b/python-ecosys/requests/requests/__init__.py
index a9a183619..2951035f7 100644
--- a/python-ecosys/requests/requests/__init__.py
+++ b/python-ecosys/requests/requests/__init__.py
@@ -46,6 +46,8 @@ def request(
 ):
     if headers is None:
         headers = {}
+    else:
+        headers = headers.copy()

     redirect = None  # redirection url, None means no redirection
     chunked_data = data and getattr(data, "__next__", None) and not getattr(data, "__len__", None)

diff --git a/python-ecosys/requests/test_requests.py b/python-ecosys/requests/test_requests.py
index 513e533a3..ac77291b0 100644
--- a/python-ecosys/requests/test_requests.py
+++ b/python-ecosys/requests/test_requests.py
@@ -102,11 +102,11 @@ def chunks():

 def test_overwrite_get_headers():
     response = requests.request(
-        "GET", "http://example.com", headers={"Connection": "keep-alive", "Host": "test.com"}
+        "GET", "http://example.com", headers={"Host": "test.com", "Connection": "keep-alive"}
     )

     assert response.raw._write_buffer.getvalue() == (
-        b"GET / HTTP/1.0\r\n" + b"Host: test.com\r\n" + b"Connection: keep-alive\r\n\r\n"
+        b"GET / HTTP/1.0\r\n" + b"Connection: keep-alive\r\n" + b"Host: test.com\r\n\r\n"
     ), format_message(response)

@@ -145,6 +145,14 @@ def chunks():
     ), format_message(response)

+def test_do_not_modify_headers_argument():
+    global do_not_modify_this_dict
+    do_not_modify_this_dict = {}
+    requests.request("GET", "http://example.com", headers=do_not_modify_this_dict)
+
+    assert do_not_modify_this_dict == {}, do_not_modify_this_dict
+
+
 test_simple_get()
 test_get_auth()
 test_get_custom_header()
@@ -153,3 +161,4 @@ def chunks():
 test_overwrite_get_headers()
 test_overwrite_post_json_headers()
 test_overwrite_post_chunked_data_headers()
+test_do_not_modify_headers_argument()

SECURITY: Requests module leaks passwords & usernames for HTTP Basic Auth
While looking at the MicroPython `requests` module (on the git HEAD), I noticed this:

If you make a request with HTTP basic auth (a username/password) and did not specify a headers dict, then I believe the username and password would be added to the default headers to be used for **every subsequent HTTP request**.  Even if that request is to a completely different server, which you don't trust with your username and password.  That's probably not a good idea.

I haven't verified this, it's just from reading the code, but someone should probably look into it.

This is because there is `headers={}` in the function prototype, specifying a default for the `headers` parameter.  But (at least in cPython) that same dictionary will get reused for every call that doesn't explicitly specify a `headers` parameter.  So if the function changes the `headers` dictionary - such as by adding an `Authorization` header - that change will be there for every future call of the function.  This is a known dangerous part of the Python language, you're not the first people to write this kind of bug.

To fix this, you could keep the auth headers separate from the `headers` variable.  Something like this (totally untested!) commit: https://github.com/jonfoster/micropython-lib/commit/92e9b2208814faa22ba3ff2a19cbc8e0c5210a47 - feel free to use that as a starting point.
SECURITY: Requests module leaks passwords & usernames for HTTP Basic Auth
While looking at the MicroPython `requests` module (on the git HEAD), I noticed this:

If you make a request with HTTP basic auth (a username/password) and did not specify a headers dict, then I believe the username and password would be added to the default headers to be used for **every subsequent HTTP request**.  Even if that request is to a completely different server, which you don't trust with your username and password.  That's probably not a good idea.

I haven't verified this, it's just from reading the code, but someone should probably look into it.

This is because there is `headers={}` in the function prototype, specifying a default for the `headers` parameter.  But (at least in cPython) that same dictionary will get reused for every call that doesn't explicitly specify a `headers` parameter.  So if the function changes the `headers` dictionary - such as by adding an `Authorization` header - that change will be there for every future call of the function.  This is a known dangerous part of the Python language, you're not the first people to write this kind of bug.

To fix this, you could keep the auth headers separate from the `headers` variable.  Something like this (totally untested!) commit: https://github.com/jonfoster/micropython-lib/commit/92e9b2208814faa22ba3ff2a19cbc8e0c5210a47 - feel free to use that as a starting point.

The MicroPython way aiui would be to mirror CPython's solution to this problem, which uses a `None` default value and then sets it to an empty dict at runtime:

https://github.com/psf/requests/blob/0e322af87745eff34caffe4df68456ebc20d9068/src/requests/models.py#L258-L276

And I see this is exactly what #823 does. That'll teach me not to look at PRs first 😆
The MicroPython way aiui would be to mirror CPython's solution to this problem, which uses a `None` default value and then sets it to an empty dict at runtime:

https://github.com/psf/requests/blob/0e322af87745eff34caffe4df68456ebc20d9068/src/requests/models.py#L258-L276

And I see this is exactly what #823 does. That'll teach me not to look at PRs first 😆


diff --git a/.gitignore b/.gitignore
index c1a2a16..0f72b94 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,5 +5,7 @@
 /cv/testImages/kanti_img2.jpg
 /cv/testImages/kanti_img1.jpeg
 /cv/testImages/kanti_img2.jpeg
+/cv/testImages/kanti_telegram_compressed_1.jpg
+/cv/testImages/kanti_telegram_compressed_2.jpg
 /cv/__pycache__/
 /cv/Models/__pycache__/
\ No newline at end of file
diff --git a/backend/app/core/cv_result.py b/backend/app/core/cv_result.py
index 923d430..d2dd06c 100644
--- a/backend/app/core/cv_result.py
+++ b/backend/app/core/cv_result.py
@@ -12,11 +12,12 @@ def __init__(self, number, date_of_birth):

 class Exam:
-    def __init__(self, year, subject, score, confidence, exercises):
+    def __init__(self, year, subject, total_score, total_score_confidence, exercises):
         self.year = year
         self.subject = subject
-        self.score = score
-        self.confidence = confidence
+        self.total_score = total_score
+        #This is the confidence of the total score reached (how many points there are in the total cell)
+        self.total_score_confidence = total_score_confidence
         self.exercises = exercises

     def calc_exercises_score(self):
diff --git a/cv/DigitRecognizer.py b/cv/DigitRecognizer.py
index 9ee434b..ccceb95 100644
--- a/cv/DigitRecognizer.py
+++ b/cv/DigitRecognizer.py
@@ -11,11 +11,10 @@
 sys.path.append(sys.path[0] + '/..')
 from backend.app.core.cv_result import *

-
 class DigitRecognizer:

     global DEBUG_MODE
-
+
     def __init__(self, debug_mode=False):
         # initialize any variables
         global DEBUG_MODE
@@ -51,15 +50,15 @@ def recognize_digits_in_photo(self, photo):
         else:
             segmentation = DocumentSegmentationCV()

-        #aligned_photo = segmentation.align_document(photo)
+        aligned_photo = segmentation.align_document(photo)

-        #if(DEBUG_MODE):
-            #self.debug_display_image('aligned',aligned_photo)
+        if(DEBUG_MODE):
+            self.debug_display_image('aligned',aligned_photo)

-        grid_mask, grid = self.find_grid_in_image(photo)
+        grid_mask, grid = self.find_grid_in_image(aligned_photo)

         if(DEBUG_MODE):
-            self.debug_display_image("grid_only", grid)
+            self.debug_display_image("grid_mask", grid_mask)

         grid_cells, column_count = self.get_grid_cells(grid, grid_mask)

@@ -82,20 +81,21 @@ def recognize_digits_in_photo(self, photo):
             result_cell = grid_cells[index + 2*column_count]

             if(index % column_count == 0):
-                print("First, 'Erreichte Punkte text'")
+                #print("First, 'Erreichte Punkte text'")
                 #TODO: OCR over header_cell, points_cell and result_cell and check that they say the right thing.
+                pass

             if(index % column_count > 0 and index % column_count < column_count-1):
-                print("Handwritten Cell")
+                #print("Handwritten Cell")

-                if DEBUG_MODE:
+                if DEBUG_MODE and index == 1:
                     self.debug_display_image("cell", result_cell)

                 pred_class_label, pred_confidence = self.predict_handwritten_cell(result_cell, class_labels, model)

                 exercises.append(Exercise(index, pred_class_label, pred_confidence, "?"))
             elif(index % column_count != 0):
-                print("Last Handwritten Cell, 'Total'")
+                #print("Last Handwritten Cell, 'Total'")

                 total_score, total_score_confidence = self.predict_double_number(result_cell, class_labels, model)

@@ -103,7 +103,7 @@ def recognize_digits_in_photo(self, photo):

         cv_res = CVResult(Candidate("?", "?"), exam=exam, result_validated=False)

-        if(exam.score == exam.calc_total_score()):
+        if(exam.total_score == exam.calc_exercises_score()):
             cv_res.result_validated = True
         return cv_res

@@ -125,8 +125,14 @@ def predict_double_number(self, original_cell, class_labels, model):
             widthToAdd = int((totalHeight - w) / 2)
             res = cv2.copyMakeBorder(res, 20, 20, widthToAdd, widthToAdd, cv2.BORDER_CONSTANT, value=(0,0,0))
             class_label, class_confidence = self.predict_handwritten_cell(res, class_labels, model, False)
+
+            #cv2.imshow("Handwritten Digit", res)
+            #cv2.waitKey(0)
             found_numbers.append((class_label, class_confidence))

+        if(found_numbers == 0):
+            return 0, 0
+
         result = ""
         confidence_sum = 0
         for number, confidence in found_numbers:
@@ -153,8 +159,8 @@ def predict_handwritten_cell(self, original_cell, class_labels, model, do_thresh
             tcell = np.expand_dims(tcell, axis=(0, -1))
             tcell = utils.normalize_images(tcell)

-            prediction = model.predict(tcell)
-            print(prediction)
+            prediction = model.predict(tcell, verbose=0)
+            #print(prediction)

             # Get the index of the highest probability
             predicted_class_index = np.argmax(prediction)
@@ -163,7 +169,7 @@ def predict_handwritten_cell(self, original_cell, class_labels, model, do_thresh
             predicted_class_label = class_labels[predicted_class_index]

             # Print the predicted class label
-            print("The predicted class is:", predicted_class_label)
+            #print("The predicted class is:", predicted_class_label)

             return int(predicted_class_label), prediction[0][predicted_class_index]

@@ -194,27 +200,37 @@ def find_grid_in_image(self, image):
         #The grid might be a bit warped so we want to fix this.
         out = self.fix_perspective(out, best_cnt)

+        #Resize the grid to scale to our wanted reference
+        width = out.shape[1]
+        wanted_width = 2000
+        scale_percent = wanted_width / width
+
+        out = cv2.resize(out, (int(out.shape[1] * scale_percent), int(out.shape[0] * scale_percent)))
+
         if(DEBUG_MODE):
-            self.debug_display_image("Grid, perspective fixed", out)
+            self.debug_display_image("Grid, perspective fixed and resized", out)

         #Out is already crayscale so we don't need to convert to grey but we need to blur it
-        blur = cv2.GaussianBlur(out, (5,5), 0)
+        blur = cv2.GaussianBlur(out, (7,7), 0)

         #Apply adaptive threshold so we have independent illumination
         thresh = cv2.adaptiveThreshold(blur, 255, 1, 1, 11, 2)

-        horizontal = self.get_lines(thresh.copy(), (50,1), 100)
-        vertical = self.get_lines(thresh.copy(), (1, 40), 80, False)
+        horizontal = self.get_lines(thresh.copy(), (70,1), 80)
+        vertical = self.get_lines(thresh.copy(), (1, 50), 70, False)

         eroded = cv2.bitwise_or(horizontal, vertical)

         if DEBUG_MODE:
-            self.debug_display_image("grid, before blur", eroded)
+            self.debug_display_image("grid mask, before blur", eroded)

         #Blur the result a little bit so the lines are more prevalent
         cv2.blur(eroded, (7,7), eroded)
         _, eroded = cv2.threshold(eroded, 100, 255, cv2.THRESH_BINARY) #we can take anything that isn't really black.

+        if DEBUG_MODE:
+            self.debug_display_image("grid mask, after blur", eroded)
+
         return eroded, out

     def get_grid_cells(self, grid, grid_mask):
@@ -230,11 +246,11 @@ def zoom_border(image, zoom):
             return image

         #Get vertical lines only
-        vert = self.get_lines(grid_mask.copy(), (1, 40), 80, False)
+        vert = self.get_lines(grid_mask.copy(), (1, 50), 70, False)

         #Zoom into the image so the outer borders are gone
         vert = zoom_border(vert, 1.1)
-
+
         contours, _ = cv2.findContours(vert, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

         column_count = len(contours)+1
@@ -242,6 +258,9 @@ def zoom_border(image, zoom):
         result_cells = []
         invert = 255 - grid_mask

+        if(DEBUG_MODE):
+            self.debug_display_image("inverted", invert)
+
         #Find contours of inverted
         contours, _ = cv2.findContours(invert, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

@@ -252,11 +271,11 @@ def zoom_border(image, zoom):
         row = []
         for (i, c) in enumerate(contours, 1):
             area = cv2.contourArea(c)
-            if area > 4000:
+            if area > 3000:
                 row.append(c)
                 if i % column_count == 0:
-                    (contours, _) = imutils_contours.sort_contours(row, method="left-to-right")
-                    grid_rows.append(contours)
+                    (conts, _) = imutils_contours.sort_contours(row, method="left-to-right")
+                    grid_rows.append(conts)
                     row = []

         if len(grid_rows) != 3 or len(row) != 0:
@@ -283,8 +302,6 @@ def get_lines(self, mat, kernel, min_line_size, is_horizontal = True):
         structure = cv2.getStructuringElement(cv2.MORPH_RECT, kernel)
         mat = cv2.erode(mat, structure)

-        #self.debug_display_image("matuneroded",mat)
-
         #The horizontal / vertical structures have to be wide enough to be a line.
         contours, _ = cv2.findContours(mat, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
         for c in contours:
@@ -293,10 +310,19 @@ def get_lines(self, mat, kernel, min_line_size, is_horizontal = True):
                 cv2.drawContours(mat, [c], -1, (0,0,0), -1)
             if not is_horizontal and h < min_line_size:
                  cv2.drawContours(mat, [c], -1, (0,0,0), -1)
+
+        if(DEBUG_MODE):
+            self.debug_display_image("eroded structures",mat)
+
+        mat = cv2.dilate(mat, structure, iterations=50)

-        #self.debug_display_image("matuneroded",mat)
+        if(DEBUG_MODE):
+            self.debug_display_image("dilated structures",mat)

-        mat = cv2.dilate(mat, structure, iterations=4)
+        mat = cv2.dilate(mat, cv2.getStructuringElement(cv2.MORPH_RECT, (2,2)))
+
+        if(DEBUG_MODE):
+            self.debug_display_image("'blurred', dilated structures",mat)
         return mat

     def find_largest_contours(self, contours):
diff --git a/cv/app.py b/cv/app.py
index 78c985c..3b6402c 100644
--- a/cv/app.py
+++ b/cv/app.py
@@ -1,13 +1,38 @@
 import cv2
 from DigitRecognizer import DigitRecognizer
+import time

-image = cv2.imread("cv/testImages/white_with_watermark.jpg")
-#image = cv2.imread("cv/testImages/small_nrs.png")
-#image = cv2.imread("cv/testImages/kanti_img1.jpeg")
-#image = cv2.imread("cv/testImages/kanti_img2.jpeg")
-#image = cv2.imread("cv/testImages/corner_outside_frame.jpg",1)
+images = []

+images.append(cv2.imread("cv/testImages/kanti_img1.jpeg"))
+images.append(cv2.imread("cv/testImages/kanti_img2.jpeg"))
+images.append(cv2.imread("cv/testImages/kanti_telegram_compressed_1.jpg"))
+images.append(cv2.imread("cv/testImages/kanti_telegram_compressed_2.jpg"))
+images.append(cv2.imread("cv/testImages/straight.jpg"))
+images.append(cv2.imread("cv/testImages/perspective.jpg"))
+images.append(cv2.imread("cv/testImages/crooked.jpg"))
+images.append(cv2.imread("cv/testImages/lighting.jpg"))
+images.append(cv2.imread("cv/testImages/mirror.jpg"))
+images.append(cv2.imread("cv/testImages/multiple.jpg"))
+images.append(cv2.imread("cv/testImages/rug.jpg"))
+images.append(cv2.imread("cv/testImages/wavy.jpg"))
+images.append(cv2.imread("cv/testImages/weird_bg.jpg"))
+images.append(cv2.imread("cv/testImages/crunched.jpg"))

-recognizer = DigitRecognizer()
+
+recognizer = DigitRecognizer(False)
 #cv2.imshow('image', image)
-exam = recognizer.recognize_digits_in_photo(image)
\ No newline at end of file
+for index, image in enumerate(images):
+    start = time.time()
+    try:
+        cv_result = recognizer.recognize_digits_in_photo(image)
+        print("Result validated: " + str(cv_result.result_validated))
+
+        print(', '.join([str(exercise.score) for exercise in cv_result.exam.exercises]) + " | Total: " + str(cv_result.exam.total_score))
+
+        print("IMAGE " + str(index+1) + " PASSED")
+    except:
+        print("IMAGE " + str(index+1) + " DID NOT PASS")
+        pass
+    end = time.time()
+    print(end - start)

diff --git a/cv/testImages/crooked.jpg b/cv/testImages/crooked.jpg
new file mode 100644
index 0000000..2fade6f
Binary files /dev/null and b/cv/testImages/crooked.jpg differ
diff --git a/cv/testImages/crunched.jpg b/cv/testImages/crunched.jpg
new file mode 100644
index 0000000..3c214c5
Binary files /dev/null and b/cv/testImages/crunched.jpg differ
diff --git a/cv/testImages/lighting.jpg b/cv/testImages/lighting.jpg
new file mode 100644
index 0000000..53c97fc
Binary files /dev/null and b/cv/testImages/lighting.jpg differ
diff --git a/cv/testImages/mirror.jpg b/cv/testImages/mirror.jpg
new file mode 100644
index 0000000..c904272
Binary files /dev/null and b/cv/testImages/mirror.jpg differ
diff --git a/cv/testImages/multiple.jpg b/cv/testImages/multiple.jpg
new file mode 100644
index 0000000..bcb3a8c
Binary files /dev/null and b/cv/testImages/multiple.jpg differ
diff --git a/cv/testImages/perspective.jpg b/cv/testImages/perspective.jpg
new file mode 100644
index 0000000..01fedee
Binary files /dev/null and b/cv/testImages/perspective.jpg differ
diff --git a/cv/testImages/rug.jpg b/cv/testImages/rug.jpg
new file mode 100644
index 0000000..0ed13ad
Binary files /dev/null and b/cv/testImages/rug.jpg differ
diff --git a/cv/testImages/straight.jpg b/cv/testImages/straight.jpg
new file mode 100644
index 0000000..78b8dcf
Binary files /dev/null and b/cv/testImages/straight.jpg differ
diff --git a/cv/testImages/wavy.jpg b/cv/testImages/wavy.jpg
new file mode 100644
index 0000000..9e090c7
Binary files /dev/null and b/cv/testImages/wavy.jpg differ
diff --git a/cv/testImages/weird_bg.jpg b/cv/testImages/weird_bg.jpg
new file mode 100644
index 0000000..c9360e2
Binary files /dev/null and b/cv/testImages/weird_bg.jpg differ

Fix CV - test on more samples and make adjustments to make it more robust


diff --git a/bangazonapi/models/customer.py b/bangazonapi/models/customer.py
index a6d4cdd..61bf2e6 100644
--- a/bangazonapi/models/customer.py
+++ b/bangazonapi/models/customer.py
@@ -14,4 +14,12 @@ def recommends(self):

     @recommends.setter
     def recommends(self, value):
-        self.__recommends = value
\ No newline at end of file
+        self.__recommends = value
+
+    @property
+    def store(self):
+        return self.__store
+
+    @store.setter
+    def store(self, value):
+        self.__store = value
\ No newline at end of file
diff --git a/bangazonapi/models/store.py b/bangazonapi/models/store.py
index af288df..e92b3a8 100644
--- a/bangazonapi/models/store.py
+++ b/bangazonapi/models/store.py
@@ -5,4 +5,10 @@ class Store(models.Model):
     description = models.TextField()
     seller = models.ForeignKey('Customer', on_delete=models.CASCADE)  # Linking to Customer

-    # Use reverse relationship in views to access products
+    # Define a many-to-many relationship through a separate table
+    products = models.ManyToManyField('Product', through='StoreProduct', related_name='stores')
+
+class StoreProduct(models.Model):
+    store = models.ForeignKey(Store, on_delete=models.CASCADE)
+    product = models.ForeignKey('Product', on_delete=models.CASCADE)
+    # Additional fields if needed
\ No newline at end of file
diff --git a/bangazonapi/views/profile.py b/bangazonapi/views/profile.py
index 7c5d4dd..9d23753 100644
--- a/bangazonapi/views/profile.py
+++ b/bangazonapi/views/profile.py
@@ -8,12 +8,12 @@
 from rest_framework.permissions import IsAuthenticatedOrReadOnly
 from rest_framework.response import Response
 from rest_framework.viewsets import ViewSet
-from bangazonapi.models import Order, Customer, Product
+from bangazonapi.models import Order, Customer, Product, Store
 from bangazonapi.models import OrderProduct, Favorite
 from bangazonapi.models import Recommendation
 from .product import ProductSerializer
 from .order import OrderSerializer
-
+from .store import StoreSerializer

 class Profile(ViewSet):
     """Request handlers for user profile info in the Bangazon Platform"""
@@ -87,6 +87,9 @@ def list(self, request):
             current_user.recommends = Recommendation.objects.filter(
                 recommender=current_user
             )
+            current_user.store = Store.objects.get(
+                seller_id = current_user
+            )

             serializer = ProfileSerializer(
                 current_user, many=False, context={"request": request}
@@ -397,6 +400,7 @@ class ProfileSerializer(serializers.ModelSerializer):

     user = UserSerializer(many=False)
     recommends = RecommenderSerializer(many=True)
+    store = StoreSerializer(many=False, read_only=True)

     class Meta:
         model = Customer
@@ -408,6 +412,7 @@ class Meta:
             "address",
             "payment_types",
             "recommends",
+            "store",
         )
         depth = 1

diff --git a/bangazonapi/views/store.py b/bangazonapi/views/store.py
index d2bc926..906c803 100644
--- a/bangazonapi/views/store.py
+++ b/bangazonapi/views/store.py
@@ -11,7 +11,7 @@ class StoreSellerSerializer(serializers.ModelSerializer):

     class Meta:
         model = Customer
-        fields = ('first_name', 'last_name')
+        fields = ('first_name', 'last_name', 'id')

 class StoreProductSerializer(serializers.ModelSerializer):
     """JSON serializer for products in a store"""
@@ -28,7 +28,7 @@ class StoreSerializer(serializers.ModelSerializer):

     class Meta:
         model = Store
-        fields = ('id', 'name', 'description', 'seller', 'products')
+        fields = ('id', 'name', 'description', 'seller', 'products', 'seller_id')

 class StoreViewSet(ViewSet):
     """View for handling requests about stores"""
@@ -53,3 +53,12 @@ def create(self, request):
         print("Store Created with ID:", new_store.id)  # Debugging log for backend
         print("Serialized data:", serializer.data)  # Add this log for debugging
         return Response(serializer.data, status=status.HTTP_201_CREATED)
+
+    def retrieve(self, request, pk=None):
+        """GET a single store by ID"""
+        try:
+            store = Store.objects.get(pk=pk)
+            serializer = StoreSerializer(store)
+            return Response(serializer.data, status=status.HTTP_200_OK)
+        except Store.DoesNotExist:
+            return Response({'message': 'Store not found.'}, status=status.HTTP_404_NOT_FOUND)
\ No newline at end of file

Feature: Users can create a store in order to sell items
**Given** a user clicks on the profile icon in the navbar
**And** the user has not created a store yet
**When** the _Interested in selling?_ option is chosen
**Then** the user should see a form to create a store with a text field for store name, and a textarea for store description

**Given** a user has filled out the store form
**When** the **Save** button is clicked
**Then** a new store should be created for the user
**And** the user should be shown the **My Store** view


diff --git a/examples/ntlmrelayx.py b/examples/ntlmrelayx.py
index 5377a695e..e3efbbf81 100755
--- a/examples/ntlmrelayx.py
+++ b/examples/ntlmrelayx.py
@@ -57,10 +57,11 @@
 RELAY_SERVERS = []

 class MiniShell(cmd.Cmd):
-    def __init__(self, relayConfig, threads):
+    def __init__(self, relayConfig, threads, api_address):
         cmd.Cmd.__init__(self)

         self.prompt = 'ntlmrelayx> '
+        self.api_address = api_address
         self.tid = None
         self.relayConfig = relayConfig
         self.intro = 'Type help for list of commands'
@@ -108,7 +109,7 @@ def do_socks(self, line):
         '''

         headers = ["Protocol", "Target", "Username", "AdminStatus", "Port"]
-        url = "http://localhost:9090/ntlmrelayx/api/v1.0/relays"
+        url = "http://{}/ntlmrelayx/api/v1.0/relays".format(self.api_address)
         try:
             proxy_handler = ProxyHandler({})
             opener = build_opener(proxy_handler)
@@ -305,6 +306,9 @@ def stop_servers(threads):
                                                                              'SMB Server (16 hex bytes long. eg: 1122334455667788)')
     parser.add_argument('-socks', action='store_true', default=False,
                         help='Launch a SOCKS proxy for the connection relayed')
+    parser.add_argument('-socks-address', default='127.0.0.1', help='SOCKS5 server address (also used for HTTP API)')
+    parser.add_argument('-socks-port', default=1080, type=int, help='SOCKS5 server port')
+    parser.add_argument('-http-api-port', default=9090, type=int, help='SOCKS5 HTTP API port')
     parser.add_argument('-wh','--wpad-host', action='store',help='Enable serving a WPAD file for Proxy Authentication attack, '
                                                                    'setting the proxy host to the one supplied.')
     parser.add_argument('-wa','--wpad-auth-num', action='store', type=int, default=1, help='Prompt for authentication N times for clients without MS16-077 installed '
@@ -471,8 +475,9 @@ def stop_servers(threads):
     threads = set()
     socksServer = None
     if options.socks is True:
+
         # Start a SOCKS proxy in the background
-        socksServer = SOCKS()
+        socksServer = SOCKS(server_address=(options.socks_address, options.socks_port), api_port=options.http_api_port)
         socksServer.daemon_threads = True
         socks_thread = Thread(target=socksServer.serve_forever)
         socks_thread.daemon = True
@@ -485,7 +490,7 @@ def stop_servers(threads):
     logging.info("Servers started, waiting for connections")
     try:
         if options.socks:
-            shell = MiniShell(c, threads)
+            shell = MiniShell(c, threads, api_address='{}:{}'.format(options.socks_address, options.http_api_port))
             shell.cmdloop()
         else:
             sys.stdin.read()
diff --git a/impacket/examples/ntlmrelayx/servers/socksserver.py b/impacket/examples/ntlmrelayx/servers/socksserver.py
index 2fe53989e..616142e54 100644
--- a/impacket/examples/ntlmrelayx/servers/socksserver.py
+++ b/impacket/examples/ntlmrelayx/servers/socksserver.py
@@ -243,36 +243,40 @@ def activeConnectionsWatcher(server):
             LOG.info('Relay connection for %s at %s(%d) already exists. Discarding' % (userName, target, port))
             client.killConnection()

-def webService(server):
-    from flask import Flask, jsonify

-    app = Flask(__name__)
+def webService(addr, port):
+    def _webService(server):
+        from flask import Flask, jsonify

-    log = logging.getLogger('werkzeug')
-    log.setLevel(logging.ERROR)
+        app = Flask(__name__)

-    @app.route('/')
-    def index():
-        print(server.activeRelays)
-        return "Relays available: %s!" % (len(server.activeRelays))
+        log = logging.getLogger('werkzeug')
+        log.setLevel(logging.ERROR)

-    @app.route('/ntlmrelayx/api/v1.0/relays', methods=['GET'])
-    def get_relays():
-        relays = []
-        for target in server.activeRelays:
-            for port in server.activeRelays[target]:
-                for user in server.activeRelays[target][port]:
-                    if user != 'data' and user != 'scheme':
-                        protocol = server.activeRelays[target][port]['scheme']
-                        isAdmin = server.activeRelays[target][port][user]['isAdmin']
-                        relays.append([protocol, target, user, isAdmin, str(port)])
-        return jsonify(relays)
+        @app.route('/')
+        def index():
+            print(server.activeRelays)
+            return "Relays available: %s!" % (len(server.activeRelays))

-    @app.route('/ntlmrelayx/api/v1.0/relays', methods=['GET'])
-    def get_info(relay):
-        pass
+        @app.route('/ntlmrelayx/api/v1.0/relays', methods=['GET'])
+        def get_relays():
+            relays = []
+            for target in server.activeRelays:
+                for port in server.activeRelays[target]:
+                    for user in server.activeRelays[target][port]:
+                        if user != 'data' and user != 'scheme':
+                            protocol = server.activeRelays[target][port]['scheme']
+                            isAdmin = server.activeRelays[target][port][user]['isAdmin']
+                            relays.append([protocol, target, user, isAdmin, str(port)])
+            return jsonify(relays)

-    app.run(host='0.0.0.0', port=9090)
+        @app.route('/ntlmrelayx/api/v1.0/relays', methods=['GET'])
+        def get_info(relay):
+            pass
+
+        app.run(host=addr, port=port)
+
+    return _webService

 class SocksRequestHandler(socketserver.BaseRequestHandler):
     def __init__(self, request, client_address, server):
@@ -453,8 +457,8 @@ def handle(self):

 class SOCKS(socketserver.ThreadingMixIn, socketserver.TCPServer):
-    def __init__(self, server_address=('0.0.0.0', 1080), handler_class=SocksRequestHandler):
-        LOG.info('SOCKS proxy started. Listening at port %d', server_address[1] )
+    def __init__(self, server_address=('127.0.0.1', 1080), handler_class=SocksRequestHandler, api_port=9090):
+        LOG.info('SOCKS proxy started. Listening on %s:%d', server_address[0], server_address[1])

         self.activeRelays = {}
         self.socksPlugins = {}
@@ -476,7 +480,7 @@ def __init__(self, server_address=('0.0.0.0', 1080), handler_class=SocksRequestH
         self.__timer = RepeatedTimer(KEEP_ALIVE_TIMER, keepAliveTimer, self)

         # Let's start our RESTful API
-        self.restAPI = Thread(target=webService, args=(self, ))
+        self.restAPI = Thread(target=webService(server_address[0], api_port), args=(self, ))
         self.restAPI.daemon = True
         self.restAPI.start()

SOCKS server listens on all interfaces by default
### Configuration
impacket version:  v0.10.1.dev1+20230207.182628.6cd68a05
Python version:  3.9.2
Target OS:  NA

By default, the SOCKS server used by ntlmrelayx.py listens on all interfaces (0.0.0.0)1080, which is dangerous.

Please see:
https://github.com/fortra/impacket/blob/6328a9b74a1df675fb47d8cc3a626bddc1f49b1d/impacket/examples/ntlmrelayx/servers/socksserver.py#L456

Any malicious agent on the same network could take advantage of the listening SOCKS proxy.
This service should only be exposed on localhost by default, and a supplemental flag should be provided in order to make it listen "publicly".