Remove DB records for expired offers option.

basicswap/basicswap.py

@@ -150,7 +150,9 @@ from .basicswap_util import (
     VisibilityOverrideOptions,
     inactive_states,
 )
-
+from basicswap.db_util import (
+    remove_expired_data,
+)
 
 PROTOCOL_VERSION_SECRET_HASH = 1
 MINPROTO_VERSION_SECRET_HASH = 1
@@ -255,6 +257,8 @@ class BasicSwap(BaseApp):
         self._disabled_notification_types = self.settings.get('disabled_notification_types', [])
         self._keep_notifications = self.settings.get('keep_notifications', 50)
         self._show_notifications = self.settings.get('show_notifications', 10)
+        self._expire_db_records = self.settings.get('expire_db_records', False)
+        self._expire_db_records_after = self.settings.get('expire_db_records_after', 86400 * 7)  # Seconds
         self._notifications_cache = {}
         self._is_encrypted = None
         self._is_locked = None
@@ -4134,13 +4138,19 @@ class BasicSwap(BaseApp):
             if num_messages + num_removed > 0:
                 self.log.info('Expired {} / {} messages.'.format(num_removed, num_messages))
 
-            self.log.debug('TODO: Expire records from db')
-
         finally:
             if rpc_conn:
                 ci_part.close_rpc(rpc_conn)
             self.mxDB.release()
 
+    def expireDBRecords(self) -> None:
+        if self._is_locked is True:
+            self.log.debug('Not expiring database records while system locked')
+            return
+        if not self._expire_db_records:
+            return
+        remove_expired_data(self, self._expire_db_records_after)
+
     def checkAcceptedBids(self) -> None:
         # Check for bids stuck as accepted (not yet in-progress)
         if self._is_locked is True:
@@ -5969,6 +5979,7 @@ class BasicSwap(BaseApp):
 
             if now - self._last_checked_expired >= self.check_expired_seconds:
                 self.expireMessages()
+                self.expireDBRecords()
                 self.checkAcceptedBids()
                 self._last_checked_expired = now
 
@@ -6054,6 +6065,14 @@ class BasicSwap(BaseApp):
                     settings_copy['debug_ui'] = new_value
                     settings_changed = True
 
+            if 'expire_db_records' in data:
+                new_value = data['expire_db_records']
+                ensure(type(new_value) == bool, 'New expire_db_records value not boolean')
+                if settings_copy.get('expire_db_records', False) != new_value:
+                    self._expire_db_records = new_value
+                    settings_copy['expire_db_records'] = new_value
+                    settings_changed = True
+
             if 'show_chart' in data:
                 new_value = data['show_chart']
                 ensure(type(new_value) == bool, 'New show_chart value not boolean')

basicswap/basicswap_util.py

@@ -306,6 +306,8 @@ def strBidState(state):
         return 'Request sent'
     if state == BidStates.BID_REQUEST_ACCEPTED:
         return 'Request accepted'
+    if state == BidStates.BID_STATE_UNKNOWN:
+        return 'Unknown bid state'
 
     return 'Unknown' + ' ' + str(state)
 

basicswap/db_util.py

@@ -9,8 +9,7 @@ from .db import (
 )
 
 
-def remove_expired_data(self):
+def remove_expired_data(self, time_offset: int = 0):
-    self.log.warning('Removing expired data')
     now: int = self.getTime()
     try:
         session = self.openSession()
@@ -18,11 +17,11 @@ def remove_expired_data(self):
         active_bids_insert = self.activeBidsQueryStr(now, '', 'b2')
         query_str = f'''
                     SELECT o.offer_id FROM offers o
-                    WHERE o.expire_at <= :now AND 0 = (SELECT COUNT(*) FROM bids b2 WHERE b2.offer_id = o.offer_id AND {active_bids_insert})
+                    WHERE o.expire_at <= :expired_at AND 0 = (SELECT COUNT(*) FROM bids b2 WHERE b2.offer_id = o.offer_id AND {active_bids_insert})
                     '''
         num_offers = 0
         num_bids = 0
-        offer_rows = session.execute(query_str, {'now': now})
+        offer_rows = session.execute(query_str, {'expired_at': now - time_offset})
         for offer_row in offer_rows:
             num_offers += 1
             bid_rows = session.execute('SELECT bids.bid_id FROM bids WHERE bids.offer_id = :offer_id', {'offer_id': offer_row[0]})
@@ -48,9 +47,10 @@ def remove_expired_data(self):
             session.execute('DELETE FROM sentoffers WHERE sentoffers.offer_id = :offer_id', {'offer_id': offer_row[0]})
             session.execute('DELETE FROM actions WHERE actions.linked_id = :offer_id', {'offer_id': offer_row[0]})
             session.execute('DELETE FROM offers WHERE offers.offer_id = :offer_id', {'offer_id': offer_row[0]})
-            session.execute('DELETE FROM message_links WHERE linked_type = :type_ind AND linked_id = :linked_id', {'type_ind': int(Concepts.OFFER), 'offer_id': offer_row[0]})
+            session.execute('DELETE FROM message_links WHERE linked_type = :type_ind AND linked_id = :offer_id', {'type_ind': int(Concepts.OFFER), 'offer_id': offer_row[0]})
 
-        self.log.warning(f'Removed data for {num_offers} expired offers and {num_bids} bids.')
+        if num_offers > 0 or num_bids > 0:
+            self.log.info('Removed data for {} expired offer{} and {} bid{}.'.format(num_offers, 's' if num_offers != 1 else '', num_bids, 's' if num_bids != 1 else ''))
 
     finally:
         self.closeSession(session)

basicswap/templates/settings.html

@@ -308,7 +308,7 @@
                </div>
               </td>
              </tr>
-              <tr class="opacity-100 text-gray-500 dark:text-gray-100">
+             <tr class="opacity-100 text-gray-500 dark:text-gray-100">
               <td class="py-3 px-6 bold">Debug Mode UI</td>
               <td class="py-3 px-6">
                <div class="relative">
@@ -322,6 +322,20 @@
                </div>
               </td>
              </tr>
+             <tr class="opacity-100 text-gray-500 dark:text-gray-100">
+              <td class="py-3 px-6 bold">Remove DB records for expired offers</td>
+              <td class="py-3 px-6">
+               <div class="relative">
+                <svg class="absolute right-4 top-1/2 transform -translate-y-1/2" width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+                 <path d="M11.3333 6.1133C11.2084 5.98913 11.0395 5.91943 10.8633 5.91943C10.6872 5.91943 10.5182 5.98913 10.3933 6.1133L8.00001 8.47329L5.64001 6.1133C5.5151 5.98913 5.34613 5.91943 5.17001 5.91943C4.99388 5.91943 4.82491 5.98913 4.70001 6.1133C4.63752 6.17527 4.58792 6.249 4.55408 6.33024C4.52023 6.41148 4.50281 6.49862 4.50281 6.58663C4.50281 6.67464 4.52023 6.76177 4.55408 6.84301C4.58792 6.92425 4.63752 6.99799 4.70001 7.05996L7.52667 9.88663C7.58865 9.94911 7.66238 9.99871 7.74362 10.0326C7.82486 10.0664 7.912 10.0838 8.00001 10.0838C8.08801 10.0838 8.17515 10.0664 8.25639 10.0326C8.33763 9.99871 8.41136 9.94911 8.47334 9.88663L11.3333 7.05996C11.3958 6.99799 11.4454 6.92425 11.4793 6.84301C11.5131 6.76177 11.5305 6.67464 11.5305 6.58663C11.5305 6.49862 11.5131 6.41148 11.4793 6.33024C11.4454 6.249 11.3958 6.17527 11.3333 6.1133Z" fill="#8896AB"></path>
+                </svg>
+                <select name="expire_db_records" class="hover:border-blue-500 bg-gray-50 text-gray-900 appearance-none pr-10 dark:bg-gray-500 dark:text-white border border-gray-300 dark:border-gray-400 dark:text-gray-50 dark:placeholder-gray-400 text-sm rounded-lg outline-none focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 focus:ring-0">
+                 <option {% if general_settings.expire_db_records %}selected{% endif %} value="true">True</option>
+                 <option {% if not general_settings.expire_db_records %}selected{% endif %} value="false">False</option>
+                </select>
+               </div>
+              </td>
+             </tr>
             </table>
            </div>
           </div>
@@ -445,7 +459,7 @@
   <div class="hidden rounded-lg" id="tor" role="tabpanel" aria-labelledby="tor-tab">
    <section class="pl-6">
     <div class="flex flex-wrap items-center">
-     <div class="w-full">  
+     <div class="w-full">
       <h4 class="font-semibold text-black dark:text-white text-2xl align-middle">
        <span class="mr-2 inline-block align-middle items-center justify-center w-0 h-10 bg-white-50 rounded">
        </span>Tor
@@ -557,4 +571,4 @@
 </script>
 {% include 'footer.html' %}
 </body>
-</html>
+</html>

basicswap/ui/page_debug.py

@@ -36,6 +36,7 @@ def page_debug(self, url_split, post_string):
 
         if have_data_entry(form_data, 'remove_expired'):
             try:
+                swap_client.log.warning('Removing expired data.')
                 remove_expired_data(swap_client)
                 messages.append('Done.')
             except Exception as e:

basicswap/ui/page_settings.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2022 tecnovert
+# Copyright (c) 2022-2023 tecnovert
 # Distributed under the MIT software license, see the accompanying
 # file LICENSE or http://www.opensource.org/licenses/mit-license.php.
 
@@ -37,17 +37,17 @@ def page_settings(self, url_split, post_string):
                 data = {
                     'debug': toBool(get_data_entry(form_data, 'debugmode')),
                     'debug_ui': toBool(get_data_entry(form_data, 'debugui')),
+                    'expire_db_records': toBool(get_data_entry(form_data, 'expire_db_records')),
                 }
                 swap_client.editGeneralSettings(data)
-            if have_data_entry(form_data, 'apply_chart'):
+            elif have_data_entry(form_data, 'apply_chart'):
                 active_tab = 'general'
                 data = {
                     'show_chart': toBool(get_data_entry(form_data, 'showchart')),
                     'chart_api_key': html.unescape(get_data_entry_or(form_data, 'chartapikey', '')),
                 }
                 swap_client.editGeneralSettings(data)
-
+            elif have_data_entry(form_data, 'apply_tor'):
-            if have_data_entry(form_data, 'apply_tor'):
                 active_tab = 'tor'
                 # TODO: Detect if running in docker
                 raise ValueError('TODO: If running in docker see doc/tor.md to enable/disable tor.')
@@ -124,6 +124,7 @@ def page_settings(self, url_split, post_string):
     general_settings = {
         'debug': swap_client.debug,
         'debug_ui': swap_client.debug_ui,
+        'expire_db_records': swap_client._expire_db_records,
     }
     if 'chart_api_key_enc' in swap_client.settings:
         chart_api_key = html.escape(bytes.fromhex(swap_client.settings.get('chart_api_key_enc', '')).decode('utf-8'))

doc/release-notes.md

@@ -7,6 +7,9 @@
     enable offers from no-script coins to script coins.
 - smsg: Outbox messages are removed when expired.
 - Fixed BTC witness size estimation.
+- Added option to remove Offers and bids from the database automatically one week
+  after they expire if they have no active bids.
+  - Controlled by new settings: expire_db_records and expire_db_records_after
 
 
 0.0.63

tests/basicswap/test_xmr.py

@@ -832,6 +832,11 @@ class Test(BaseTest):
         assert (compare_bid_states(offerer_states, self.states_offerer[0]) is True)
         assert (compare_bid_states(bidder_states, self.states_bidder[0]) is True)
 
+        # Test remove_expired_data
+        remove_expired_data(swap_clients[0], -swap_clients[0]._expire_db_records_after * 2)
+        offers = swap_clients[0].listOffers(filters={'offer_id': offer_id})
+        assert (len(offers) == 0)
+
     def test_011_smsgaddresses(self):
         logging.info('---------- Test address management and private offers')
         swap_clients = self.swap_clients
@@ -1547,9 +1552,6 @@ class Test(BaseTest):
             logging.info('Restoring XMR mining')
             pause_event.set()
 
-    def test_98_remove_expired_data(self):
-        remove_expired_data(self.swap_clients[0])
-
 
 if __name__ == '__main__':
     unittest.main()