preparescript: Download BTC utxo snapshot first, allow resume.

basicswap/basicswap.py

@@ -1423,11 +1423,8 @@ class BasicSwap(BaseApp):
     def revokeOffer(self, offer_id, security_token=None):
         self.log.info('Revoking offer %s', offer_id.hex())
 
-        session = None
+        session = self.openSession()
-        self.mxDB.acquire()
         try:
-            session = scoped_session(self.session_factory)
-
             offer = session.query(Offer).filter_by(offer_id=offer_id).first()
 
             if offer.security_token is not None and offer.security_token != security_token:
@@ -1445,10 +1442,21 @@ class BasicSwap(BaseApp):
             msg_id = self.sendSmsg(offer.addr_from, self.network_addr, payload_hex, offer.time_valid)
             self.log.debug('Revoked offer %s in msg %s', offer_id.hex(), msg_id.hex())
         finally:
-            if session:
+            self.closeSession(session, commit=False)
-                session.close()
+
-                session.remove()
+    def archiveOffer(self, offer_id):
-            self.mxDB.release()
+        self.log.info('Archiving offer %s', offer_id.hex())
+        session = self.openSession()
+        try:
+            offer = session.query(Offer).filter_by(offer_id=offer_id).first()
+
+            if offer.active_ind != 1:
+                raise ValueError('Offer is not active')
+
+            offer.active_ind = 3
+
+        finally:
+            self.closeSession(session)
 
     def grindForEd25519Key(self, coin_type, evkey, key_path_base):
         ci = self.ci(coin_type)

basicswap/templates/offer.html

@@ -396,6 +396,13 @@
                           <button name="revoke_offer" value="Revoke Offer" type="submit" onclick="return confirmPopup();" class="flex flex-wrap justify-center w-full px-4 py-2.5 font-medium text-sm text-red-500 hover:text-red-600 border border-red-400 hover:border-red-500 bg-white rounded-md shadow-button focus:ring-0 focus:outline-none"><svg class="text-gray-500 w-5 h-5 mr-2" xmlns="http://www.w3.org/2000/svg" height="24" width="24" viewBox="0 0 24 24"><g stroke-linecap="round" stroke-width="2" fill="none" stroke="#ef5844" stroke-linejoin="round" ><line x1="16" y1="8" x2="8" y2="16" stroke="#ef5844"></line> <line x1="16" y1="16" x2="8" y2="8" stroke="#ef5844"></line> <circle cx="12" cy="12" r="11"></circle></g></svg> Revoke Offer </button>
                         </div>
                     {% endif %}
+                    <!-- TODO:
+                    {% if data.active_ind == 1 %}
+                       <div class="w-full md:w-auto p-1.5">
+                          <button name="archive_offer" value="Archive Offer" type="submit" onclick="return confirmPopup();" class="flex flex-wrap justify-center w-full px-4 py-2.5 font-medium text-sm text-red-500 hover:text-red-600 border border-red-400 hover:border-red-500 bg-white rounded-md shadow-button focus:ring-0 focus:outline-none"><svg class="text-gray-500 w-5 h-5 mr-2" xmlns="http://www.w3.org/2000/svg" height="24" width="24" viewBox="0 0 24 24"><g stroke-linecap="round" stroke-width="2" fill="none" stroke="#ef5844" stroke-linejoin="round" ><line x1="16" y1="8" x2="8" y2="16" stroke="#ef5844"></line> <line x1="16" y1="16" x2="8" y2="8" stroke="#ef5844"></line> <circle cx="12" cy="12" r="11"></circle></g></svg>Archive Offer</button>
+                        </div>
+                    {% endif %}
+                    -->
                     {% endif %}
                     <!-- todo
 <div class="w-full md:w-auto p-1.5 ml-2">

basicswap/ui/page_offers.py

@@ -449,6 +449,12 @@ def page_offer(self, url_split, post_string):
     bid_rate = ci_to.format_amount(offer.rate)
 
     if form_data:
+        if b'archive_offer' in form_data:
+            try:
+                swap_client.archiveOffer(offer_id)
+                messages.append('Offer archived')
+            except Exception as ex:
+                err_messages.append('Archive offer failed: ' + str(ex))
         if b'revoke_offer' in form_data:
             try:
                 swap_client.revokeOffer(offer_id)
@@ -525,7 +531,8 @@ def page_offer(self, url_split, post_string):
         'bid_rate': bid_rate,
         'debug_ui': swap_client.debug_ui,
         'automation_strat_id': -1,
-        'is_expired': offer.expire_at <= now
+        'is_expired': offer.expire_at <= now,
+        'active_ind': offer.active_ind
     }
     data.update(extend_data)
 
@@ -622,7 +629,7 @@ def page_offers(self, url_split, post_string, sent=False):
             filters['sent_from'] = sent_from
         if have_data_entry(form_data, 'active'):
             active_filter = get_data_entry(form_data, 'active')
-            ensure(active_filter in ['any', 'active', 'expired', 'revoked'], 'Invalid active filter')
+            ensure(active_filter in ['any', 'active', 'expired', 'revoked', 'archived'], 'Invalid active filter')
             filters['active'] = active_filter
 
     set_pagination_filters(form_data, filters)

bin/basicswap_prepare.py

@@ -8,6 +8,7 @@
 import os
 import sys
 import json
+import time
 import mmap
 import stat
 import gnupg
@@ -20,8 +21,11 @@ import tarfile
 import zipfile
 import logging
 import platform
+import contextlib
 import urllib.parse
-from urllib.request import urlretrieve
+from urllib.error import ContentTooShortError
+from urllib.request import Request, urlopen
+from urllib.parse import _splittype
 
 import basicswap.config as cfg
 from basicswap import __version__
@@ -173,25 +177,118 @@ default_socket_timeout = socket.getdefaulttimeout()
 default_socket_getaddrinfo = socket.getaddrinfo
 
 
-def make_reporthook():
+def make_reporthook(read_start=0):
-    read = 0  # Number of bytes read so far
+    read = read_start  # Number of bytes read so far
     last_percent_str = ''
+    time_last = time.time()
+    read_last = read_start
+    display_last = time_last
+    abo = 7
+    average_buffer = [-1] * 8
 
     def reporthook(blocknum, blocksize, totalsize):
-        nonlocal read
+        nonlocal read, last_percent_str, time_last, read_last, display_last, read_start
-        nonlocal last_percent_str
+        nonlocal average_buffer, abo
         read += blocksize
-        if totalsize > 0:
+
-            percent_str = '%5.0f%%' % (read * 1e2 / totalsize)
+        # totalsize excludes read_start
-            if percent_str != last_percent_str:
+        use_size = totalsize + read_start
-                logger.info(percent_str)
+        dl_complete: bool = totalsize > 0 and read >= use_size
-                last_percent_str = percent_str
+        time_now = time.time()
+        time_delta = time_now - time_last
+        if time_delta < 4 and not dl_complete:
+            return
+
+        bytes_delta = read - read_last
+        time_last = time_now
+        read_last = read
+        bits_per_second = (bytes_delta * 8) / time_delta
+
+        abo = 0 if abo >= 7 else abo + 1
+        average_buffer[abo] = bits_per_second
+
+        samples = 0
+        average_bits_per_second = 0
+        for sample in average_buffer:
+            if sample < 0:
+                continue
+            average_bits_per_second += sample
+            samples += 1
+        average_bits_per_second /= samples
+
+        speed_str: str
+        if average_bits_per_second > 1000 ** 3:
+            speed_str = '{:.2f} Gbps'.format(average_bits_per_second / (1000 ** 3))
+        elif average_bits_per_second > 1000 ** 2:
+            speed_str = '{:.2f} Mbps'.format(average_bits_per_second / (1000 ** 2))
         else:
-            logger.info('read %d' % (read,))
+            speed_str = '{:.2f} kbps'.format(average_bits_per_second / 1000)
+
+        if totalsize > 0:
+            percent_str = '%5.0f%%' % (read * 1e2 / use_size)
+            if percent_str != last_percent_str or time_now - display_last > 10:
+                logger.info(percent_str + '  ' + speed_str)
+                last_percent_str = percent_str
+                display_last = time_now
+        else:
+            logger.info(f'Read {read}, {speed_str}')
     return reporthook
 
 
-def setConnectionParameters():
+def urlretrieve(url, filename, reporthook=None, data=None, resume_from=0):
+    # urlretrieve with resume
+    url_type, path = _splittype(url)
+
+    req = Request(url)
+    if resume_from > 0:
+        logger.info(f'Attempting to resume from byte {resume_from}')
+        req.add_header('Range', f'bytes={resume_from}-')
+    with contextlib.closing(urlopen(req)) as fp:
+        headers = fp.info()
+
+        # Just return the local path and the "headers" for file://
+        # URLs. No sense in performing a copy unless requested.
+        if url_type == "file" and not filename:
+            return os.path.normpath(path), headers
+
+        with open(filename, 'ab' if resume_from > 0 else 'wb') as tfp:
+            result = filename, headers
+            bs = 1024 * 8
+            size = -1
+            read = resume_from
+            blocknum = 0
+            range_from = 0
+            if "content-length" in headers:
+                size = int(headers["Content-Length"])
+            if "Content-Range" in headers:
+                range_str = headers["Content-Range"]
+                offset = range_str.find('-')
+                range_from = int(range_str[6:offset])
+            if resume_from != range_from:
+                raise ValueError('Download is not resuming from the expected byte')
+
+            if reporthook:
+                reporthook(blocknum, bs, size)
+
+            while True:
+                block = fp.read(bs)
+                if not block:
+                    break
+                read += len(block)
+                tfp.write(block)
+                blocknum += 1
+                if reporthook:
+                    reporthook(blocknum, bs, size)
+
+    if size >= 0 and read < size:
+        raise ContentTooShortError(
+            "retrieval incomplete: got only %i out of %i bytes"
+            % (read, size), result)
+
+    return result
+
+
+def setConnectionParameters(timeout=5):
     opener = urllib.request.build_opener()
     opener.addheaders = [('User-agent', 'Mozilla/5.0')]
     urllib.request.install_opener(opener)
@@ -202,7 +299,7 @@ def setConnectionParameters():
         socket.getaddrinfo = getaddrinfo_tor  # Without this accessing .onion links would fail
 
     # Set low timeout for urlretrieve connections
-    socket.setdefaulttimeout(5)
+    socket.setdefaulttimeout(timeout)
 
 
 def popConnectionParameters():
@@ -212,12 +309,12 @@ def popConnectionParameters():
     socket.setdefaulttimeout(default_socket_timeout)
 
 
-def downloadFile(url, path):
+def downloadFile(url, path, timeout=5, resume_from=0):
     logger.info('Downloading file %s', url)
     logger.info('To %s', path)
     try:
-        setConnectionParameters()
+        setConnectionParameters(timeout=timeout)
-        urlretrieve(url, path, make_reporthook())
+        urlretrieve(url, path, make_reporthook(resume_from), resume_from=resume_from)
     finally:
         popConnectionParameters()
 
@@ -766,27 +863,10 @@ def prepareDataDir(coin, settings, chain, particl_mnemonic, extra_opts={}):
 
         sync_file_path = os.path.join(base_dir, BITCOIN_FASTSYNC_FILE)
         if not os.path.exists(sync_file_path):
-            sync_file_url = os.path.join(BITCOIN_FASTSYNC_URL, BITCOIN_FASTSYNC_FILE)
+            raise ValueError(f'BTC fastsync file not found: {sync_file_path}')
-            downloadFile(sync_file_url, sync_file_path)
 
-        asc_filename = BITCOIN_FASTSYNC_FILE + '.asc'
+        # Double check
-        asc_file_path = os.path.join(base_dir, asc_filename)
+        check_btc_fastsync_data(base_dir, sync_file_path)
-        if not os.path.exists(asc_file_path):
-            asc_file_urls = (
-                'https://raw.githubusercontent.com/tecnovert/basicswap/master/pgp/sigs/' + asc_filename,
-                'https://gitlab.com/particl/basicswap/-/raw/master/pgp/sigs/' + asc_filename,
-            )
-            for url in asc_file_urls:
-                try:
-                    downloadFile(url, asc_file_path)
-                    break
-                except Exception as e:
-                    logging.warning('Download failed: %s', str(e))
-        gpg = gnupg.GPG()
-        with open(asc_file_path, 'rb') as fp:
-            verified = gpg.verify_file(fp, sync_file_path)
-
-        ensureValidSignatureBy(verified, 'tecnovert')
 
         with tarfile.open(sync_file_path) as ft:
             ft.extractall(path=data_dir)
@@ -1092,6 +1172,27 @@ def signal_handler(sig, frame):
     logger.info('Signal %d detected' % (sig))
 
 
+def check_btc_fastsync_data(base_dir, sync_file_path):
+    asc_filename = BITCOIN_FASTSYNC_FILE + '.asc'
+    asc_file_path = os.path.join(base_dir, asc_filename)
+    if not os.path.exists(asc_file_path):
+        asc_file_urls = (
+            'https://raw.githubusercontent.com/tecnovert/basicswap/master/pgp/sigs/' + asc_filename,
+            'https://gitlab.com/particl/basicswap/-/raw/master/pgp/sigs/' + asc_filename,
+        )
+        for url in asc_file_urls:
+            try:
+                downloadFile(url, asc_file_path)
+                break
+            except Exception as e:
+                logging.warning('Download failed: %s', str(e))
+    gpg = gnupg.GPG()
+    with open(asc_file_path, 'rb') as fp:
+        verified = gpg.verify_file(fp, sync_file_path)
+
+    ensureValidSignatureBy(verified, 'tecnovert')
+
+
 def main():
     global use_tor_proxy
     data_dir = None
@@ -1249,6 +1350,29 @@ def main():
         os.makedirs(data_dir)
     config_path = os.path.join(data_dir, cfg.CONFIG_FILENAME)
 
+    if extra_opts.get('use_btc_fastsync', False) is True:
+        logger.info(f'Preparing BTC Fastsync file {BITCOIN_FASTSYNC_FILE}')
+        sync_file_path = os.path.join(data_dir, BITCOIN_FASTSYNC_FILE)
+        sync_file_url = os.path.join(BITCOIN_FASTSYNC_URL, BITCOIN_FASTSYNC_FILE)
+        try:
+            check_sig = False
+            remote_file = urlopen(sync_file_url)
+            if not os.path.exists(sync_file_path):
+                downloadFile(sync_file_url, sync_file_path, timeout=50)
+                check_sig = True
+            else:
+                file_size = os.stat(sync_file_path).st_size
+                if file_size < remote_file.length:
+                    logger.warning(f'{BITCOIN_FASTSYNC_FILE} is an unexpected size, {file_size} < {remote_file.length}')
+                    downloadFile(sync_file_url, sync_file_path, timeout=50, resume_from=file_size)
+                    check_sig = True
+
+            if check_sig:
+                check_btc_fastsync_data(data_dir, sync_file_path)
+        except Exception as e:
+            logger.error(f'Failed to download BTC fastsync file: {e}\nTry manually downloading from {sync_file_url}')
+            return 1
+
     withchainclients = {}
     chainclients = {
         'particl': {