Merge bitcoin#30008: seeds: Pull additional nodes from my seeder and …

…update fixed seeds

41ad84a seeds: Use fjahr's more up to date asmap (Ava Chow)
d8fd1e0 seeds: Fixed seeds update (Ava Chow)
f1f24d7 seeds: Add testnet4 fixed seeds file (Ava Chow)
8ace71c seeds: Remove manual onion and i2p seeds (Ava Chow)
ed5b86c seeds: Add testnet instructions (Ava Chow)
0676515 seeds: Also pull from achow101 seeder (Ava Chow)
5bab317 makeseeds: Configurable minimum blocks for testnet4's smaller chain (Ava Chow)
d2465df makeseeds: Shuffle ips after parsing (Ava Chow)
af550b3 makeseeds: Support CJDNS (Ava Chow)
d5a8c4c makeseeds: Update user agent regex (Ava Chow)

Pull request description:

  The [DNS seeder](https://github.com/achow101/dnsseedrs) that I wrote collects statistics on node reliability in the same way that sipa's seeder does, and also outputs this information in the same file format. Thus it can also be used in our fixed seeds update scripts. My seeder additionally crawls onion v3, i2p, and cjdns, so will now be able to set those fixed seeds automatically rather than curating manual lists.

  In doing this update, I've found that `makeseeds.py` is missing newer versions from the regex as well as cjdns support; both of these have been updated.

  I also noticed that the testnet fixed seeds are all manually curated and sipa's seeder does not appear to publish any testnet data. Since I am also running the seeder for testnet, I've added the commands to generate testnet fixed seeds from my seeder's data too.

  Lastly, I've updated all of the fixed seeds. However, since my seeder has not found any cjdns nodes that met the reliability criteria (possibly due to connectivity issues present in those networks), I've left the previous manual seeds for that network.

ACKs for top commit:
  fjahr:
    re-ACK 41ad84a
  virtu:
    ACK [41ad84a](bitcoin@41ad84a)

Tree-SHA512: 6ba0141f053d9d6ae7d8c9574f061be38f3e65b28de1d6660c1885ab942623b5a0ec70754b4fcfc5d98fe970f5f179a940d5880b5061ed698f7932500e01d3ee

contrib/seeds/.gitignore

@@ -1,2 +1,3 @@
 seeds_main.txt
+seeds_test.txt
 asmap-filled.dat

contrib/seeds/README.md

@@ -8,13 +8,18 @@ and remove old versions as necessary (at a minimum when SeedsServiceFlags()
 changes its default return value, as those are the services which seeds are added
 to addrman with).
 
-The seeds compiled into the release are created from sipa's DNS seed and AS map
+The seeds compiled into the release are created from sipa's and achow101's DNS seed and AS map
 data. Run the following commands from the `/contrib/seeds` directory:
 
 ```
 curl https://bitcoin.sipa.be/seeds.txt.gz | gzip -dc > seeds_main.txt
-curl https://bitcoin.sipa.be/asmap-filled.dat > asmap-filled.dat
+curl https://mainnet.achownodes.xyz/seeds.txt.gz | gzip -dc >> seeds_main.txt
+curl https://testnet.achownodes.xyz/seeds.txt.gz | gzip -dc > seeds_test.txt
+curl https://raw.githubusercontent.com/fjahr/asmap-data/main/latest_asmap.dat > asmap-filled.dat
 python3 makeseeds.py -a asmap-filled.dat -s seeds_main.txt > nodes_main.txt
 cat nodes_main_manual.txt >> nodes_main.txt
+python3 makeseeds.py -a asmap-filled.dat -s seeds_test.txt > nodes_test.txt
+# TODO: Uncomment when a seeder publishes seeds.txt.gz for testnet4
+# python3 makeseeds.py -a asmap-filled.dat -s seeds_testnet4.txt -m 30000 > nodes_testnet4.txt
 python3 generate-seeds.py . > ../../src/chainparamsseeds.h
 ```

contrib/seeds/makeseeds.py

@@ -10,6 +10,7 @@
 import collections
 import ipaddress
 from pathlib import Path
+import random
 import re
 import sys
 from typing import Union
@@ -25,7 +26,7 @@
     'ipv6': 10,
 }
 
-MIN_BLOCKS = 730000
+MIN_BLOCKS = 840000
 
 PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$")
 PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$")
@@ -41,11 +42,13 @@
     r"0.19.(0|1|2|99)|"
     r"0.20.(0|1|2|99)|"
     r"0.21.(0|1|2|99)|"
-    r"22.(0|1|99)|"
-    r"23.(0|1|99)|"
-    r"24.(0|1|99)|"
-    r"25.(0|1|99)|"
-    r"26.(0|99)|"
+    r"22.(0|1|99).0|"
+    r"23.(0|1|99).0|"
+    r"24.(0|1|2|99).(0|1)|"
+    r"25.(0|1|2|99).0|"
+    r"26.(0|1|99).0|"
+    r"27.(0|1|99).0|"
+    r"28.(0|99).0|"
     r")")
 
 def parseline(line: str) -> Union[dict, None]:
@@ -86,6 +89,8 @@ def parseline(line: str) -> Union[dict, None]:
             if m.group(1) in ['::']: # Not interested in localhost
                 return None
             ipstr = m.group(1)
+            if ipstr.startswith("fc"): # cjdns looks like ipv6 but always begins with fc
+                net = "cjdns"
             sortkey = ipstr # XXX parse IPv6 into number, could use name_to_ipv6 from generate-seeds
             port = int(m.group(2))
     else:
@@ -152,6 +157,7 @@ def filterbyasn(asmap: ASMap, ips: list[dict], max_per_asn: dict, max_per_net: i
     ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']]
     ips_onion = [ip for ip in ips if ip['net'] == 'onion']
     ips_i2p = [ip for ip in ips if ip['net'] == 'i2p']
+    ips_cjdns = [ip for ip in ips if ip["net"] == "cjdns"]
 
     # Filter IPv46 by ASN, and limit to max_per_net per network
     result = []
@@ -176,6 +182,7 @@ def filterbyasn(asmap: ASMap, ips: list[dict], max_per_asn: dict, max_per_net: i
     # Add back Onions (up to max_per_net)
     result.extend(ips_onion[0:max_per_net])
     result.extend(ips_i2p[0:max_per_net])
+    result.extend(ips_cjdns[0:max_per_net])
     return result
 
 def ip_stats(ips: list[dict]) -> str:
@@ -185,12 +192,13 @@ def ip_stats(ips: list[dict]) -> str:
         if ip is not None:
             hist[ip['net']] += 1
 
-    return f"{hist['ipv4']:6d} {hist['ipv6']:6d} {hist['onion']:6d} {hist['i2p']:6d}"
+    return f"{hist['ipv4']:6d} {hist['ipv6']:6d} {hist['onion']:6d} {hist['i2p']:6d} {hist['cjdns']:6d}"
 
 def parse_args():
     argparser = argparse.ArgumentParser(description='Generate a list of bitcoin node seed ip addresses.')
     argparser.add_argument("-a","--asmap", help='the location of the asmap asn database file (required)', required=True)
     argparser.add_argument("-s","--seeds", help='the location of the DNS seeds file (required)', required=True)
+    argparser.add_argument("-m", "--minblocks", help="The minimum number of blocks each node must have", default=MIN_BLOCKS, type=int)
     return argparser.parse_args()
 
 def main():
@@ -205,9 +213,10 @@ def main():
     with open(args.seeds, 'r', encoding='utf8') as f:
         lines = f.readlines()
     ips = [parseline(line) for line in lines]
+    random.shuffle(ips)
     print('Done.', file=sys.stderr)
 
-    print('\x1b[7m  IPv4   IPv6  Onion  I2P Pass                                               \x1b[0m', file=sys.stderr)
+    print('\x1b[7m  IPv4   IPv6  Onion  I2P    CJDNS Pass                                               \x1b[0m', file=sys.stderr)
     print(f'{ip_stats(ips):s} Initial', file=sys.stderr)
     # Skip entries with invalid address.
     ips = [ip for ip in ips if ip is not None]
@@ -216,7 +225,7 @@ def main():
     ips = dedup(ips)
     print(f'{ip_stats(ips):s} After removing duplicates', file=sys.stderr)
     # Enforce minimal number of blocks.
-    ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS]
+    ips = [ip for ip in ips if ip['blocks'] >= args.minblocks]
     print(f'{ip_stats(ips):s} Enforce minimal number of blocks', file=sys.stderr)
     # Require service bit 1.
     ips = [ip for ip in ips if (ip['service'] & 1) == 1]
@@ -227,6 +236,7 @@ def main():
         'ipv6': 50,
         'onion': 10,
         'i2p' : 10,
+        'cjdns': 10,
     }
     ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]]
     print(f'{ip_stats(ips):s} Require minimum uptime', file=sys.stderr)
@@ -244,7 +254,7 @@ def main():
     # Sort the results by IP address (for deterministic output).
     ips.sort(key=lambda x: (x['net'], x['sortkey']))
     for ip in ips:
-        if ip['net'] == 'ipv6':
+        if ip['net'] == 'ipv6' or ip["net"] == "cjdns":
             print(f"[{ip['ip']}]:{ip['port']}", end="")
         else:
             print(f"{ip['ip']}:{ip['port']}", end="")