net: CAddress & CAddrMan: (un)serialize as ADDRv2

furszy · furszy · commit cec956740837 · 2021-08-10T09:43:24.000-03:00
Change the serialization of `CAddrMan` to serialize its addresses
in ADDRv2/BIP155 format by default. Introduce a new `CAddrMan` format
version (3).

Add support for ADDRv2 format in `CAddress` (un)serialization.

Adaptation of btc@201a4596d92d640d5eb7e76cc8d959228fa09dbb
diff --git a/doc/release-notes.md b/doc/release-notes.md
@@ -38,6 +38,13 @@ From PIVX Core 6.0 onwards, macOS versions earlier than 10.12 are no longer supp
 
 PIVX Core should also work on most other Unix-like systems but is not frequently tested on them.
 
+The node's known peers are persisted to disk in a file called `peers.dat`. The
+format of this file has been changed in a backwards-incompatible way in order to
+accommodate the storage of Tor v3 and other BIP155 addresses. This means that if
+the file is modified by v5.3 or newer then older versions will not be able to
+read it. Those old versions, in the event of a downgrade, will log an error
+message that deserialization has failed and will continue normal operation
+as if the file was missing, creating a new empty one. (#2411)
 
 Notable Changes
 ==============
diff --git a/src/addrman.h b/src/addrman.h
@@ -13,6 +13,7 @@
 #include "random.h"
 #include "sync.h"
 #include "timedata.h"
+#include "tinyformat.h"
 #include "util/system.h"
 
 #include <fs.h>
@@ -271,6 +272,14 @@ friend class CAddrManTest;
     void SetServices_(const CService& addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs);
 
 public:
+    //! Serialization versions.
+    enum class Format : uint8_t {
+        V0_HISTORICAL = 0,    //!< historic format, before commit e6b343d88
+        V1_DETERMINISTIC = 1, //!< for pre-asmap files
+        V2_ASMAP = 2,         //!< for files including asmap version
+        V3_BIP155 = 3,        //!< same as V2_ASMAP plus addresses are in BIP155 format
+    };
+
     // Compressed IP->ASN mapping, loaded from a file when a node starts.
     // Should be always empty if no file was provided.
     // This mapping is then used for bucketing nodes in Addrman.
@@ -292,8 +301,8 @@ friend class CAddrManTest;
 
 
     /**
-     * serialized format:
-     * * version byte (1 for pre-asmap files, 2 for files including asmap version)
+     * Serialized format.
+     * * version byte (@see `Format`)
      * * 0x20 + nKey (serialized as if it were a vector, for backward compatibility)
      * * nNew
      * * nTried
@@ -321,12 +330,15 @@ friend class CAddrManTest;
      * very little in common.
      */
     template <typename Stream>
-    void Serialize(Stream& s) const
+    void Serialize(Stream& s_) const
     {
         LOCK(cs);
 
-        unsigned char nVersion = 2;
-        s << nVersion;
+        // Always serialize in the latest version (currently Format::V3_BIP155).
+
+        OverrideStream<Stream> s(&s_, s_.GetType(), s_.GetVersion() | ADDRV2_FORMAT);
+
+        s << static_cast<uint8_t>(Format::V3_BIP155);
         s << ((unsigned char)32);
         s << nKey;
         s << nNew;
@@ -378,13 +390,33 @@ friend class CAddrManTest;
     }
 
     template <typename Stream>
-    void Unserialize(Stream& s)
+    void Unserialize(Stream& s_)
     {
         LOCK(cs);
 
         Clear();
-        unsigned char nVersion;
-        s >> nVersion;
+
+        Format format;
+        s_ >> Using<CustomUintFormatter<1>>(format);
+
+        static constexpr Format maximum_supported_format = Format::V3_BIP155;
+        if (format > maximum_supported_format) {
+            throw std::ios_base::failure(strprintf(
+                "Unsupported format of addrman database: %u. Maximum supported is %u. "
+                "Continuing operation without using the saved list of peers.",
+                static_cast<uint8_t>(format),
+                static_cast<uint8_t>(maximum_supported_format)));
+        }
+
+        int stream_version = s_.GetVersion();
+        if (format >= Format::V3_BIP155) {
+            // Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress
+            // unserialize methods know that an address in addrv2 format is coming.
+            stream_version |= ADDRV2_FORMAT;
+        }
+
+        OverrideStream<Stream> s(&s_, s_.GetType(), stream_version);
+
         unsigned char nKeySize;
         s >> nKeySize;
         if (nKeySize != 32) throw std::ios_base::failure("Incorrect keysize in addrman deserialization");
@@ -393,7 +425,7 @@ friend class CAddrManTest;
         s >> nTried;
         int nUBuckets = 0;
         s >> nUBuckets;
-        if (nVersion != 0) {
+        if (format >= Format::V1_DETERMINISTIC) {
             nUBuckets ^= (1 << 30);
         }
 
@@ -456,21 +488,21 @@ friend class CAddrManTest;
             supplied_asmap_version = SerializeHash(m_asmap);
         }
         uint256 serialized_asmap_version;
-        if (nVersion > 1) {
+        if (format >= Format::V2_ASMAP) {
             s >> serialized_asmap_version;
         }
 
         for (int n = 0; n < nNew; n++) {
             CAddrInfo &info = mapInfo[n];
             int bucket = entryToBucket[n];
             int nUBucketPos = info.GetBucketPosition(nKey, true, bucket);
-            if (nVersion == 2 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 &&
+            if (format >= Format::V2_ASMAP && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 &&
                 info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS && serialized_asmap_version == supplied_asmap_version) {
                 // Bucketing has not changed, using existing bucket positions for the new table
                 vvNew[bucket][nUBucketPos] = n;
                 info.nRefCount++;
             } else {
-                // In case the new table data cannot be used (nVersion unknown, bucket count wrong or new asmap),
+                // In case the new table data cannot be used (format unknown, bucket count wrong or new asmap),
                 // try to give them a reference based on their primary source address.
                 LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n");
                 bucket = info.GetNewBucket(nKey, m_asmap);
diff --git a/src/protocol.h b/src/protocol.h
@@ -301,7 +301,8 @@ class CAddress : public CService
 
 public:
     CAddress() : CService{} {};
-    explicit CAddress(CService ipIn, ServiceFlags nServicesIn) : CService{ipIn}, nServices{nServicesIn} {};
+    CAddress(CService ipIn, ServiceFlags nServicesIn) : CService{ipIn}, nServices{nServicesIn} {};
+    CAddress(CService ipIn, ServiceFlags nServicesIn, uint32_t nTimeIn) : CService{ipIn}, nServices{nServicesIn}, nTime{nTimeIn} {};
 
     SERIALIZE_METHODS(CAddress, obj)
     {
@@ -314,7 +315,14 @@ class CAddress : public CService
             (nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) {
             READWRITE(obj.nTime);
         }
-        READWRITE(Using<CustomUintFormatter<8>>(obj.nServices));
+        if (nVersion & ADDRV2_FORMAT) {
+            uint64_t services_tmp;
+            SER_WRITE(obj, services_tmp = obj.nServices);
+            READWRITE(Using<CompactSizeFormatter<false>>(services_tmp));
+            SER_READ(obj, obj.nServices = static_cast<ServiceFlags>(services_tmp));
+        } else {
+            READWRITE(Using<CustomUintFormatter<8>>(obj.nServices));
+        }
         READWRITEAS(CService, obj);
     }
 
diff --git a/src/streams.h b/src/streams.h
@@ -63,6 +63,7 @@ class OverrideStream
     int GetVersion() const { return nVersion; }
     int GetType() const { return nType; }
     size_t size() const { return stream->size(); }
+    void ignore(size_t size) { return stream->ignore(size); }
 };
 
 /** Double ended buffer combining vector and stream-like interfaces.
diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp
@@ -4,9 +4,14 @@
 // Distributed under the MIT/X11 software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+#include "test/test_pivx.h"
+
 #include "net.h"    // validateMasternodeIP
 #include "netbase.h"
-#include "test/test_pivx.h"
+#include "protocol.h"
+#include "serialize.h"
+#include "streams.h"
+#include "version.h"
 
 #include <string>
 
@@ -385,4 +390,105 @@ BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters)
     BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com\0", 35), ret));
 }
 
+// Since CNetAddr (un)ser is tested separately in net_tests.cpp here we only
+// try a few edge cases for port, service flags and time.
+
+static const std::vector<CAddress> fixture_addresses({
+    CAddress(
+        CService(CNetAddr(in6addr_loopback), 0 /* port */),
+        NODE_NONE,
+        0x4966bc61U /* Fri Jan  9 02:54:25 UTC 2009 */
+    ),
+    CAddress(
+        CService(CNetAddr(in6addr_loopback), 0x00f1 /* port */),
+        NODE_NETWORK,
+        0x83766279U /* Tue Nov 22 11:22:33 UTC 2039 */
+    ),
+    CAddress(
+        CService(CNetAddr(in6addr_loopback), 0xf1f2 /* port */),
+        static_cast<ServiceFlags>(NODE_BLOOM),
+        0xffffffffU /* Sun Feb  7 06:28:15 UTC 2106 */
+    )
+});
+
+// fixture_addresses should equal to this when serialized in V1 format.
+// When this is unserialized from V1 format it should equal to fixture_addresses.
+static constexpr const char* stream_addrv1_hex =
+    "03" // number of entries
+
+    "61bc6649"                         // time, Fri Jan  9 02:54:25 UTC 2009
+    "0000000000000000"                 // service flags, NODE_NONE
+    "00000000000000000000000000000001" // address, fixed 16 bytes (IPv4 embedded in IPv6)
+    "0000"                             // port
+
+    "79627683"                         // time, Tue Nov 22 11:22:33 UTC 2039
+    "0100000000000000"                 // service flags, NODE_NETWORK
+    "00000000000000000000000000000001" // address, fixed 16 bytes (IPv6)
+    "00f1"                             // port
+
+    "ffffffff"                         // time, Sun Feb  7 06:28:15 UTC 2106
+    "0400000000000000"                 // service flags, NODE_BLOOM
+    "00000000000000000000000000000001" // address, fixed 16 bytes (IPv6)
+    "f1f2";                            // port
+
+// fixture_addresses should equal to this when serialized in V2 format.
+// When this is unserialized from V2 format it should equal to fixture_addresses.
+static constexpr const char* stream_addrv2_hex =
+    "03" // number of entries
+
+    "61bc6649"                         // time, Fri Jan  9 02:54:25 UTC 2009
+    "00"                               // service flags, COMPACTSIZE(NODE_NONE)
+    "02"                               // network id, IPv6
+    "10"                               // address length, COMPACTSIZE(16)
+    "00000000000000000000000000000001" // address
+    "0000"                             // port
+
+    "79627683"                         // time, Tue Nov 22 11:22:33 UTC 2039
+    "01"                               // service flags, COMPACTSIZE(NODE_NETWORK)
+    "02"                               // network id, IPv6
+    "10"                               // address length, COMPACTSIZE(16)
+    "00000000000000000000000000000001" // address
+    "00f1"                             // port
+
+    "ffffffff"                         // time, Sun Feb  7 06:28:15 UTC 2106
+    "04"                               // service flags, COMPACTSIZE(NODE_BLOOM)
+    "02"                               // network id, IPv6
+    "10"                               // address length, COMPACTSIZE(16)
+    "00000000000000000000000000000001" // address
+    "f1f2";                            // port
+
+BOOST_AUTO_TEST_CASE(caddress_serialize_v1)
+{
+    CDataStream s(SER_NETWORK, PROTOCOL_VERSION);
+
+    s << fixture_addresses;
+    BOOST_CHECK_EQUAL(HexStr(s), stream_addrv1_hex);
+}
+
+BOOST_AUTO_TEST_CASE(caddress_unserialize_v1)
+{
+    CDataStream s(ParseHex(stream_addrv1_hex), SER_NETWORK, PROTOCOL_VERSION);
+    std::vector<CAddress> addresses_unserialized;
+
+    s >> addresses_unserialized;
+    BOOST_CHECK(fixture_addresses == addresses_unserialized);
+}
+
+BOOST_AUTO_TEST_CASE(caddress_serialize_v2)
+{
+    CDataStream s(SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT);
+
+    s << fixture_addresses;
+    BOOST_CHECK_EQUAL(HexStr(s), stream_addrv2_hex);
+}
+
+BOOST_AUTO_TEST_CASE(caddress_unserialize_v2)
+{
+    CDataStream s(ParseHex(stream_addrv2_hex), SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT);
+    std::vector<CAddress> addresses_unserialized;
+
+    s >> addresses_unserialized;
+    BOOST_CHECK(fixture_addresses == addresses_unserialized);
+}
+
 BOOST_AUTO_TEST_SUITE_END()
