Commit 481d6a7ea6ac1021190394a606da8fdb8cc0be0f

Authored by Marius Hanne
1 parent 9616bc91d8
Exists in bloom_filters

port BloomFilter from python-bitcoinlib

Showing 3 changed files with 276 additions and 0 deletions Side-by-side Diff

... ... @@ -19,6 +19,7 @@
19 19 autoload :Config, 'bitcoin/config'
20 20 autoload :Builder, 'bitcoin/builder'
21 21 autoload :Validation, 'bitcoin/validation'
  22 + autoload :BloomFilter,'bitcoin/bloom_filter'
22 23  
23 24 autoload :Namecoin, 'bitcoin/namecoin'
24 25  
lib/bitcoin/bloom_filter.rb
  1 +require "bitcoin"
  2 +
  3 +# Bitcoin compatible Bloom Filter.
  4 +# Thanks to https://github.com/jgarzik/python-bitcoinlib
  5 +class Bitcoin::BloomFilter
  6 +
  7 + MAX_SIZE = 36000
  8 + MAX_HASH_FUNCS = 50
  9 +
  10 + UPDATE_NONE = 0
  11 + UPDATE_ALL = 1
  12 + UPDATE_P2PUBKEY_ONLY = 2
  13 + UPDATE_MASK = 3
  14 +
  15 + LN2SQUARED = 0.4804530139182014246671025263266649717305529515945455
  16 + LN2 = 0.6931471805599453094172321214581765680755001343602552
  17 +
  18 + attr_reader :size, :fp_rate, :tweak, :flags, :hash_funcs, :data
  19 +
  20 + def initialize size, fp_rate, tweak, flags
  21 + @data = Array.new([-1 / LN2SQUARED * size * Math.log(fp_rate), MAX_SIZE * 8].min / 8, 0)
  22 + @hash_funcs = [@data.size * 8 / size * LN2, MAX_HASH_FUNCS].min.to_i
  23 + @size, @fp_rate, @tweak, @flags = size, fp_rate, tweak, flags
  24 + @bit_mask = [0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80]
  25 + end
  26 +
  27 + def bloom_hash(num, data)
  28 + seed = ((num * 0xFBA4C795) + @tweak) & 0xFFFFFFFF
  29 + murmurhash3(seed, data) % (@data.size * 8)
  30 + end
  31 +
  32 + # Insert an element in the filter.
  33 + def insert(elem)
  34 + return if @data.size == 1 && @data[0] == 0xff
  35 + for i in (0...@hash_funcs)
  36 + nIndex = bloom_hash(i, elem)
  37 + @data[nIndex >> 3] |= @bit_mask[7 & nIndex]
  38 + end
  39 + end
  40 +
  41 + # Test if the filter contains an element
  42 + def contains(elem)
  43 + return true if @data.size == 1 && @data[0] == 0xff
  44 + for i in (0...@hash_funcs)
  45 + nIndex = bloom_hash(i, elem)
  46 + return false unless (@data[nIndex >> 3] & @bit_mask[7 & nIndex]) > 0
  47 + end
  48 + true
  49 + end
  50 +
  51 + def IsWithinSizeConstraints
  52 + @data.size <= MAX_SIZE && @hash_funcs <= MAX_HASH_FUNCS
  53 + end
  54 +
  55 + def IsRelevantAndUpdate(tx, tx_hash)
  56 + # Not useful for a client, so not implemented yet.
  57 + raise "BloomFilter#IsRelevantAndUpdate not implemented"
  58 + end
  59 +
  60 + def serialize
  61 + (ser_string(@data.map(&:chr).join) + [@hash_funcs, @tweak, @flags].pack("IIC")).unpack("H*")[0]
  62 + end
  63 +
  64 + def deserialize str
  65 + str = str.htb
  66 + @data, str = *deser_string(str)
  67 + @data = @data.split("").map(&:ord)
  68 + @hash_funcs, @tweak, @flags = *str.unpack("IIC")
  69 + end
  70 +
  71 + def ser_string(s)
  72 + if s.bytesize < 253
  73 + return s.bytesize.chr + s
  74 + elsif s.bytesize < 0x1000
  75 + return 253.chr + [s.bytesize].pack("v") + s # struct.pack(b"<H", len(s))
  76 + elsif s.bytesize < 0x100000000
  77 + return 254.chr + [s.bytesize].pack("I") + s # struct.pack(b"<I", len(s))
  78 + end
  79 + 255.chr + [s.bytesize].pack("Q") + s # struct.pack(b"<Q", len(s))
  80 + end
  81 +
  82 + def deser_string(str)
  83 + nit, str = *str.unpack("Ca*")
  84 + if nit == 253
  85 + nit, str = *str.unpack("va*")
  86 + elsif nit == 254
  87 + nit, str = *str.unpack("Ia*")
  88 + elsif nit == 255
  89 + nit, str = *str.unpack("Qa*")
  90 + end
  91 + [str[0...nit], str[nit..-1]]
  92 + end
  93 +
  94 + def rotl32(x, r)
  95 + raise "x too big." unless x <= 0xFFFFFFFF
  96 + ((x << r) & 0xFFFFFFFF) | (x >> (32 - r))
  97 + end
  98 +
  99 + def murmurhash3(seed, data)
  100 + raise "hash_seed too big." unless seed <= 0xFFFFFFFF
  101 +
  102 + h1 = seed
  103 + c1 = 0xcc9e2d51
  104 + c2 = 0x1b873593
  105 +
  106 + # body
  107 + i = 0
  108 + while i < data.bytesize - data.bytesize % 4 && data.bytesize - i >= 4
  109 +
  110 + k1 = data[i..i+4].unpack("V")[0]
  111 +
  112 + k1 = (k1 * c1) & 0xFFFFFFFF
  113 + k1 = rotl32(k1, 15)
  114 + k1 = (k1 * c2) & 0xFFFFFFFF
  115 +
  116 + h1 ^= k1
  117 + h1 = rotl32(h1, 13)
  118 + h1 = (((h1*5) & 0xFFFFFFFF) + 0xe6546b64) & 0xFFFFFFFF
  119 +
  120 + i += 4
  121 + end
  122 +
  123 + # tail
  124 + k1 = 0
  125 + j = (data.bytesize / 4) * 4
  126 +
  127 + k1 ^= data[j+2].ord << 16 if data.bytesize & 3 >= 3
  128 + k1 ^= data[j+1].ord << 8 if data.bytesize & 3 >= 2
  129 + k1 ^= data[j].ord if data.bytesize & 3 >= 1
  130 +
  131 + k1 &= 0xFFFFFFFF
  132 + k1 = (k1 * c1) & 0xFFFFFFFF
  133 + k1 = rotl32(k1, 15)
  134 + k1 = (k1 * c2) & 0xFFFFFFFF
  135 + h1 ^= k1
  136 +
  137 + # finalization
  138 + h1 ^= data.bytesize & 0xFFFFFFFF
  139 + h1 ^= (h1 & 0xFFFFFFFF) >> 16
  140 + h1 *= 0x85ebca6b
  141 + h1 ^= (h1 & 0xFFFFFFFF) >> 13
  142 + h1 *= 0xc2b2ae35
  143 + h1 ^= (h1 & 0xFFFFFFFF) >> 16
  144 +
  145 + h1 & 0xFFFFFFFF
  146 + end
  147 +
  148 +end
spec/bitcoin/bloom_filter_spec.rb
  1 +require_relative 'spec_helper.rb'
  2 +include Bitcoin
  3 +
  4 +describe BloomFilter do
  5 +
  6 + before { @filter = BloomFilter.new(10, 0.1, 0, BloomFilter::UPDATE_ALL) }
  7 +
  8 + it "should do rotl32" do
  9 + {
  10 + [0, 0] => 0,
  11 + [0, 1] => 0,
  12 + [0, 2] => 0,
  13 + [0, 3] => 0,
  14 + [1, 0] => 1,
  15 + [2, 0] => 2,
  16 + [3, 0] => 3,
  17 + [1, 2] => 4,
  18 + [2, 2] => 8,
  19 + [3, 2] => 12,
  20 + [1, 1] => 2,
  21 + [2, 1] => 4,
  22 + [3, 1] => 6,
  23 + [2, 3] => 16,
  24 + }.each {|i, o| @filter.rotl32(*i).should == o }
  25 + end
  26 +
  27 + it "should do murmurhash3" do
  28 + [
  29 + [0x00000000, 0x00000000, ""],
  30 + [0x6a396f08, 0xFBA4C795, ""],
  31 + [0x81f16f39, 0xffffffff, ""],
  32 +
  33 + [0x514e28b7, 0x00000000, "00"],
  34 + [0xea3f0b17, 0xFBA4C795, "00"],
  35 + [0xfd6cf10d, 0x00000000, "ff"],
  36 +
  37 + [0x16c6b7ab, 0x00000000, "0011"],
  38 + [0x8eb51c3d, 0x00000000, "001122"],
  39 + [0xb4471bf8, 0x00000000, "00112233"],
  40 + [0xe2301fa8, 0x00000000, "0011223344"],
  41 + [0xfc2e4a15, 0x00000000, "001122334455"],
  42 + [0xb074502c, 0x00000000, "00112233445566"],
  43 + [0x8034d2a0, 0x00000000, "0011223344556677"],
  44 + [0xb4698def, 0x00000000, "001122334455667788"],
  45 + ].each {|e, s, d| @filter.murmurhash3(s, d.htb).should == e }
  46 + end
  47 +
  48 + def t data
  49 + @filter.insert(data.htb)
  50 + @filter.contains(data.htb).should == true
  51 + end
  52 +
  53 + def f data
  54 + @filter.contains(data.htb).should == false
  55 + end
  56 +
  57 + it "should create filter" do
  58 + @filter.size.should == 10
  59 + @filter.fp_rate.should == 0.1
  60 + @filter.tweak.should == 0
  61 + @filter.hash_funcs.should == 2
  62 +
  63 + @filter = BloomFilter.new(10, 1.0, 0, BloomFilter::UPDATE_ALL)
  64 + @filter.contains("foo").should == true
  65 + end
  66 +
  67 + it "should de/serialize filter" do
  68 + data = "050000a00000020000000000000001"
  69 + @filter.insert("foobar")
  70 + @filter.serialize.should == data
  71 +
  72 + filter = BloomFilter.new(10, 0.1, 0, BloomFilter::UPDATE_ALL)
  73 + filter.deserialize(data)
  74 +
  75 + [:size, :fp_rate, :tweak, :flags, :hash_funcs, :data].each do |m|
  76 + @filter.send(m).should == filter.send(m)
  77 + end
  78 +
  79 + filter.contains("foobar").should == true
  80 +
  81 + @filter = BloomFilter.new(10000, 0.1, 0, BloomFilter::UPDATE_ALL)
  82 + @filter.insert("foobar")
  83 +
  84 + filter = BloomFilter.new(10000, 0.1, 0, BloomFilter::UPDATE_ALL)
  85 + filter.deserialize(@filter.serialize)
  86 +
  87 + [:size, :fp_rate, :tweak, :flags, :hash_funcs, :data].each do |m|
  88 + @filter.send(m).should == filter.send(m)
  89 + end
  90 +
  91 + filter.contains("foobar").should == true
  92 + end
  93 +
  94 + it "should create, insert and serialize" do
  95 + @filter = BloomFilter.new(3, 0.01, 0, BloomFilter::UPDATE_ALL)
  96 + t("99108ad8ed9bb6274d3980bab5a85c048f0950c8")
  97 + f("19108ad8ed9bb6274d3980bab5a85c048f0950c8")
  98 + t("b5a2c786d9ef4658287ced5914b37a1b4aa32eee")
  99 + t("b9300670b4c5366e95b2699e8b18bc75e5f729c5")
  100 + @filter.serialize.should == "03614e9b050000000000000001"
  101 + end
  102 +
  103 + it "should create, insert and serialize with tweak" do
  104 + # Same test as before, but we add a tweak of 100
  105 + @filter = BloomFilter.new(3, 0.01, 2147483649, BloomFilter::UPDATE_ALL)
  106 + t("99108ad8ed9bb6274d3980bab5a85c048f0950c8")
  107 + f("19108ad8ed9bb6274d3980bab5a85c048f0950c8")
  108 + t("b5a2c786d9ef4658287ced5914b37a1b4aa32eee")
  109 + t("b9300670b4c5366e95b2699e8b18bc75e5f729c5")
  110 + @filter.serialize.should == "03ce4299050000000100008001"
  111 + end
  112 +
  113 +
  114 +
  115 + # def test_bloom_create_insert_key(self):
  116 + # filter = CBloomFilter(2, 0.001, 0, CBloomFilter.UPDATE_ALL)
  117 +
  118 + # pubkey = unhexlify(b'045B81F0017E2091E2EDCD5EECF10D5BDD120A5514CB3EE65B8447EC18BFC4575C6D5BF415E54E03B1067934A0F0BA76B01C6B9AB227142EE1D543764B69D901E0')
  119 + # pubkeyhash = ser_uint160(Hash160(pubkey))
  120 +
  121 + # filter.insert(pubkey)
  122 + # filter.insert(pubkeyhash)
  123 +
  124 + # self.assertEqual(filter.serialize(), unhexlify(b'038fc16b080000000000000001'))
  125 +
  126 +
  127 +end