Spectral Bloom Filters - Stanford CS Theory

Spectral Bloom Filters Saar Cohen

Yossi Matias

School of Computer Science Tel Aviv University

School of Computer Science Tel Aviv University

[email protected]

[email protected]

-012B7;+ C 324 %*+ 56 #DFEG H H7824 $)*9* $AI# !:0* 0;"#-)J $ %&NEG]*'\0* '9d- 9*%* $%?/ $)*78+ 5+X(( 7; #%1AI

AJ %;5+0*$@;8 A+ J:&0*=> RZJ+%;3)J9* A $5+ 965+0;(+7*%&/0*(+o3% A $O+IRAIg+% K)k+ % M6 K9%N9;9; + 5+ $%;+W)*(-D A#0*Z78%;T -+39; #< $+%Y ,R )*%;)*(*+q?7; K+ %R15 => %K324)*

%* J(5* R =>CZ0; %;5K(9+ # %q+ j)*%$IJ-3/B0; %Q0*)*(D $#-)2If%&%; -%6)+I 780;K #0; K-3).I'24+ AM89* $9*%4?);9*( $Z(/+(9&-)JIP +%MMV -'+(9Jd* K-+a& %I(DW+[\ %i9* +[\9;-C24]1H0*Mp #+H KC-Io\7;+(49;C-++(9;5

%*5j A9; 9*1 # 801+ $- )#, %* $%#5X9* ( #*Z ;= + 8 D 2Bu+3vw4R $24 # A*9kq #=+%- $3)J $ +%q%k9*+'M]*9;: 0;' $)J+5 2\ 7 A9;* -a&-IK$--)k9*\j+(+9;MoO#2Bj3 K5 t /& R

#EO;%@ #%!-)KG %Q '$/3I1/ C #= RY + %@% "#MH]* :0;% $')VD 5 + A9* X%;-)1-) S# R\/+(%;7;) 9d9*69*+'0*%;-q3@/C-%*+&()*9' -9;D+$)*EG% %;5Kt u+u3#vwR4+P&Iq/+0*MH 0*-q:0*-+/--)) 7%; \%& '-+S;%&A0;I?-%?)k3H6($$Iq+78 ¡+MT-). #A24 ; A¢?'9k N9;)* A =>78 A-H)V $Dd%P (+8 Y #C1 Aq -=

ADjX #EG%i+% #t u\*vW9-9*=>X-)?)*78 ( ,#%Q /Rc*(+0;8 9* 01j-)QX9;2B3+=>+%dC< A j*9*%;X #+ K, 5 -+ A) 78 ,O+2\Rl Q%Q$-2 =>MnR Q'9*9* )*)*(((R+ %;).9*5j %.+);o)1(9;&/#RF+$);+Mo3/(N%/ #)c-)Dc;m %DK9;9*EO%% AG9*I6 X+4+N87*0;X 2\A!# $%q9;+-7*2^ $ AGI9;R*0*X%)19; J]*\9*jg+10*3=1 # $)1*j %K9; j93Mn0*\%;9;j +);%+(A

--=& $%*5X9;4 o)1 %;5X24 %)*-2 $H-= o7;ZMp+$* %cR>249; Z+A Mn -%k24\t $u+ %*%35i)1v!D =1_w+ $)1*0*+;-a1 $ K #3DT3`ZZ #%#- N378 $ \9;=> @+ +%;%6)@ NMY0;9*A F ;2 + AOL#I@:;0;8-3 $ / -I1+%;[\+A9;Iq; $% YN2489* + $( #9j11 RB%; H%g]1\ #r%+MTMp') ( % ,J+_ '?"`N #Rc 9;RT24]*9; * +_ '?&` (#,J-_ Y`N7* AJ-+%7867;0; A# . '0/ x 1 24365 75 89 DN;+'%Id5

=>%@: *3=& o)1B :% ,J_/-` $ #C+%q/ K? = R 7S;+% A) GI.+ %;%N_5Pa*/+8Mj + K%; $ % #\ $+%AIQ+243 $= 9@ D 78[\ $6,9;-6)QUV]* $ # j=>Mp+A+Ik%kX7* 78%*0;J A K %* RV%& 0;(V # )*%;-%)@(Z+ $A% +AI@%;,J)?Mn_/+3)*`j/ a1 8- +%* # Rc -%*)P+5d%;+)Q); #-(+&% D mk]*\ji1/0;%40* a1 8D #%\/ %;5J=+ +0; F0*+** +(9;+%;)k+5++ A9; ##0*-) mk.9*-2O2\+5 A9; #?MnP]*\b K $%%;+%*k%) $++L&010;(+Y R-I24Df9* [\9j9*qS;0*/Al-7R 24B A9X %; K0;7*+/4(+EG%%; $+-AIX

R\ # *J3 => #-)9QMp+0;$%;# $+%;Rc $y %*z A{( oy |{-I}}} {y8~J. K&+Dg*;[\ %;9*5 L H $I1 % 7889*#+ Mn0* A+5

$+9?q%;8DF"@+mk AjR8 $= A6MT++Y9*9*+ $ C( L '* ),+-9;I1+K4./Mo7; A e [9*'\9*'*

7;%@+7*+ $MF AOIq9;'MnX3(+M # N8Z+ 0 @+ B{ =>xV+DZ%)q4Mp)* C%*Z+ Z9;X)*' A4 8%;7%;2I )*14 % 3Z7;I*+7;J; A#OI6( Mn+0;%;C$)*3N+C ¡7;7* 4 P Z9* Q 4 Za*( +%I A+9*IPMj _ J ! `R % K9*X249* 9Q %* /+(+X% AC 9;O4jDj 4*+9;C+ 5 4O9*(A/$IK' 1 UVR8 'Mp+ _/X" =>7 ja;u+` #=>/^;%K _wMn&R+} 419* X M/P+` O Nd8DH0; [\ A%;9&5 0;=> R 9*%- -R;++%dMp0;]1%;\i $+%%$7; AGI#5N 7&I $%*%*C55J9;%q9;Z' A 0*% !D %;" 4 ( +Z L Mo^ %_* ` Mnb{0C9*Z X%*^-c-]1` \b . }%;} 56B}{ +!lZ (R*^9;%d ` = #b -%)6DTAIEO %6 #)1* $IJ)*- %;7&/I5 24 j %-=+)1+3 24$+B7;q _w%+3C $ + M8X(=>> R]*D\DNW%*[\-9;F %Bj78:*0;\+ 0*8%;IZ A YO-Id)'-: +0; #%@ A;78-A)I 0*; Mn0*4+MnMWB9*j3 $ + %;

%c%;D 5N'9* oB3Mn %60* $%ZMW=> 9;(']1(\+k7*$3= =>%< %*5 )1 / $7*0*-)6 a101 $+%Q+M /

%*C3X*%-)6 %d]1 $+% &D &D

Distributed processing

N]&+0;%=>A (78 ;i - #;*; +$ %d- #3 %& :&%;0* A) RFZ0*"#9*9. +]1%N\AI7 KI 78/ + 0* $%;)*A $%*_A I* -6 +%*l]15 $%? ]* 9*+%.Dj AC&[\D;>9*` R ( #AIK 0*+ %;*).$ -9)9.5 Mn0*%;%; (3 +C%;Dq%Q[\]19;\K@+0;1%'%=> %;5X+9; /+K %q %*M %;NAI? $ ##1 K3=>-$)@cMp]*(\ g ;%@(+MT #9;' +# _5+ $+%; #+W%&0; N)d78 -9;%Q $9;+%'% =>% %&+24BO2\+< #+j 0;% (`D

[l+7;@9;-24Q a18 #%(+40*q249;%0* $%*5iS;A 24=+ A 9f+0;x / T &= { #$0*/6+-%;+ )&R\V 1S8++%%g)*);I(.]124\ Ar9g ,L 6 2 +MZ1 D &a D /H\+0*7+u& R 9*9*- 0*24'%k -%;5X] #T _%*RlJ $0* N_ 1 0;0 H@ _H>J$`_`@ +R; 9;``£TRTdMp(;( (14 + 3 #%#RcMp(MVZ /Mp + KV9* sZ 05 ++$+V AN9* %¢?)QH2\$I1+L& $ %*H5J7£WINK o)1+%;#%01_nMn A+Mp(I1+ %; ,!5X8N8 +9* A Q=>%* D )*(+Wl(`Z AC %;)* N0; AjC+9;%d %* ,IQ )*M)1+Zj Mp+ ' # AZ24 #9;Z (249d9*+ *9@8-+*X8 -%Qj9* $% 9* A9* ); $( ¢#7I+9;C A #D A9; e 24+ %) 9 %

AB+ \L10*9;q+%;Mp+)* -!l)6R + \ %# $B9;(9;Z +L> 5 )# %c+)* 5 #9* $ Q%* %* 5d%*5E9*!#q24 9*%* %K-0;+6 %;5 8 (0* % %*5EB A $/%*K )10; - q AJ% o 0;%I#

A (24+R TB= EH80; Mp+D6 #;4 78a* +0* #;X Rc #24 A\9k78 1 A

5 % A 9; # B jMT%* +'+(/+ ll8 # Mp0*+%& #+-MW)6 #78 # +BIJ9Mp+4%-9; 9qB? #] +9;&5 )V R 0;9; A 9;%* 5NB_9;7*C+0*5+++B # AE\C9;+3 0*8 Mp+ #4 A`D . + 0/ 213!4%," [\9* B 5 + A9* Y9*T # /c #* aj # 9*&)cR Mn789;6 -\ 9;0; + #B*8M24%;9;F A\Mn9g+9;B%;+++9*78%* $ w%*015C4 01u%HCQ $%*O5P2MYX #]19* \%*Z $- N0;R> Q7*0*R&DW24 ((I?D 5 [\d 9;%Q B?9* # ]q* # Ba1 G9;I&5 +)q+Mj AB9*9*d Q9;C+R5

$9; #* 9* / J Q-D D 8*56+/-&N)1+0;(9;!TM*q7(8+ !%; $2+5 !%;3.D =>#[\" 9* +B96E AM*+9*0;5++-B?)V A]6D9;£W ¡a1%8) )* Be #J%* #%+-YRH9;+9*&%;3)*)i2.H K0;9-1I

+%?7* 9;DB;Nu&u &D +

J_ @ ` J_ 1 @ ` T a

&&DD &&DD 1+ 3u &D

11DD +++>4u 11DD ++>1u3u

11DD + u 11DD uu + 1D &

1 3a

11DD + 11DD ++ u&} CM G

1

&&DD 1 +- u &&DD 11 &} u&: M - G z

1:3071 D

+ D u 1 1DD + + u : (! ! ) : + 3! ) ! !#" 1 3 ! ) 3 . : J_ @` ! )6 ! 3 : J_ 1 @ ` ! )2! : . '. : T a { 1 3a :! ) . + *% /8. 2!: 3! ) 7!"# 1 13hfi $ +! ) ! #.% ! ) 8 " # #+IJ $%* - 1D 1D 1D u 1D u 1D £T 3 $ 1D +1 1D +1 *D 1D >u8 u&D > 1D >u B&)* AS)Qx - x /< " !& (' #! + $8*%) /: *%+*,-8 + # +8 %.*,-/0 ! ): 6 String-Array Index 9;M%:&0*%; MN+ # F A #R4+(+ %;5 Hs)t +VU4q"AH U @H:Hu0 ,J_/3`= us .+ 0 >=" L p. #C7* D4 $%;>)* aVR +F%;) AFW8+$9;4 WW j9; UV\7;+( $= L&78AIN %;H-)1+0;M9*4Z/5 => %;%J5G1( 71I )(13 %;[\ 5? 9*%YqR9;7;#+ %*+ q0;%01* $= Z AI%& ( $ 3%f%k $+9;%f'0 A *MZ\5 9* q 05 Z ]*i\& 9 !+7*MC

$%*,7* $%; $/' #IP7* MC*D 78_w;-%*)& $- 9;+ (C9Y5+RZ %=>+d%;%)gQ*:& 0*3N=&+%; $)*M4 j6 K %&+UV$E FfN7GF 0;%*C 71Mn!0** K7; 7*-: 0; #%g %;DQMp+qBE!N;HM9;dF C 78 A A 5 # %*%1 ( #*+$- D `i AGIdEO%?+M\9;Na*8/(+ A J $+%Y-Rc2\J9;24N F 0* #%& AX78X-2^J*$9;+N-)d 24 $%* A59; +01* %;+UV %;5X MlD-[\9;9#Q0* >7;-:0*=>%; +R#0; -%;:50; A%KQI_(FfI#7GF +C MV`S* a*S5 )1!7; , A-) R )&%I1Iq%+5 #+q #78( 9O2%;5 %qN2\9*q Q R>)*)(BD+Cp %;F5 $'e 7* 7; AR&24789* O2-+B%iN+0;0;%;8 R + ( k L 24)10;9* CE #B C (g9+N $Z%*5 J $ K%d+ a1c 0*%*/ (%;+5J%-9*D[\9; + B +#%6Mn78+ Q+ CD60*mk9#L 0;71 * 5H1+7;$+75 #'P%D< D\%;*Z5+%Zi-%&+0;(7;9?I60;:&J(0*9Q9;%;X % HI+R1L1C C'0*9;+'%() +7* L&%0; Q $ %;%;)*( 7;a /XZRN24+MF $lJ / 0* %; %5 +9;MW9*+CUV j9QMp A+ 9;% 78D 5+ $%*%; %;56+7M HC CF %d9*J]1\7;+N(IR [\9;$+L&0;f(7;QC C+%; /K+M'P $ #*Q("I !NR249;

%%)1 $ $J# *C C { &%R;%K%)#+Mn=" +] +. nu +qV ]Z us s [\p 4pV + )Vq.' $+n pVs

+V pn4 .n+ B C *+ /B C / B07 u F V pVVq.' n+ .'0 U0 %Z [%pVS 0 ),H'*)t 0),Hu @ H [+ s _ F` / ,J_/-75B(`} %*5 13$+Z ,8(50L#RV56 A4.4MF9 -9*= ]1N0* $;$%&80*7; ZB7824 S* A/9;& %K9M39;6 9;+9*% %DFMp+]&4 %;%&\I A- #:0;T3%*B49+MB 9;C -) %;%Z $9*+\%;7R &

Zq78%; 0;R %;)*- _ )g & 7` Ig k7B7;C %;+71 #_ B o?CF` 24/ A9 7B C 75B(Dg #[\9;Q 1@ _ 7BDCpP7; { A71OIi_ B?9`3` D

A #T24 A 9; %nl(9&0;%*L124 %*-)NX #3=>B0*8 %#%# $%* %# & 2478+ A0;9g%;)*-9*)?@7$I / & %;/ :& 0;+* _ A OI& > )*0*GQ` / *9; _ & %*&> +U

78 _0; %;& )*`` D$C_ Fz %* & ` RR _ ` 1&4 3 z 75B*M & 1& 3 z _ 75B( ` M$_ F ` /r- 75B 1& 3 z (_ | ` u75E!BBD K+ %*4J++0;%BMnZ 8--)q a1+%* %;4MW3 $ 0* $!@3' a1 0*%;%& %*DJDj]&*0;+* )K %;( %q+5 *#_?$ $+`%Q+L&7**0; AD u -%q78)*

%; $)*5N)c9;DFj;%&0;4 N a*78+ #4;u+` 2\R&jJ%* %;-)q&)1C7; A9; D EO+%?MF&D &R;R89* #X -/=>Z ( 0*5 %X j AI0; $%*A /%; ';4 9;-%;)NC+= 91%;) $0*@B9;+ FT$)o-'8+%*D %; --)V) R # 8+(\+9;-+R%#+9;%; 6B % #@N %;DAmgId9;%J K0*+ B%;Mp5 ( 4 ¢# 2% _ Hk +>4M ++ 249; A9'%P0;- %;$561+Ml9;V 5( 1/C3A¢Qhj iH8R> (%3)'+8-R G2\ 7;GAI $)1 -)*).)* A % M[\ 9;G+J$]*\ #U D X7; AOI?K K+ %(+ %d 0*% j-%.$6780*-)

%?:0; \249* 9Q8Mn+ %*NSA %*5*R0*9+B9;jMp -24 $%*5 !4,J.0

01( !-.A "# !4,2%37,689:!; `M RR 24019*-) Oi2e X B(3+(L>+ #%q 0;%* *Mp+W #9;\AIqS;/4H( +F%;)*9*+Be #Mp-++% 5 :&t &0;{e vw-DF)mk );j)1 # A- $= 3 +-R;249* 9Q 4-$ 0;$-)Q7I D |

1 & 2 "; &j5 & K 6.

1 / 9;X[\Mo9&(+0; R [\ +9*_ %1)N+ MN & `%;9;)di/ :--)c)dRl785 9%*- =&( +j %*2\5@+' K %*$q /_w%%X).247* 5+9 9*XEG%# a19;8B S; #/B%\+Mc8/-)R&9;9;4 L>D 2+Mc9;Z);3(X2\+B(9%;5 )cR Mo 0;AW/H_9;0;-%*24 AMn%'+ %N )* 5+(0* `W\+ 1R76/_ u $_)= %;I(L>`DW2\Zl));+3%'(78>`HDF[\9*%Y R +9*);)1 B =>EH4 5 + A\9* %)9;+H+9*4(378 */R&8%)JMn+ H K= %;I/4(787;B9#24 $%J A9J #5>H3() M B?]@(9++5 %*5 A9; $%# 9* %.Z78L>+2X9@DF[\3(9;+ # B+R5 7;01 X9* b 01+/8X -+Mn # H'%;9* . % # K3EG%969;B9; %;B)JEB +5 MV A9;/ QR>D 9;B/(+5+Z ,- #)1 %&x.Z+24MB% # $F%* %& K9;H-+2g9V3EO B%*$ 9;- A+IJ X9; #401 %;9; $%*&65)c-)*R1B 7;% %;Ik)1 %k*9++R Q+MB9*# A #X2\ #(+%;)*+ #I@(9; 9;%-24+%%) %f2\ 5+?0*Q%&& ARA If #)*$36-)9*Moq +J9;?(3]*\ $P+DH%;[\)k9;Q9;q 0;);A)1 A R 8 =>QMp+ K+6%*+MX249; A?9;]*+0*\6#)124 9; %%;D0;7 EG/K K#i=& $)*)*% J +9;%;#J.9* 9;B E MnM+5+ #+ !- A%;)c9;DN5 [\)19* =>N6 9; A+(+()d35+D (3j8 $= J\_ (B?9; ]? +4%;24) 9; %qB)19- => %;%* %*%1-#-Rc)cKRW(+24(( ALf+9iT+MX)*M8(19;Q -1?- =& #7 $.%*C5Q #2\ 9;/ 24 d%)1 N320;(9a*78* $ 6AI@)193q -9*)c DXB[\9;EK6+B?5 ]d A+%;9;)Q 9;Mn+ q 9;B 65 +% A39* *R 24 A9? )1= %(+5 jN9; BPD Encoding £T%*$ $&B)* # %; 59; #&methods. )VR&9;+&%;)1)K')* => mk (+ '788- )@ /%* S;-%)?5 01]1(9;N $ +/%*%.+\(1++D 5+M&JRW/%;+ #-*)*¢-)dMp+ %*74IQ5)*9;9;( 24N A9J =+5 01I1G %;&5XR2-=> (#5 B+ #Mp:&0*)@%;INMV9* A # RY+DT5d[\9; \ 0*% 0;A¢*RRW249;9*-24 $(%9

B K #;kAI d1 _ J& 3z #$+ 65 /NZ &/DF;¢`j4);9;3#(N/*N #24 A9q9;&-)*=>' (3+5+KC #Mp+:&#0*%;I %;%*+ #& )*- +%;5R)* $ #0*@13=>X9;j Ad9;-#2-=>3 =>( 5 9;J Mo)c-D :0;C%*- 2Id=> %; -Rj-9*kRT£T o%) 78-349*j8 Mp+ K+%*X+MW9*j/*< #9;&)*D / [\0; 9; J0*; $+A80* ;C8*7&IX/ 0;%&+V );%;)356]18X+0; %%()@4)*C %* /

- %* )dRl+ 24AMH9* 7; $ AJ RH1 +a&5+ DX%)1=&[\ %;%;9;55 /9; +0*( 5 /j 6 ,CMp++qM '$%;( * _+'?%*` ( %,J)_?$+`HL&7*0; ;6DW m.%Ci1]1\%D -B)# %* $ N=> 0;( ]1B $%; K +%kEG%;0*X+9;\# WA -I D -2 N(3RT7;01%;Q0*;8+'Mp+N0;V);3+%;)k)* +%;D +);M 3 0*1# 24 7;%* A5 9g KB%;3 i%; ¢.0*-01 (+0; IfR4 %)Dr Jm. %;0;?);;8+IPi/#1+)*( $5 #% $K+%;K9;(QL>%/#)- 0;%;*5 =+(78 $+I.Z7*0;$ $%;G-)*)K %*aV 5+%6RF?99*)*'%*]1(Q8&)*/ -)(0;+)*V0*(?#+2424 9;9; ( $9iA78*- DT%;3=&5dm. o)1C#+ #9;M+-/2 J J+ AB %*/ N0*0;5 9 0*q+%;)k K+ %%%;#Mp+J/(3 Q);3(d+%)i)10* %;5@)&I*%;+ #

7. CONCLUSIONS

0

50

45

120

10

Minimum Selection Recurring Minimum Minimal Increase



100

Additive Error

35

30

25

20

15

−1

10

80

Additive Error

Error Ratio (logarithmic scale)

40

−2

10

60

40 −3

10

10 20 5

0 0.2

−4

0.4

0.6

0.8

1

1.2

1.4

1.6

#% ! 3 ! ) T &}

+ 3 ' x ' !A*"

1.8

10

2

0.2

0.4

0.6

0.8

1

1.2

1.4

1.6

1.8

1

2

3

4

5

6

k

Gamma

N! * 38 "# 8 2 "# 8: $0 / A"

0

35

0.95

10

Minimum Selection Recurring Minimum Minimal Increase MS with deletions RM with deletions MI with deletions

0.9

False Negatives/Total Errors

30

log(Error Ratio)

25

Additive Error

0

2

+ 8 ,: ! ) '8: ' 8% T ( ! ) x / ( ! ),+ + ! '.A + , ! ! + A2/ ! ,/ ! T " + + ! '. :+Z % a1(+ %;0* %* 5'+%6)* #Ml+a*5+ (/+* %;9;5 T;+;Mc j-3 N $0;+A%; D F]*+F\ --R&+%K%78)J+91E! (]19\k%;5+BT++%K %9*)1\ );a#3()*` % u m m)1u S &+RT.4-+q & D

tA v BPD DB+%;G2B51VR Y%; D8RV]&+9;%; ) = L&D*0; KD*hC-R KD + %cD+ $ # B;01+$ $ %*%;51R 785 :&S 0*+ -- 4 ["#u0 +-%AV, I D1 0EO/% f. 6 +-u + -+Ap>4 ;0sV 0,0[. +>4[R*) +5+ u+ &&-1RW 1D tA3u-vN]VD D; ]1 +%;785+ 0;A(I9R*D,+ZD8D A Mn 7;+78+%; #R1N;D %;BQ5Mp+ $ %2B) w /(+% /r ;# "%Qm ,% '/;0V K 3 0[ $+%Y +D*[EO% ) 6S ++--u - + C [pVu02+'335mX4