1 Introduction - Center for Language and Speech Processing

  
     


 
   
     
            
  
      !" "#$
 

 %  
 &'&(  )  #* "& '++'



 
        
   
 
       
                                       
                                                           
      
  


 
   
  
 

    
       
 
 Æ
  
   !

    

Æ
 

      
 ""    
#

 
   $" %    
   $
 


 

&# '()*+ ,
 
 - '()   

 
   


" .       
/ 

   














 
 

    

   
  ""   
 

 





Ê
 "

.  


  
 

        

       
          
 
 "

.  
 

  
    

 

   
    $  

    

   
   
"



     

 





 


   


   

 

  

 

   0 " 1 $ 

  
   $
 
       
$   ) 
 $
    

 !
    
 "

'


 
   2 
   
 
  
  
  
  
 


   
!
  
  
 
 
    
       "



      

. !      
#     
  
 
   
  
 
"


       


       
 
 

 
   
 

   

      
 
   
         
 
 $ " 3  
    
 
       
  
  
 
  4
 '(((+ 4
 '((*+ 4
 '(((+ 4 
 ,
 + 5
 '(((  

$#   
   
      

   "


 
  

6

       
  
 

 ""

  7 83


       

    9 
   
  
 
        
 $  
 
 ""   
  
    
!
 :
 

  $  " 1   
 

  

 
  
  
 !  
 
   
      

  
 $  
    "

1 $  !
  
   
 

  
!
 
  0      
 
   " . 
            

   
   

  "




 
  

 

  


 :
     
 

 
   

 

     
"




      

; 
   
       

    

   0  

   

  

         

  


 
   

 


 

   
   
  

   
   
       





C

 C  C







 

 

 


  
     
          @ A









'

  


 

     
 @ 

A 
  

 
 

        
  " %    


  
 
     

D


       

    
 
      @  A

  " .     

    -$ B
   
 
 
              
 

  

   
        :
   
     

 
 
" .    
   
  

  

 

   
  

 
  

,
 '(()     

""

  
 
   0
   $

  
 $  
 
 

-  " 6

 
 
       

    
 $ 

"

-


  $ 

  
  

    :   

  

 
  " .   
 
 
   
 $
,
 " '(((+ , 
 ,
 '((( 
  
    
E  "
   

"" 
   
    

"" 


      

    
-$ B
   
 $

  Æ
 
@ A  

  
  
 
  

 "

%     Æ  # ""


 

 

   B- 
  
   
 $
 
    
  



#
 

 
-$ B
  "


 

       

7
   
  
 
  
   - 


  '(('    
 
    $
  
 
  
 
  
 
" .  
  

     

  
 

        

1 '" .       


     
   3 '((> 
        
 
 
 
8
 
 %

 " . 
     


  
 
    

       
 
 3# '((D  

 

   " '(( F
 3



$
" .       



  
 
$ 

       
 
    
  "" 
5 
 & '((>  
  

 
,


 5    $  
"



.   

  
 

$ 




C












  "




   

C C

  
 '              




G


   


       



 

  /

 







´




´      ½      


  
 





 µ





 µ





  ½     





tanh ...

... only some of the computation redone for each












...



¾


...

...

Matrix

Table look−up in

shared parameters across words

index for 1 '/





½


...

...






index for



index for

¾





     

½

 
    

 

  
   

 
 $ 
 HDH  

 "     




 
     


 

$  "





 

 


 

  

$     




 


   
$   
/

 C 
  I   





       

    




    

 

  
   





  $   



  
   
  
  

 I   "

1
 

    
 








   C 




I

  I  





          


  
 








 
 



 








   


 # 

 

 " . 


   
#  $

 
 /




  


  



C







. 
  
     
#
 -$ B
    " ' 
  

 
   
   
    J

   



 $#  
#  
" 6

 


     
   
       


 -$ B
  " . 
  
          

 

 + %

    

     

  %

+









  

 C

C

  

 







C


 


  

 C  C

   # 

      
!
    




    
 "

. Æ    
   


 

   
 

  
$ 
  






  %

 


#
 

" %

   

  Æ
 $  
            
 
  


      
 

  


 
$


   "

 
 
 




   



   
 
 $
  

C   
  
  C  
   
    C     C  /  
  

   

C



   I 






 

 /

      
    

" - 
  
 
   


    

D

.   
  
   $   
 
        
    -
4 - 4
""  8  B$  
  
  


 "

= 
 

 
   
   





      
 
        


 



  


 
  
     
  "" 

    
   "


! "  # $  " F     ""

  
 
=
  


K '()>+ %   '(()  -$ B
    8 
  8 '((> 
 
     
 
  
     
 


     
%

9  
  

 /







C

>







 %

 G

     
 $   
    
#
 

  

 
" 3      
 
-$ 
   




   8 
  8 '((> 
  
 @$A   

  C      




 

  
   @$ A    

      ""    


 C 



 


 C 



 

 




 C 





  
  
 @
A $ " .    
 
  

  

  %-- 
 
 

  B-
  " 4 
 
$  $ 
   $  
 
       $ 
  
!
 
 
    " 1 $ 1 9 $
 1   

  
#    
   -$  B
        "


    
 

  

 
 
     

   
 ! " #$  
$ $ #% $  % 


   


  $  % $ #% $  % 



E 
  
   $  




   
 $ #% $  % "



   
 ! " #$  
$
 
/

.  
    
 

    $  @
  A  


   

  @
  A"

E 

    
 $
    
#
       
 

#" .  
    
 

 
/

      @
  A 
   




  

  @
  A" .     $
  $ 
  

  
  
         
 
#  1  "

;
           4
9 $
 4
 '(((     " -
  
   :
 
   $#   " L
    

    
  

    $ 
  -$ B
      
  
    




 
     " . 
      
?  
  
  
    'H" 4
 !
    -$  B
   
  
  "GJH   




*

    

  ">H 
 :
   
'H  



 
  'H  " E      
 M  
0    
  
 

    $  
       "


 



$      
0

   
  0

/  
    
    

   

" $

 


 
     
 





           

 
  

 "

  $ 
     



"" 
 
  
   
  
   
   


   


" .

    

 
   
   

 :



 " . $ 
  
 
 
  
  %

%

 "


% &   '()



 
       

    $  "

;           











 
  :
 

 
     I ' 9

    
 
 
  
      
   






 "

;  
 
 


     

 
 

 
#  

    

  


 M

 
  "

%  
  

%  7 
#  

 

 




M '

    

              
    !" " "  I  #        

"

The cat walks in the bedroom

1 / ;
 
   
# 
   

 
 

   
 
   
 
 
 
 

 " .   
#
   7        
   

  
      
  

   
 "


* '( + , $ ;     
#     
     
 
    
   
      

        
      

" 1 $     
#   

 

)


 
  
 

     


    
 

     


 
  
 



 
    


     

<

 

    
 
 
   
"

.      
  
 
  

7   
 9 9  
 
   
      0           

" ; 

     
 

    
  
  
 
   
 
"

  


 

.     
/         ""

  
 
 

    

 



         
   
     


   
  


"


- '(  $  " . 1 

    
#

  
        

 

        
" F       
  
#
 ""  
        

      $     
  
  / '" 3 
  

? :    " . 
 
    
 
 
    
    
     " .  
 
     
 
   
  
      

  
:      
"

.    

&



&

  

    

 " %  
 
  #   
" .  
               


  "

   
     




     
  
  
 

 
 
 
  

    
 
 


 
  
  # " " 1 
 
 
$ "" 7   
  
   
 

    
   
 
 
    

 " . 


    
$
   

">" D" 3 
    
 " . 
  
           
 

 " .   
$
   

"*" G" =  
     
 "

.    

  " 3    

   

D"*" (

J" =
#
  

 

   

 

 " """      
 


 
 
 

.  


    

@ 
 A     

  
 

" .     

  ""

   

  

   
 



    
"


/ 0 1 # 2  +3 $Æ ) E 
0  

  
  
 
 $  $ 
Æ
 
        
   
     $    " ;
$ 
    F 6
     
" 4
 
   
    
  @ 
 A      
   
 
  

""   

 
B
   

 
 



      
$" 3
       
  
   @ 
 A  
     
         
   
" 4             

 
    @ 
 A" L  

         

 

  
 

0   $ 
        "" 
 
    
"

%      
 
  

            
     $

      
 

&    $/







 

 
 $



 

&  

E      M   
 
    

 
   
     
      

"

%   
      
   /




 $   




6 
     
   
 
    
  
/







  



 
    





   
         '
  N    '




   
    $

'   /

 
     7  


* O  P C

 

  









  

 
  


 ´ 
µ 














 


 





 ´ 
µ 




            

 "

% 

        

 
  

 ""    
 ´ 
µ  





0       
/ 
   @ A



     @ A 
 

 
  
 
 " L   
       
   $ N  
    

'  





>"

.  
   # 
; D  " 8
 $
    ,
3 
3
  
   
 ;  D   
   
'J" % 

  

    
  
!
 


          
 


 $
"

!" "!   
  '  




   ""  
   
 #   
         
" 3 
  
   
 
   
       

 $
 " 'D

 
 !


 3
 =
 ;$





 
 

     
  

  '
  

 !  C   


 # 
  # 






   


'  D

G

L
        



'         $

" ;   
        
 


 " E


   



  
 !   
 
 

   
 
 
 

   !!    







7 
   # 
     
 





" %   



   
  
     
   
   
   "


           
 
    






  !     ! 

      F
     " .  
  
  
   
 
   
  F



  !  C ' 
 
       !    #"



F   
    $  
 
     
  
    

+



  
 
 
   

    
$  "

F  C  
 $    
 C   
  
$  C

    " F Q C  + 
    +        
 Q C 
   77  
 " E     
            + 
  

   /





  + 
  ,+ 
 I '  ,



    
 Q  Q


.         
        




+



       
   

 



*O


  + 
 P C *O P Q Q 


'G

,   / !     +         
" 3


.  
  
   @

 A 

     

+ 

  $      
   
 
 
  
  

  

 
" .       :
 
$   


" .  +     
   
 +           ""  
 
 
      
 
  
" 1   
  0  :   
      
" L  

+       $

 
 ! 
"

!" "( )
 #   & 
 ;

 :
   
 $
  
 ! 
    ;

 '    



  
 
 "



  


 
  

#

 
/

. 

    

 
 
#
 









   



  C







 
   
  



     
   0
  



   
 




 





 


  


  C C













" .              
          
 
     
  "   


 

$
  


  
  
; '  D 
      " %  
 
      
   / '" .     
   
  


 


 

C






 
   I  





  



 
    


   
 
 
J   
  $#
 







 




         
 

'J




 

 
*


7        @A 
  " % 

 
0   
    " " . 
 
       

     
 


 
  


    
     " 


   

 

    !
   
 
   $#       
 
#     
      !
  0
 
   


 

 

""       
     

 


   
   

" .       




   "

     

; @ 

A   


    
 
 @ 

A  
 @  A   
          " L
 #
    
   

  

  
   
 
    
  
$
   

; G" 

 
 ( ,
 '  D



$
N





$ 


 3
 =
 ;$





 
 

 C    ##
     '    

 
 
     


 C


 ´  µ   

 #
   ,N I '  ,  J


   G N




  





!" "+  
  
 N 4

      $ 
 



 $ 
 

    # 
       " 1 $ 
         
 
     "

  @ A     
 
  
 

   



   @
A     



       

"

1  
     

 

    

 

  $   @ #A     
    #/

*O




 
   P C *O  




   




 
   


   
    P C *O P     '>

   $
 
 

    
" 7        

   3
  



 "


  
 
 
      
 
   

 
   
     


" L
  
  $

    

    # 
    
 
  
   
"


 0
 
      
   


   


  
    
      
 

 
   
 
 

"




1      $  + 

3        
     
  
 
D">" .      
   

 
 

 

  
 " .           
$  
   
  

       
     
 
 
  
 " .    
    
  
 
 = =

 - '" 1   




    "

3 

!       
  
      


  
       
   


" .
 

   
  
    /


C






! 
   ! 
 !  ! 
     



% $ 
    

    
     
      
    

 

  0  
  
 
 
 " 1 $    
   
   
     

&

  
  
 

   
 

&  

   
!
 " ;      2  
 
 
 /


C




%-#$    %-#$   



  $ 
  

     
 



&"



&/

 


7   
9
      
 
 

#
    
  
    
"

  
9 "" 

    

       :
 
 
 " 8
 $
   2  
   

        @

 A



%-#$   


%-#$   



   
#
 

    
 
  

 
     
 :


  " L
  
      
   
 


" .     

   *

  

   

  !      





  " E  
 

 



 

% % !




% % ! ! % % !  

! % % !


  

  
  


 

    
    
 
  

   
 '*

 

 " 7#
  
 
     
  

  /

  !       C



 !

 



!

 
" #!
!









 
            
     
      

  
    

       

 0     " ; 

D">  
 
  
 
 
 
 "


  

 

   
 

  
       
 @
A    
 
  $ 
  '  

   
 " L
  
        
 
    
    /    '  
  

   
    " .        
#
     

     

          

     "


  $ 5 L
 
 
 
 

    
   

   
         

 "






     
      
    
    :
 

       

 

     " .  
 
 
          
        
 
 
 
 $  
   
    
   
 
$" .      
   
   
  
 :
 
    ""



E7 1 '(()" E7               
  
       
  @ 
A   $   
 
   

  " E7         


  

 
#
/  
"

          
  


  


  
 
 
 
"

.   

   


 0 

  
 
 


    
       
 
" F      
    
 
             
 
  "


 ! 

    

  
   "


 
  




  
 
      
 


 
  
 "

      #  $   
   

 
       
      
  
 " .   $ 

D" 
 D">" F  
 
     
    


    " .    
 
 
  
 
  ')

 
 
@0A    
   "" 

   

.     
   

 

"


A 

.


"

B        @ 
A/ 
 

.


 
 

/

    
 

"

 

   
 

    @

L
    


       
  /







/ 

.



.     
           
       $     # "

 . " 7  E7   .  
  /   

. 
 

 

   

  
     
  ""     
 
 

M   
       
     
 M ; Æ             
     
 
    
  " ; 
  

        

    
       
  
     
 !
  E7" L
              
    
  
 
      
   
" 3
!
 :
 
         :
 
 
 
"  

 $ 

.     

       
   "

. @ 
A    
 
 



    
    @A   
 """ 1     
  
 
   
       


 
 / "

* 
 0   

 

;    
  
 

 
  /

* 



0 



/





.



  
        

6

    " E
   
 "

*



0

   
  

.  
       
   "

*   
 0 *    
  < 0 *        $        
  "

. 
    6





7   
9   
   $  
 

           
 
  " L
     
        $  
     

         " 6

 
         
  "

'(

0 *



E 
!
     

 

 


/


     


 

0 "

&

 


 
  
 
 

 
 

. 

  


*



&"

.    
  


 /

* 



0 

&





  
   



/



.



    

7      
    $   



* & " 6 
 B-   
   
 
 .   * &  0 /  /  " ;
 
 B-   
   
 
 
/  *  & 0 " .  

   # 
"

 E7 $  
  

 /

 
 + L
 B-    L
3  .

" $ 
  
 
   
 % 
 
  

 * & 0      

.  * & 0   . C    . C  & ' 

 
  

 * & 0      

.  * & 0    C
 . C     C
 . C  & . C  
 . C  &


      
 & /   ( )
 
  

  .  & *  / 0   /  * 
  

 
   
 + 
 
  

 * & 0      

.  * & 0   . C    . C  & , 

 
  

 * & 0      

.  * & 0    C
 . C     C
 . C  & . C  
 . C  &


      
* 0  - )
 

  

 *   0 & " .      





C



 . C  C





; J    /



C

 C  C . C 



    $    

  
     . C   C
 
 

 
      " 1   0
     







   " .           /

C 



C



C


. C   C





C

 C  C . C   C   C  . C   

C


. C  

   
   
  
 
 /

C 

C


. C   C





C 

C



C


. C  

C 

C



C


. C  







 C  C

C



.         
 /





C

 C  C    C  C  

 C  

7     
  
 Æ 
      
    
      
      

  & C   C
 . C  

  

   
     

" 1 $  

&


   
 
 ""   

    $   
     
 

 " .

&

   

#   









0&



C

 

&



0

 
@ 
A 
 
"



"

0



C

 . 1



C

 .


 

   

$     


&

C





.   
  


' 
   
     
   


L
   
  $        



   
          $ 
Æ
" .  

                

&




& *



$"

&

C



L
   
  



  
      @

A  
 
" 1 $ 
   

&

   

  

& * 



  

@A 

 $" .   
 
   

 
 
    
 



*   ""

 



  

  

*

 

& " .    *  
  *         

*   """ & *  $         






7    

 .



 $    
   
    

 /    

 . 

& * 

 
 


          

    
" . 

    
   "



$ 
     
 

 
     
    
   
         

"


 .$ .4# . . $
      %--  

  



 
     

 $ 
    

    
   


 
 
 $*   " .     
                
   

  2  " .    $   0
 

       
 

'

   

 



 


 $   

 
   

 " . 

   
  



.



2     2


'

     . 

   $ #
    
  
  



 " . 
  
 
 
@A  






""   
 

  " . 
   #   
 /

$


       1 3  
 2 2  
  
 
 
O' 2  'P

2  C ' 
   CI2



 R C 2 

     R C '



7   
 Æ   

 
    
     B- " %    0   
    
  
 
    
  

 

D"D"


!  $ ... .   0     
      
  
  
  /  

       


"

 

/    

/

 

&    


  




"


  @ 
A 
 

/

   
 
 

 "

     
     

   / 
 &  
      
   &          
   2   
"

 



2     
  
 0  
    -  
2 C  
   
   
    "

 

1 D    

 
   " F       
D"D    
   
   
 
    
/

 & "



.   

  

 
  ""        
 
 " ;


 
    
   B- 
    
 
 
  
        
D"D    #
"



.

 $





½

 ·½

Multi−step jump process

½





½

Ontological category sequence

·½



½





 ·½



½

 ·½

Meaning sequence

 ·½

Word sequence

1 D/ ;
   
    %--" 7     
  

 
  
 
   




&"






 

&

2



 

.     
 
   
 
 


"


% " ,     &4 ; 


 

D"D 

    
"G   
      ""     
  
 


  

      
     
   " L
          

  

  "

 

 


;
$
 

     
     
 


 
             
  

    


   
# B- ""  

?L %-- 
 
 1 
 '((>"

  
 E7  

   
           
  
D"  
 
  E7    
#
  

    
         
   
  
 " .  
   

     
     
   '   9 "


 
  


 
     
    
  E7   
 
 " . 
  

   
 
     ' 
   
  

   
9 
  
 
" 3
 
 
    

 

  
         
 /





/ /

/

   /  C








 /  / /

 

  
 D



/




 /  
    


         $    E7" .
  
  
  
       $

 

     


  
        ""   
 
"
 
    
$
     
     ""  


     
      
 


 "


*  23 ( $  ' 3          

   
 
    
      

" . 
           
  
   
  
  
        
     


 " 1 $     
 
   

  

    
    
$     
     " 6

 
 
 
 
      
 

     
    $"


        


   
 
 
 



  


  4
 '(((+ 4
 '((*+ 4
 '(((+ 4 
 ,
 + 5
 '((( 
 
              

 

 

       
 "



 



   
   

     " 1 $  3 F
 - 4 
 ,
  

      " .   
 
!
  $ 
 
       
  $ 
 
   

   4 
 ,
 " .  
  

 
  
           

 
  
   
      0
  
 
 
 
  
     " . 


       
 

  / 


$    
   




$ 
$ 



 
 
    



  



 
   
  Æ 




   ,
 
 4 '(((

   
 

 
 
      
  7  
  



 

" L
   
           
 



  
  

   
    
     
  :   

" .        
     
 

   
     
     

" E         

      
 
 

D"> ""  
 

 
   
        


  

 
 3 F
 - 
 G

  






  

 Æ
 " . 
        

  
/ 

B    D   

    
  

  
 
       

  

?L %-- 
 
 1 
 '((>"

   
  "



.  




. 
 
  

  
         

D">"



8 
   !   @ A  
  
    @A ""  
 
    $   "

L            
  
  

 !       
    

 
  

 @
A 
 "


 
      

 


 

   @
 A "" 
   
  
   
      "



 

.        

  
 

 
 
 
  
 
 
  

 

 
    
     

 

     
  E79 
 
 
 
 $

 

      
   
     
 
  "
  -
4 
 
      
     $ 
  
 
 

    #   
$   
     

  


   " -
   
 
       
  
 
     
 !
      
   
 

 " L  
         
  
 

 
  
 
  

      

" 
 


.   


 
 
  
     
  



 
   
 
 
"

 *    .     
 =: %

 ,
3S  
3
 
 4 
,
     

 "

.        

  

73B54 

 
  - .;43 
 5 3
   
  4
 5  4 "

J




 3" 
 
 T" " .
 
    

 
0
  


% & 
       # 
     ' 

(  )  '
  '' D/JJUJJ*"

 


  "


 T" 
 
 3" " -
 

     

  "


3 3" ;" F
 ." &" 
 -V  &"5"  

   %     
 *   

 


 '  GUG>" - . 8 "


 T"  5" 
 

 8" " ;
   
 " .
 5 ''() " 5L 6
 S   -
S "


 T" 
 1 
 8" '((>"


?L %--   
  
"

& 
       # 

%

* J/'D'U'G("

 ;"  8 3" 
  8 " '((>" ; $ 
  
 
  
"

+   , 


/D(U*'"

4
 B" '(((" ; $

  " .
 5 43((' 
6
 " 4 4" 
 ,
 1" "

,  

3 
 
"

+ *
 

 'G G/)UDD"


-.  

/      
           "

8   

4
 -" '((*" . 
 $#      
"

(    +, 4
 -" '((("

 '>UD - 3
"

6
   8


"    3"   3" 1
  =" F
 ." 
 5"% 
'(("  
 
 
  "


$


0      
 *
   %    *
 




G' >/D('UG*" 1 4" '(()" 1  =" "

1  2  
 
, 
 '  

&   
  &    "

" - . 8 "

8    "

5L 6
 S  

-
S " = ," =
 ," 
 - ," '" .     
 
    

 
"




+   , 
 + ,    


 > '

 *U>" =
  4" 
 K ," '()>" 

 "

4

   
/ ;   
 


*  
 *
 


 '/''GU'G)"

=
 ," '" ;    

 
" .
 5 -35.5 '* -  5 "

>

%   ." '(()"

+ 

  

 ?
  
     "

'/'GJU'GDD"

%

 =" '()>" F

    

 
 "

   + 
  + *
 
*
 




 
  

 'U' ;  '()>"

F 
 B % " %

 =" " .

   $  
#
 
  
"

+ 

  

'G )/'**'U')"

, ." 
 ,
 -" '(((" 
  
 

  3
  %  


  "

0  

 '/('UD"

,
 1" 
 4 4" '(((" 8
 
 
 
 
"

+ 
 *
 +
  &





  

 '  &7'U&7J 

 " ,
 1" 
 - 5" F" '()"    "






  
 -  


=  B" 3" 
 &
 F" 7"  

   4
 

" 7%
 ; "

,

 &" 
 5  3" " 3
#
    $


  " ,
 -" '(()"




 
 %+*,"

,   5 
 (  "

&   7
 "

,
 -" =
 K" , ." 
 3 F" '(((" ;


 
  
  "


,
 -" 

,   5 
 (  "

- .

8  4 -;" &# 3" -" '()*"

B 
        
 



    
#"

 


% & 
  
 
*
  * 

;338DJ D/GUG'"

-


 4" 
 3# %" '((("

6   *  
    ,    
"

- . 8 " -

 5" 
  -" '(('"
  
   $
"

7 
  
  


+ *
 


 'J/DGDUD(("

8
 ;" 
 %

 =" " B$
    

 
 
 
   

  
"

0 + 
      # %0+7!888




 
  

%   

4 " BBB 7 T"

5
 ;" '(((" F

   
 
  $ 
  "

(
 , 

DG' /'J'U'*>"

5  3" 
 & ;" '((>"


 

  
 
 



  
   
 ! "  '>DU')D" *

0    +   9

3 ," '((>" 3 

 $  
"

  # 

% & 
    

* '/'GU'G>"

3# %" '((D" E "


%

 3" ," 4 
 ," " 
 =  4" F"  

 
    %     
 *   .

 " )(JU( 3
-

4;" -
&

" E  T" " 4
    

    "

   + 

'/'UG'"

)