@@ -0,0 +1,551 @@
/ / Copyright 2 0 2 5 T h e G o A u t h o r s . A l l r i g h t s r e s e r v e d .
/ / Use o f t h i s s o u r c e c o d e i s g o v e r n e d b y a B S D - s t y l e
/ / license t h a t c a n b e f o u n d i n t h e L I C E N S E f i l e .
/ / go : build r a c e
# include " g o _ a s m . h "
# include " f u n c d a t a . h "
# include " t e x t f l a g . h "
/ / The f o l l o w i n g t h u n k s a l l o w c a l l i n g t h e g c c - c o m p i l e d r a c e r u n t i m e d i r e c t l y
/ / from G o c o d e w i t h o u t g o i n g a l l t h e w a y t h r o u g h c g o .
/ / First, i t ' s m u c h f a s t e r ( u p t o 5 0 % s p e e d u p f o r r e a l G o p r o g r a m s ) .
/ / Second, i t e l i m i n a t e s r a c e - r e l a t e d s p e c i a l c a s e s f r o m c g o c a l l a n d s c h e d u l e r .
/ / Third, i n l o n g - t e r m i t w i l l a l l o w t o r e m o v e c y c l i c r u n t i m e / r a c e d e p e n d e n c y o n c m d / g o .
/ / A b r i e f r e c a p o f t h e r i s c v C c a l l i n g c o n v e n t i o n .
/ / Arguments a r e p a s s e d i n X 1 0 . . . X 1 7
/ / Callee- s a v e d r e g i s t e r s a r e : X 8 , X 9 , X 1 8 . . X 2 7
/ / Temporary r e g i s t e r s a r e : X 5 . . X 7 , X 2 8 . . X 3 1
/ / When c a l l i n g r a c e c a l l a d d r , X 1 1 i s t h e c a l l t a r g e t a d d r e s s .
/ / The r a c e c t x , T h r e a d S t a t e * t h r b e l o w , i s p a s s e d i n X 1 0 a n d l o a d e d i n r a c e c a l l a d d r .
/ / func r u n t i m e · r a c e r e a d ( a d d r u i n t p t r )
/ / Called f r o m i n s t r u m e n t e d c o d e .
TEXT r u n t i m e · r a c e r e a d < A B I I n t e r n a l > ( S B ) , N O S P L I T , $ 0 - 8
/ / void _ _ t s a n _ r e a d ( T h r e a d S t a t e * t h r , v o i d * a d d r , v o i d * p c ) ;
$ _ _ t s a n _ r e a d ( S B ) , X 5
MOV X 1 0 , X 1 1
MOV X 1 , X 1 2
JMP r a c e c a l l a d d r < > ( S B )
/ / func r u n t i m e · R a c e R e a d ( a d d r u i n t p t r )
TEXT r u n t i m e · R a c e R e a d ( S B ) , N O S P L I T , $ 0 - 8
/ / This n e e d s t o b e a t a i l c a l l , b e c a u s e r a c e r e a d r e a d s c a l l e r p c .
JMP r u n t i m e · r a c e r e a d ( S B )
/ / func r u n t i m e · r a c e r e a d p c ( v o i d * a d d r , v o i d * c a l l p c , v o i d * p c )
TEXT r u n t i m e · r a c e r e a d p c ( S B ) , N O S P L I T , $ 0 - 2 4
/ / void _ _ t s a n _ r e a d _ p c ( T h r e a d S t a t e * t h r , v o i d * a d d r , v o i d * c a l l p c , v o i d * p c ) ;
$ _ _ t s a n _ r e a d _ p c ( S B ) , X 5
MOV a d d r + 0 ( F P ) , X 1 1
MOV c a l l p c + 8 ( F P ) , X 1 2
MOV p c + 1 6 ( F P ) , X 1 3
JMP r a c e c a l l a d d r < > ( S B )
/ / func r u n t i m e · r a c e w r i t e ( a d d r u i n t p t r )
/ / Called f r o m i n s t r u m e n t e d c o d e .
TEXT r u n t i m e · r a c e w r i t e < A B I I n t e r n a l > ( S B ) , N O S P L I T , $ 0 - 8
/ / void _ _ t s a n _ w r i t e ( T h r e a d S t a t e * t h r , v o i d * a d d r , v o i d * p c ) ;
$ _ _ t s a n _ w r i t e ( S B ) , X 5
MOV X 1 0 , X 1 1
MOV X 1 , X 1 2
JMP r a c e c a l l a d d r < > ( S B )
/ / func r u n t i m e · R a c e W r i t e ( a d d r u i n t p t r )
TEXT r u n t i m e · R a c e W r i t e ( S B ) , N O S P L I T , $ 0 - 8
/ / This n e e d s t o b e a t a i l c a l l , b e c a u s e r a c e w r i t e r e a d s c a l l e r p c .
JMP r u n t i m e · r a c e w r i t e ( S B )
/ / func r u n t i m e · r a c e w r i t e p c ( v o i d * a d d r , v o i d * c a l l p c , v o i d * p c )
TEXT r u n t i m e · r a c e w r i t e p c ( S B ) , N O S P L I T , $ 0 - 2 4
/ / void _ _ t s a n _ w r i t e _ p c ( T h r e a d S t a t e * t h r , v o i d * a d d r , v o i d * c a l l p c , v o i d * p c ) ;
$ _ _ t s a n _ w r i t e _ p c ( S B ) , X 5
MOV a d d r + 0 ( F P ) , X 1 1
MOV c a l l p c + 8 ( F P ) , X 1 2
MOV p c + 1 6 ( F P ) , X 1 3
JMP r a c e c a l l a d d r < > ( S B )
/ / func r u n t i m e · r a c e r e a d r a n g e ( a d d r , s i z e u i n t p t r )
/ / Called f r o m i n s t r u m e n t e d c o d e .
TEXT r u n t i m e · r a c e r e a d r a n g e < A B I I n t e r n a l > ( S B ) , N O S P L I T , $ 0 - 1 6
/ / void _ _ t s a n _ r e a d _ r a n g e ( T h r e a d S t a t e * t h r , v o i d * a d d r , u i n t p t r s i z e , v o i d * p c ) ;
$ _ _ t s a n _ r e a d _ r a n g e ( S B ) , X 5
MOV X 1 1 , X 1 2
MOV X 1 0 , X 1 1
MOV X 1 , X 1 3
JMP r a c e c a l l a d d r < > ( S B )
/ / func r u n t i m e · R a c e R e a d R a n g e ( a d d r , s i z e u i n t p t r )
TEXT r u n t i m e · R a c e R e a d R a n g e ( S B ) , N O S P L I T , $ 0 - 1 6
/ / This n e e d s t o b e a t a i l c a l l , b e c a u s e r a c e r e a d r a n g e r e a d s c a l l e r p c .
JMP r u n t i m e · r a c e r e a d r a n g e ( S B )
/ / func r u n t i m e · r a c e r e a d r a n g e p c1 ( v o i d * a d d r , u i n t p t r s z , v o i d * p c )
TEXT r u n t i m e · r a c e r e a d r a n g e p c1 ( S B ) , N O S P L I T , $ 0 - 2 4
/ / void _ _ t s a n _ r e a d _ r a n g e ( T h r e a d S t a t e * t h r , v o i d * a d d r , u i n t p t r s i z e , v o i d * p c ) ;
$ _ _ t s a n _ r e a d _ r a n g e ( S B ) , X 5
MOV a d d r + 0 ( F P ) , X 1 1
MOV s i z e + 8 ( F P ) , X 1 2
MOV p c + 1 6 ( F P ) , X 1 3
/ / pc i s a n i n t e r c e p t o r a d d r e s s , b u t T S a n e x p e c t s i t t o p o i n t t o t h e
/ / middle o f a n i n t e r c e p t o r ( s e e L L V M ' s S C O P E D _ I N T E R C E P T O R _ R A W ) .
ADD $ 4 , X 1 3
JMP r a c e c a l l a d d r < > ( S B )
/ / func r u n t i m e · r a c e w r i t e r a n g e ( a d d r , s i z e u i n t p t r )
/ / Called f r o m i n s t r u m e n t e d c o d e .
TEXT r u n t i m e · r a c e w r i t e r a n g e < A B I I n t e r n a l > ( S B ) , N O S P L I T , $ 0 - 1 6
/ / void _ _ t s a n _ w r i t e _ r a n g e ( T h r e a d S t a t e * t h r , v o i d * a d d r , u i n t p t r s i z e , v o i d * p c ) ;
$ _ _ t s a n _ w r i t e _ r a n g e ( S B ) , X 5
MOV X 1 1 , X 1 2
MOV X 1 0 , X 1 1
MOV X 1 , X 1 3
JMP r a c e c a l l a d d r < > ( S B )
/ / func r u n t i m e · R a c e W r i t e R a n g e ( a d d r , s i z e u i n t p t r )
TEXT r u n t i m e · R a c e W r i t e R a n g e ( S B ) , N O S P L I T , $ 0 - 1 6
/ / This n e e d s t o b e a t a i l c a l l , b e c a u s e r a c e w r i t e r a n g e r e a d s c a l l e r p c .
JMP r u n t i m e · r a c e w r i t e r a n g e ( S B )
/ / func r u n t i m e · r a c e w r i t e r a n g e p c1 ( v o i d * a d d r , u i n t p t r s z , v o i d * p c )
TEXT r u n t i m e · r a c e w r i t e r a n g e p c1 ( S B ) , N O S P L I T , $ 0 - 2 4
/ / void _ _ t s a n _ w r i t e _ r a n g e ( T h r e a d S t a t e * t h r , v o i d * a d d r , u i n t p t r s i z e , v o i d * p c ) ;
$ _ _ t s a n _ w r i t e _ r a n g e ( S B ) , X 5
MOV a d d r + 0 ( F P ) , X 1 1
MOV s i z e + 8 ( F P ) , X 1 2
MOV p c + 1 6 ( F P ) , X 1 3
/ / pc i s a n i n t e r c e p t o r a d d r e s s , b u t T S a n e x p e c t s i t t o p o i n t t o t h e
/ / middle o f a n i n t e r c e p t o r ( s e e L L V M ' s S C O P E D _ I N T E R C E P T O R _ R A W ) .
ADD $ 4 , X 1 3
JMP r a c e c a l l a d d r < > ( S B )
/ / If a d d r ( X 1 1 ) i s o u t o f r a n g e , d o n o t h i n g . O t h e r w i s e , s e t u p g o r o u t i n e c o n t e x t a n d
/ / invoke r a c e c a l l . O t h e r a r g u m e n t s a r e a l r e a d y s e t .
TEXT r a c e c a l l a d d r < > ( S B ) , N O S P L I T , $ 0 - 0
MOV r u n t i m e · r a c e a r e n a s t a r t ( S B ) , X 7
BLT X 1 1 , X 7 , d a t a / / B e f o r e r a c e a r e n a s t a r t ?
MOV r u n t i m e · r a c e a r e n a e n d ( S B ) , X 7
BLT X 1 1 , X 7 , c a l l / / B e f o r e r a c e a r e n a e n d ?
data :
MOV r u n t i m e · r a c e d a t a s t a r t ( S B ) , X 7
BLT X 1 1 , X 7 , r e t / / B e f o r e r a c e d a t a s t a r t ?
MOV r u n t i m e · r a c e d a t a e n d ( S B ) , X 7
BGE X 1 1 , X 7 , r e t / / A t o r a f t e r r a c e d a t a e n d ?
call :
MOV g _ r a c e c t x ( g ) , X 1 0
JMP r a c e c a l l < > ( S B )
ret :
RET
/ / func r u n t i m e · r a c e f u n c e n t e r ( p c u i n t p t r )
/ / Called f r o m i n s t r u m e n t e d c o d e .
TEXT r u n t i m e · r a c e f u n c e n t e r < A B I I n t e r n a l > ( S B ) , N O S P L I T , $ 0 - 8
MOV $ _ _ t s a n _ f u n c _ e n t e r ( S B ) , X 5
MOV X 1 0 , X 1 1
MOV g _ r a c e c t x ( g ) , X 1 0
JMP r a c e c a l l < > ( S B )
/ / Common c o d e f o r r a c e f u n c e n t e r
/ / X1 = c a l l e r ' s r e t u r n a d d r e s s
TEXT r a c e f u n c e n t e r < > ( S B ) , N O S P L I T , $ 0 - 0
/ / void _ _ t s a n _ f u n c _ e n t e r ( T h r e a d S t a t e * t h r , v o i d * p c ) ;
$ _ _ t s a n _ f u n c _ e n t e r ( S B ) , X 5
MOV g _ r a c e c t x ( g ) , X 1 0
MOV X 1 , X 1 1
JMP r a c e c a l l < > ( S B )
/ / func r u n t i m e · r a c e f u n c e x i t ( )
/ / Called f r o m i n s t r u m e n t e d c o d e .
TEXT r u n t i m e · r a c e f u n c e x i t < A B I I n t e r n a l > ( S B ) , N O S P L I T , $ 0 - 0
/ / void _ _ t s a n _ f u n c _ e x i t ( T h r e a d S t a t e * t h r ) ;
$ _ _ t s a n _ f u n c _ e x i t ( S B ) , X 5
MOV g _ r a c e c t x ( g ) , X 1 0
JMP r a c e c a l l < > ( S B )
/ / Atomic o p e r a t i o n s f o r s y n c / a t o m i c p a c k a g e .
/ / Load
TEXT s y n c ∕ a t o m i c · L o a d I n t 3 2 ( S B ) , N O S P L I T , $ 0 - 1 2
GO_ A R G S
MOV $ _ _ t s a n _ g o _ a t o m i c32 _ l o a d ( S B ) , X 5
CALL r a c e c a l l a t o m i c < > ( S B )
RET
TEXT s y n c ∕ a t o m i c · L o a d I n t 6 4 ( S B ) , N O S P L I T , $ 0 - 1 6
GO_ A R G S
MOV $ _ _ t s a n _ g o _ a t o m i c64 _ l o a d ( S B ) , X 5
CALL r a c e c a l l a t o m i c < > ( S B )
RET
TEXT s y n c ∕ a t o m i c · L o a d U i n t 3 2 ( S B ) , N O S P L I T , $ 0 - 1 2
GO_ A R G S
JMP s y n c ∕ a t o m i c · L o a d I n t 3 2 ( S B )
TEXT s y n c ∕ a t o m i c · L o a d U i n t 6 4 ( S B ) , N O S P L I T , $ 0 - 1 6
GO_ A R G S
JMP s y n c ∕ a t o m i c · L o a d I n t 6 4 ( S B )
TEXT s y n c ∕ a t o m i c · L o a d U i n t p t r ( S B ) , N O S P L I T , $ 0 - 1 6
GO_ A R G S
JMP s y n c ∕ a t o m i c · L o a d I n t 6 4 ( S B )
TEXT s y n c ∕ a t o m i c · L o a d P o i n t e r ( S B ) , N O S P L I T , $ 0 - 1 6
GO_ A R G S
JMP s y n c ∕ a t o m i c · L o a d I n t 6 4 ( S B )
/ / Store
TEXT s y n c ∕ a t o m i c · S t o r e I n t 3 2 ( S B ) , N O S P L I T , $ 0 - 1 2
GO_ A R G S
MOV $ _ _ t s a n _ g o _ a t o m i c32 _ s t o r e ( S B ) , X 5
CALL r a c e c a l l a t o m i c < > ( S B )
RET
TEXT s y n c ∕ a t o m i c · S t o r e I n t 6 4 ( S B ) , N O S P L I T , $ 0 - 1 6
GO_ A R G S
MOV $ _ _ t s a n _ g o _ a t o m i c64 _ s t o r e ( S B ) , X 5
CALL r a c e c a l l a t o m i c < > ( S B )
RET
TEXT s y n c ∕ a t o m i c · S t o r e U i n t 3 2 ( S B ) , N O S P L I T , $ 0 - 1 2
GO_ A R G S
JMP s y n c ∕ a t o m i c · S t o r e I n t 3 2 ( S B )
TEXT s y n c ∕ a t o m i c · S t o r e U i n t 6 4 ( S B ) , N O S P L I T , $ 0 - 1 6
GO_ A R G S
JMP s y n c ∕ a t o m i c · S t o r e I n t 6 4 ( S B )
TEXT s y n c ∕ a t o m i c · S t o r e U i n t p t r ( S B ) , N O S P L I T , $ 0 - 1 6
GO_ A R G S
JMP s y n c ∕ a t o m i c · S t o r e I n t 6 4 ( S B )
/ / Swap
TEXT s y n c ∕ a t o m i c · S w a p I n t 3 2 ( S B ) , N O S P L I T , $ 0 - 2 0
GO_ A R G S
MOV $ _ _ t s a n _ g o _ a t o m i c32 _ e x c h a n g e ( S B ) , X 5
CALL r a c e c a l l a t o m i c < > ( S B )
RET
TEXT s y n c ∕ a t o m i c · S w a p I n t 6 4 ( S B ) , N O S P L I T , $ 0 - 2 4
GO_ A R G S
MOV $ _ _ t s a n _ g o _ a t o m i c64 _ e x c h a n g e ( S B ) , X 5
CALL r a c e c a l l a t o m i c < > ( S B )
RET
TEXT s y n c ∕ a t o m i c · S w a p U i n t 3 2 ( S B ) , N O S P L I T , $ 0 - 2 0
GO_ A R G S
JMP s y n c ∕ a t o m i c · S w a p I n t 3 2 ( S B )
TEXT s y n c ∕ a t o m i c · S w a p U i n t 6 4 ( S B ) , N O S P L I T , $ 0 - 2 4
GO_ A R G S
JMP s y n c ∕ a t o m i c · S w a p I n t 6 4 ( S B )
TEXT s y n c ∕ a t o m i c · S w a p U i n t p t r ( S B ) , N O S P L I T , $ 0 - 2 4
GO_ A R G S
JMP s y n c ∕ a t o m i c · S w a p I n t 6 4 ( S B )
/ / Add
TEXT s y n c ∕ a t o m i c · A d d I n t 3 2 ( S B ) , N O S P L I T , $ 0 - 2 0
GO_ A R G S
MOV $ _ _ t s a n _ g o _ a t o m i c32 _ f e t c h _ a d d ( S B ) , X 5
CALL r a c e c a l l a t o m i c < > ( S B )
/ / TSan p e r f o r m e d f e t c h _ a d d , b u t G o n e e d s a d d _ f e t c h .
MOVW a d d + 8 ( F P ) , X 5
MOVW r e t + 1 6 ( F P ) , X 6
ADD X 5 , X 6 , X 5
MOVW X 5 , r e t + 1 6 ( F P )
RET
TEXT s y n c ∕ a t o m i c · A d d I n t 6 4 ( S B ) , N O S P L I T , $ 0 - 2 4
GO_ A R G S
MOV $ _ _ t s a n _ g o _ a t o m i c64 _ f e t c h _ a d d ( S B ) , X 5
CALL r a c e c a l l a t o m i c < > ( S B )
/ / TSan p e r f o r m e d f e t c h _ a d d , b u t G o n e e d s a d d _ f e t c h .
MOV a d d + 8 ( F P ) , X 5
MOV r e t + 1 6 ( F P ) , X 6
ADD X 5 , X 6 , X 5
MOV X 5 , r e t + 1 6 ( F P )
RET
TEXT s y n c ∕ a t o m i c · A d d U i n t 3 2 ( S B ) , N O S P L I T , $ 0 - 2 0
GO_ A R G S
JMP s y n c ∕ a t o m i c · A d d I n t 3 2 ( S B )
TEXT s y n c ∕ a t o m i c · A d d U i n t 6 4 ( S B ) , N O S P L I T , $ 0 - 2 4
GO_ A R G S
JMP s y n c ∕ a t o m i c · A d d I n t 6 4 ( S B )
TEXT s y n c ∕ a t o m i c · A d d U i n t p t r ( S B ) , N O S P L I T , $ 0 - 2 4
GO_ A R G S
JMP s y n c ∕ a t o m i c · A d d I n t 6 4 ( S B )
/ / And
TEXT s y n c ∕ a t o m i c · A n d I n t 3 2 ( S B ) , N O S P L I T , $ 0 - 2 0
GO_ A R G S
MOV $ _ _ t s a n _ g o _ a t o m i c32 _ f e t c h _ a n d ( S B ) , X 5
CALL r a c e c a l l a t o m i c < > ( S B )
RET
TEXT s y n c ∕ a t o m i c · A n d I n t 6 4 ( S B ) , N O S P L I T , $ 0 - 2 4
GO_ A R G S
MOV $ _ _ t s a n _ g o _ a t o m i c64 _ f e t c h _ a n d ( S B ) , X 5
CALL r a c e c a l l a t o m i c < > ( S B )
RET
TEXT s y n c ∕ a t o m i c · A n d U i n t 3 2 ( S B ) , N O S P L I T , $ 0 - 2 0
GO_ A R G S
JMP s y n c ∕ a t o m i c · A n d I n t 3 2 ( S B )
TEXT s y n c ∕ a t o m i c · A n d U i n t 6 4 ( S B ) , N O S P L I T , $ 0 - 2 4
GO_ A R G S
JMP s y n c ∕ a t o m i c · A n d I n t 6 4 ( S B )
TEXT s y n c ∕ a t o m i c · A n d U i n t p t r ( S B ) , N O S P L I T , $ 0 - 2 4
GO_ A R G S
JMP s y n c ∕ a t o m i c · A n d I n t 6 4 ( S B )
/ / Or
TEXT s y n c ∕ a t o m i c · O r I n t 3 2 ( S B ) , N O S P L I T , $ 0 - 2 0
GO_ A R G S
MOV $ _ _ t s a n _ g o _ a t o m i c32 _ f e t c h _ o r ( S B ) , X 5
CALL r a c e c a l l a t o m i c < > ( S B )
RET
TEXT s y n c ∕ a t o m i c · O r I n t 6 4 ( S B ) , N O S P L I T , $ 0 - 2 4
GO_ A R G S
MOV $ _ _ t s a n _ g o _ a t o m i c64 _ f e t c h _ o r ( S B ) , X 5
CALL r a c e c a l l a t o m i c < > ( S B )
RET
TEXT s y n c ∕ a t o m i c · O r U i n t 3 2 ( S B ) , N O S P L I T , $ 0 - 2 0
GO_ A R G S
JMP s y n c ∕ a t o m i c · O r I n t 3 2 ( S B )
TEXT s y n c ∕ a t o m i c · O r U i n t 6 4 ( S B ) , N O S P L I T , $ 0 - 2 4
GO_ A R G S
JMP s y n c ∕ a t o m i c · O r I n t 6 4 ( S B )
TEXT s y n c ∕ a t o m i c · O r U i n t p t r ( S B ) , N O S P L I T , $ 0 - 2 4
GO_ A R G S
JMP s y n c ∕ a t o m i c · O r I n t 6 4 ( S B )
/ / CompareAndSwap
TEXT s y n c ∕ a t o m i c · C o m p a r e A n d S w a p I n t 3 2 ( S B ) , N O S P L I T , $ 0 - 1 7
GO_ A R G S
MOV $ _ _ t s a n _ g o _ a t o m i c32 _ c o m p a r e _ e x c h a n g e ( S B ) , X 5
CALL r a c e c a l l a t o m i c < > ( S B )
RET
TEXT s y n c ∕ a t o m i c · C o m p a r e A n d S w a p I n t 6 4 ( S B ) , N O S P L I T , $ 0 - 2 5
GO_ A R G S
MOV $ _ _ t s a n _ g o _ a t o m i c64 _ c o m p a r e _ e x c h a n g e ( S B ) , X 5
CALL r a c e c a l l a t o m i c < > ( S B )
RET
TEXT s y n c ∕ a t o m i c · C o m p a r e A n d S w a p U i n t 3 2 ( S B ) , N O S P L I T , $ 0 - 1 7
GO_ A R G S
JMP s y n c ∕ a t o m i c · C o m p a r e A n d S w a p I n t 3 2 ( S B )
TEXT s y n c ∕ a t o m i c · C o m p a r e A n d S w a p U i n t 6 4 ( S B ) , N O S P L I T , $ 0 - 2 5
GO_ A R G S
JMP s y n c ∕ a t o m i c · C o m p a r e A n d S w a p I n t 6 4 ( S B )
TEXT s y n c ∕ a t o m i c · C o m p a r e A n d S w a p U i n t p t r ( S B ) , N O S P L I T , $ 0 - 2 5
GO_ A R G S
JMP s y n c ∕ a t o m i c · C o m p a r e A n d S w a p I n t 6 4 ( S B )
/ / Generic a t o m i c o p e r a t i o n i m p l e m e n t a t i o n .
/ / X5 = a d d r o f t a r g e t f u n c t i o n
TEXT r a c e c a l l a t o m i c < > ( S B ) , N O S P L I T , $ 0
/ / Set u p t h e s e r e g i s t e r s
/ / X1 0 = * T h r e a d S t a t e
/ / X1 1 = c a l l e r p c
/ / X1 2 = p c
/ / X1 3 = a d d r o f i n c o m i n g a r g l i s t
/ / Trigger S I G S E G V e a r l y .
MOV 2 4 ( X 2 ) , X 6 / / 1 s t a r g i s a d d r . a f t e r t w o t i m e s C A L L , g e t i t a t 2 4 ( X 2 )
MOVB ( X 6 ) , X 0 / / s e g v h e r e i f a d d r i s b a d
/ / Check t h a t a d d r i s w i t h i n [ a r e n a s t a r t , a r e n a e n d ) o r w i t h i n [ r a c e d a t a s t a r t , r a c e d a t a e n d ) .
MOV r u n t i m e · r a c e a r e n a s t a r t ( S B ) , X 7
BLT X 6 , X 7 , r a c e c a l l a t o m i c _ d a t a
MOV r u n t i m e · r a c e a r e n a e n d ( S B ) , X 7
BLT X 6 , X 7 , r a c e c a l l a t o m i c _ o k
racecallatomic_data :
MOV r u n t i m e · r a c e d a t a s t a r t ( S B ) , X 7
BLT X 6 , X 7 , r a c e c a l l a t o m i c _ i g n o r e
MOV r u n t i m e · r a c e d a t a e n d ( S B ) , X 7
BGE X 6 , X 7 , r a c e c a l l a t o m i c _ i g n o r e
racecallatomic_ok :
/ / Addr i s w i t h i n t h e g o o d r a n g e , c a l l t h e a t o m i c f u n c t i o n .
MOV g _ r a c e c t x ( g ) , X 1 0 / / g o r o u t i n e c o n t e x t
MOV 8 ( X 2 ) , X 1 1 / / c a l l e r p c
MOV X 1 , X 1 2 / / p c
ADD $ 2 4 , X 2 , X 1 3
CALL r a c e c a l l < > ( S B )
RET
racecallatomic_ignore :
/ / Addr i s o u t s i d e t h e g o o d r a n g e .
/ / Call _ _ t s a n _ g o _ i g n o r e _ s y n c _ b e g i n t o i g n o r e s y n c h r o n i z a t i o n d u r i n g t h e a t o m i c o p .
/ / An a t t e m p t t o s y n c h r o n i z e o n t h e a d d r e s s w o u l d c a u s e c r a s h .
MOV X 1 , X 2 0 / / s a v e P C
MOV X 5 , X 2 1 / / s a v e t a r g e t f u n c t i o n
MOV $ _ _ t s a n _ g o _ i g n o r e _ s y n c _ b e g i n ( S B ) , X 5
MOV g _ r a c e c t x ( g ) , X 1 0 / / g o r o u t i n e c o n t e x t
CALL r a c e c a l l < > ( S B )
MOV X 2 1 , X 5 / / r e s t o r e t h e t a r g e t f u n c t i o n
/ / Call t h e a t o m i c f u n c t i o n .
MOV g _ r a c e c t x ( g ) , X 1 0 / / g o r o u t i n e c o n t e x t
MOV 8 ( X 2 ) , X 1 1 / / c a l l e r p c
MOV X 2 0 , X 1 2 / / p c
ADD $ 2 4 , X 2 , X 1 3 / / a r g u m e n t s
CALL r a c e c a l l < > ( S B )
/ / Call _ _ t s a n _ g o _ i g n o r e _ s y n c _ e n d .
MOV $ _ _ t s a n _ g o _ i g n o r e _ s y n c _ e n d ( S B ) , X 5
MOV g _ r a c e c t x ( g ) , X 1 0 / / g o r o u t i n e c o n t e x t
CALL r a c e c a l l < > ( S B )
RET
/ / func r u n t i m e · r a c e c a l l ( v o i d ( * f ) ( . . . ) , . . . )
/ / Calls C f u n c t i o n f f r o m r a c e r u n t i m e a n d p a s s e s u p t o 4 a r g u m e n t s t o i t .
/ / The a r g u m e n t s a r e n e v e r h e a p - o b j e c t - p r e s e r v i n g p o i n t e r s , s o w e p r e t e n d t h e r e
/ / are n o a r g u m e n t s .
TEXT r u n t i m e · r a c e c a l l ( S B ) , N O S P L I T , $ 0 - 0
MOV f n + 0 ( F P ) , X 5
MOV a r g 0 + 8 ( F P ) , X 1 0
MOV a r g 1 + 1 6 ( F P ) , X 1 1
MOV a r g 2 + 2 4 ( F P ) , X 1 2
MOV a r g 3 + 3 2 ( F P ) , X 1 3
JMP r a c e c a l l < > ( S B )
/ / Switches S P t o g 0 s t a c k a n d c a l l s X 5 . A r g u m e n t s a r e a l r e a d y s e t .
TEXT r a c e c a l l < > ( S B ) , N O S P L I T | N O F R A M E , $ 0 - 0
MOV X 1 , X 1 8 / / S a v e R A i n c a l l e e s a v e r e g i s t e r
MOV X 2 , X 1 9 / / S a v e S P i n c a l l e e s a v e r e g i s t e r
CALL r u n t i m e · s a v e _ g ( S B ) / / S a v e g f o r c a l l b a c k s
MOV g _ m ( g ) , X 6
/ / Switch t o g 0 s t a c k i f w e a r e n ' t a l r e a d y o n g 0 o r g s i g n a l .
MOV m _ g s i g n a l ( X 6 ) , X 7
BEQ X 7 , g , c a l l
MOV m _ g 0 ( X 6 ) , X 7
BEQ X 7 , g , c a l l
MOV ( g _ s c h e d + g o b u f _ s p ) ( X 7 ) , X 2 / / S w i t c h t o g 0 s t a c k
call :
JALR R A , ( X 5 ) / / C a l l C f u n c t i o n
MOV X 1 9 , X 2 / / R e s t o r e S P
JMP ( X 1 8 ) / / R e t u r n t o G o .
/ / C- > G o c a l l b a c k t h u n k t h a t a l l o w s t o c a l l r u n t i m e · r a c e s y m b o l i z e f r o m C c o d e .
/ / Direct G o - > C r a c e c a l l h a s o n l y s w i t c h e d S P , f i n i s h g - > g 0 s w i t c h b y s e t t i n g c o r r e c t g .
/ / The o v e r a l l e f f e c t o f G o - > C - > G o c a l l c h a i n i s s i m i l a r t o t h a t o f m c a l l .
/ / R0 c o n t a i n s c o m m a n d c o d e . R 1 c o n t a i n s c o m m a n d - s p e c i f i c c o n t e x t .
/ / See r a c e c a l l b a c k f o r c o m m a n d c o d e s .
TEXT r u n t i m e · r a c e c a l l b a c k t h u n k ( S B ) , N O S P L I T | N O F R A M E , $ 0
/ / Handle c o m m a n d r a c e G e t P r o c C m d ( 0 ) h e r e .
/ / First, c o d e b e l o w a s s u m e s t h a t w e a r e o n c u r g , w h i l e r a c e G e t P r o c C m d
/ / can b e e x e c u t e d o n g 0 . S e c o n d , i t i s c a l l e d f r e q u e n t l y , s o w i l l
/ / benefit f r o m t h i s f a s t p a t h .
BNEZ X 1 0 , r e s t
MOV X 1 , X 5
MOV g , X 6
CALL r u n t i m e · l o a d _ g ( S B )
MOV g _ m ( g ) , X 7
MOV m _ p ( X 7 ) , X 7
MOV p _ r a c e p r o c c t x ( X 7 ) , X 7
MOV X 7 , ( X 1 1 )
MOV X 6 , g
JMP ( X 5 )
rest :
/ / Save c a l l e e - s a v e r e g i s t e r s ( X 8 , X 9 , X 1 8 . . X 2 7 , F 8 , F 9 , F 1 8 . . F 2 7 ) ,
/ / since G o c o d e w i l l n o t r e s p e c t t h i s .
/ / 8 ( X2 ) a n d 1 6 ( X 2 ) a r e f o r a r g s p a s s e d t o r a c e c a l l b a c k
SUB $ ( 2 7 * 8 ) , X 2
MOV X 1 , ( 0 * 8 ) ( X 2 )
MOV X 8 , ( 3 * 8 ) ( X 2 )
MOV X 9 , ( 4 * 8 ) ( X 2 )
MOV X 1 8 , ( 5 * 8 ) ( X 2 )
MOV X 1 9 , ( 6 * 8 ) ( X 2 )
MOV X 2 0 , ( 7 * 8 ) ( X 2 )
MOV X 2 1 , ( 8 * 8 ) ( X 2 )
MOV X 2 2 , ( 9 * 8 ) ( X 2 )
MOV X 2 3 , ( 1 0 * 8 ) ( X 2 )
MOV X 2 4 , ( 1 1 * 8 ) ( X 2 )
MOV X 2 5 , ( 1 2 * 8 ) ( X 2 )
MOV X 2 6 , ( 1 3 * 8 ) ( X 2 )
MOV g , ( 1 4 * 8 ) ( X 2 )
MOVD F 8 , ( 1 5 * 8 ) ( X 2 )
MOVD F 9 , ( 1 6 * 8 ) ( X 2 )
MOVD F 1 8 , ( 1 7 * 8 ) ( X 2 )
MOVD F 1 9 , ( 1 8 * 8 ) ( X 2 )
MOVD F 2 0 , ( 1 9 * 8 ) ( X 2 )
MOVD F 2 1 , ( 2 0 * 8 ) ( X 2 )
MOVD F 2 2 , ( 2 1 * 8 ) ( X 2 )
MOVD F 2 3 , ( 2 2 * 8 ) ( X 2 )
MOVD F 2 4 , ( 2 3 * 8 ) ( X 2 )
MOVD F 2 5 , ( 2 4 * 8 ) ( X 2 )
MOVD F 2 6 , ( 2 5 * 8 ) ( X 2 )
MOVD F 2 7 , ( 2 6 * 8 ) ( X 2 )
/ / Set g = g 0 .
CALL r u n t i m e · l o a d _ g ( S B )
MOV g _ m ( g ) , X 5
MOV m _ g 0 ( X 5 ) , X 6
BEQ X 6 , g , n o s w i t c h / / b r a n c h i f a l r e a d y o n g 0
MOV X 6 , g
MOV X 1 0 , 8 ( X 2 ) / / f u n c a r g
MOV X 1 1 , 1 6 ( X 2 ) / / f u n c a r g
CALL r u n t i m e · r a c e c a l l b a c k ( S B )
/ / All r e g i s t e r s a r e s m a s h e d a f t e r G o c o d e , r e l o a d .
MOV g _ m ( g ) , X 5
MOV m _ c u r g ( X 5 ) , g / / g = m - > c u r g
ret :
/ / Restore c a l l e e - s a v e r e g i s t e r s .
MOV ( 0 * 8 ) ( X 2 ) , X 1
MOV ( 3 * 8 ) ( X 2 ) , X 8
MOV ( 4 * 8 ) ( X 2 ) , X 9
MOV ( 5 * 8 ) ( X 2 ) , X 1 8
MOV ( 6 * 8 ) ( X 2 ) , X 1 9
MOV ( 7 * 8 ) ( X 2 ) , X 2 0
MOV ( 8 * 8 ) ( X 2 ) , X 2 1
MOV ( 9 * 8 ) ( X 2 ) , X 2 2
MOV ( 1 0 * 8 ) ( X 2 ) , X 2 3
MOV ( 1 1 * 8 ) ( X 2 ) , X 2 4
MOV ( 1 2 * 8 ) ( X 2 ) , X 2 5
MOV ( 1 3 * 8 ) ( X 2 ) , X 2 6
MOV ( 1 4 * 8 ) ( X 2 ) , g
MOVD ( 1 5 * 8 ) ( X 2 ) , F 8
MOVD ( 1 6 * 8 ) ( X 2 ) , F 9
MOVD ( 1 7 * 8 ) ( X 2 ) , F 1 8
MOVD ( 1 8 * 8 ) ( X 2 ) , F 1 9
MOVD ( 1 9 * 8 ) ( X 2 ) , F 2 0
MOVD ( 2 0 * 8 ) ( X 2 ) , F 2 1
MOVD ( 2 1 * 8 ) ( X 2 ) , F 2 2
MOVD ( 2 2 * 8 ) ( X 2 ) , F 2 3
MOVD ( 2 3 * 8 ) ( X 2 ) , F 2 4
MOVD ( 2 4 * 8 ) ( X 2 ) , F 2 5
MOVD ( 2 5 * 8 ) ( X 2 ) , F 2 6
MOVD ( 2 6 * 8 ) ( X 2 ) , F 2 7
ADD $ ( 2 7 * 8 ) , X 2
JMP ( X 1 )
noswitch :
/ / already o n g 0
MOV X 1 0 , 8 ( X 2 ) / / f u n c a r g
MOV X 1 1 , 1 6 ( X 2 ) / / f u n c a r g
CALL r u n t i m e · r a c e c a l l b a c k ( S B )
JMP r e t
Joel Sing