// TITLE("Interlocked Support") //++ // // Copyright (c) 1990 Microsoft Corporation // Copyright (c) 1992 Digital Equipment Corporation // // Module Name: // // intrlock.s // // Abstract: // // This module implements functions to support interlocked operations. // Interlocked operations can only operate on nonpaged data and the // specified spinlock cannot be used for any other purpose. // // Author: // // David N. Cutler (davec) 26-Mar-1990 // // Environment: // // Kernel mode. // // Revision History: // // Thomas Van Baak (tvb) 18-May-1992 // // Adapted for Alpha AXP. // //-- #include "ksalpha.h" SBTTL("Interlocked Add Large Integer") //++ // // LARGE_INTEGER // ExInterlockedAddLargeInteger ( // IN PLARGE_INTEGER Addend, // IN LARGE_INTEGER Increment, // IN PKSPIN_LOCK Lock // ) // // Routine Description: // // This function performs an interlocked add of an increment value to an // addend variable of type large integer. The initial value of the addend // variable is returned as the function value. // // Arguments: // // Addend (a0) - Supplies a pointer to a variable whose value is to be // adjusted by the increment value. // // Increment (a1) - Supplies the increment value to be added to the // addend variable. // // Lock (a2) - Supplies a pointer to a spin lock to be used to synchronize // access to the addend variable. // // Return Value: // // The result of the interlocked large integer add. // // Implementation Note: // // The specification of this function requires that the given lock must be // used to synchronize the update even though on Alpha the operation can // actually done atomically without using the specified lock. // //-- LEAF_ENTRY(ExInterlockedAddLargeInteger) 10: DISABLE_INTERRUPTS #if !defined(NT_UP) ldl_l t0, 0(a2) // get current lock value - locked bne t0, 20f // if ne, spin lock still owned mov a2, t0 // set ownership value (lock address) stl_c t0, 0(a2) // set spin lock owned - conditionally beq t0, 20f // if lock_flag eq zero, store failed mb // synchronize the read of the addend // after the spinlock is acquired #endif ldq t0, 0(a0) // get addend addq t0, a1, v0 // do the add stq v0, 0(a0) // store result #if !defined(NT_UP) mb // synchronize the write of the addend // before the spinlock is released stl zero, 0(a2) // set spin lock not owned #endif ENABLE_INTERRUPTS // turn on interrupts ret zero, (ra) // return #if !defined(NT_UP) // // We expect the store conditional will usually succeed the first time so it // is faster to branch forward (predicted not taken) to here and then branch // backward (predicted taken) to where we wanted to go. // 20: ENABLE_INTERRUPTS // turn on interrupts 22: ldl t0, 0(a2) // read current lock value beq t0, 10b // try spinlock again if available br zero, 22b // spin in cache until available #endif .end ExInterlockedAddLargeInteger SBTTL("Interlocked Add Large Statistic") //++ // // VOID // ExInterlockedAddLargeStatistic ( // IN PLARGE_INTEGER Addend, // IN ULONG Increment // ) // // Routine Description: // // This function performs an interlocked add of an increment value to an // addend variable of type large integer. // // Arguments: // // Addend (a0) - Supplies a pointer to a variable whose value is to be // adjusted by the increment value. // // Increment (a1) - Supplies the increment value to be added to the // addend variable. // // Return Value: // // None. // // Implementation Note: // // The specification of this function requires that the given lock must be // used to synchronize the update even though on Alpha the operation can // actually done atomically without using the specified lock. // //-- LEAF_ENTRY(ExInterlockedAddLargeStatistic) zap a1, 0xf0, a1 // zero extend increment value 10: ldq_l t0, 0(a0) // get addend addq t0, a1, t0 // do the add stq_c t0, 0(a0) // store result beq t0, 20f // if eq, store conditional failed ret zero, (ra) // return // // We expect the store conditional will usually succeed the first time so it // is faster to branch forward (predicted not taken) to here and then branch // backward (predicted taken) to where we wanted to go. // 20: br zero, 10b // try again .end ExInterlockedAddLargeStatistic SBTTL("Interlocked Add Unsigned Long") //++ // // ULONG // ExInterlockedAddUlong ( // IN PULONG Addend, // IN ULONG Increment, // IN PKSPIN_LOCK Lock // ) // // Routine Description: // // This function performs an interlocked add of an increment value to an // addend variable of type unsigned long. The initial value of the addend // variable is returned as the function value. // // Arguments: // // Addend (a0) - Supplies a pointer to a variable whose value is to be // adjusted by the increment value. // // Increment (a1) - Supplies the increment value to be added to the // addend variable. // // Lock (a2) - Supplies a pointer to a spin lock to be used to synchronize // access to the addend variable. // // Return Value: // // The initial value of the addend variable. // // Implementation Note: // // The specification of this function requires that the given lock must be // used to synchronize the update even though on Alpha the operation can // actually done atomically without using the specified lock. // //-- LEAF_ENTRY(ExInterlockedAddUlong) 10: DISABLE_INTERRUPTS // (PALcode) v0 is clobbered #if !defined(NT_UP) ldl_l t0, 0(a2) // get current lock value - locked bne t0, 20f // if ne, spin lock still owned mov a2, t0 // set ownership value (lock address) stl_c t0, 0(a2) // set spin lock owned - conditionally beq t0, 20f // if lock_flag eq zero, store failed mb // synchronize the read of the addend // after the spinlock is acquired #endif // // Set the return value in t0 for now since PALcode may use v0. // ldl t0, 0(a0) // get addend value (return value also) addl t0, a1, t1 // compute adjusted value stl t1, 0(a0) // store updated value #if !defined(NT_UP) mb // synchronize the update of the addend // before the spinlock is released stl zero, 0(a2) // set spin lock not owned #endif ENABLE_INTERRUPTS // (PALcode) v0 is clobbered mov t0, v0 // set return value ret zero, (ra) // return #if !defined(NT_UP) // // We expect the store conditional will usually succeed the first time so it // is faster to branch forward (predicted not taken) to here and then branch // backward (predicted taken) to where we wanted to go. // 20: ENABLE_INTERRUPTS // (PALcode) v0 is clobbered 22: ldl t0, 0(a2) // read current lock value beq t0, 10b // try spinlock again if available br zero, 22b // spin in cache until available #endif .end ExInterlockedAddUlong SBTTL("Interlocked Exchange Unsigned Long") //++ // // ULONG // ExInterlockedExchangeUlong ( // IN PULONG Source, // IN ULONG Value, // IN PKSPIN_LOCK Lock // ) // // Routine Description: // // This function performs an interlocked exchange of a longword value with // a longword in memory and returns the memory value. // // N.B. There is an alternate entry point provided for this routine which // is ALPHA target specific and whose prototype does not include the // spinlock parameter. Since the routine never refers to the spinlock // parameter, no additional code is required. // // Arguments: // // Source (a0) - Supplies a pointer to a variable whose value is to be // exchanged. // // Value (a1) - Supplies the value to exchange with the source value. // // Lock (a2) - Supplies a pointer to a spin lock to be used to synchronize // access to the source variable. // // Return Value: // // The source value is returned as the function value. // //-- LEAF_ENTRY(ExInterlockedExchangeUlong) ALTERNATE_ENTRY(ExAlphaInterlockedExchangeUlong) 10: ldl_l v0,0(a0) // get current source value xor a1,zero,t0 // set exchange value stl_c t0,0(a0) // replace source value beq t0,20f // if lock_flag eq zero, store failed ret zero,(ra) // else/ return old value to caller // // We expect the store conditional will usually succeed the first time so it // is faster to branch forward (predicted not taken) to here and then branch // backward (predicted taken) to where we wanted to go. // 20: br zero,10b // go try spin lock again .end ExInterlockedExchangeUlong SBTTL("Interlocked Exchange Add Large Integer") //++ // // LARGE_INTEGER // ExpInterlockedExchangeAddLargeInteger ( // IN PLARGE_INTEGER Addend, // IN LARGE_INTEGER Increment // ) // // Routine Description: // // This function performs an interlocked add of an increment value to an // addend variable of type large integer. The initial value of the addend // variable is returned as the function value. // // Arguments: // // Addend (a0) - Supplies a pointer to a variable whose value is to be // adjusted by the increment value. // // Increment (a1) - Supplies the increment value to be added to the // addend variable. // // Return Value: // // The initial value of the addend variable // //-- LEAF_ENTRY(ExpInterlockedExchangeAddLargeInteger) 10: ldq_l v0, 0(a0) // get addend value addq a1, v0, t1 // add increment stq_c t1, 0(a0) // store addend value beq t1, 15f // if eq, store conditional failed ret zero, (ra) // return old value to caller 15: beq zero, 10b // retry .end ExpInterlockedExchangeAddLargeInteger SBTTL("Interlocked Decrement Long") //++ // // INTERLOCKED_RESULT // ExInterlockedDecrementLong ( // IN PLONG Addend, // IN PKSPIN_LOCK Lock // ) // // Routine Description: // // This function performs an interlocked decrement on an addend variable // of type signed long. The sign and whether the result is zero is returned // as the function value. // // N.B. There is an alternate entry point provided for this routine which // is ALPHA target specific and whose prototype does not include the // spinlock parameter. Since the routine never refers to the spinlock // parameter, no additional code is required. // // Arguments: // // Addend (a0) - Supplies a pointer to a variable whose value is to be // decremented. // // Lock (a1) - Supplies a pointer to a spin lock to be used to synchronize // access to the addend variable. // // Return Value: // // RESULT_NEGATIVE is returned if the resultant addend value is negative. // RESULT_ZERO is returned if the resultant addend value is zero. // RESULT_POSITIVE is returned if the resultant addend value is positive. // // Implementation Note: // // The specification of this function does not require that the given lock // be used to synchronize the update as long as the update is synchronized // somehow. On Alpha a single load locked-store conditional does the job. // //-- LEAF_ENTRY(ExInterlockedDecrementLong) ALTERNATE_ENTRY(ExAlphaInterlockedDecrementLong) 10: ldl_l v0, 0(a0) // get current addend value - locked subl v0, 1, v0 // decrement addend value mov v0, t0 // copy updated value to t0 for store stl_c t0, 0(a0) // store updated value - conditionally beq t0, 20f // if lock_flag eq zero, store failed // // Determine the INTERLOCKED_RESULT value based on the updated addend value. // N.B. RESULT_ZERO = 0, RESULT_NEGATIVE = 1, RESULT_POSITIVE = 2. // sra v0, 63, t0 // replicate the sign bit to every bit addl t0, 2, t0 // if t0 = 0 return 2, if -1 return 1 cmovne v0, t0, v0 // if v0 = 0 return 0 ret zero, (ra) // return // // We expect the store conditional will usually succeed the first time so it // is faster to branch forward (predicted not taken) to here and then branch // backward (predicted taken) to where we wanted to go. // 20: br zero, 10b // go try spin lock again .end ExInterlockedDecrementLong SBTTL("Interlocked Increment Long") //++ // // INTERLOCKED_RESULT // ExInterlockedIncrementLong ( // IN PLONG Addend, // IN PKSPIN_LOCK Lock // ) // // Routine Description: // // This function performs an interlocked increment on an addend variable // of type signed long. The sign and whether the result is zero is returned // as the function value. // // N.B. There is an alternate entry point provided for this routine which // is ALPHA target specific and whose prototype does not include the // spinlock parameter. Since the routine never refers to the spinlock // parameter, no additional code is required. // // Arguments: // // Addend (a0) - Supplies a pointer to a variable whose value is to be // incremented. // // Lock (a1) - Supplies a pointer to a spin lock to be used to synchronize // access to the addend variable. // // Return Value: // // RESULT_NEGATIVE is returned if the resultant addend value is negative. // RESULT_ZERO is returned if the resultant addend value is zero. // RESULT_POSITIVE is returned if the resultant addend value is positive. // // Implementation Note: // // The specification of this function does not require that the given lock // be used to synchronize the update as long as the update is synchronized // somehow. On Alpha a single load locked-store conditional does the job. // //-- LEAF_ENTRY(ExInterlockedIncrementLong) ALTERNATE_ENTRY(ExAlphaInterlockedIncrementLong) 10: ldl_l v0, 0(a0) // get current addend value - locked addl v0, 1, v0 // increment addend value mov v0, t0 // copy updated value to t0 for store stl_c t0, 0(a0) // store updated value - conditionally beq t0, 20f // if lock_flag eq zero, store failed // // Determine the INTERLOCKED_RESULT value based on the updated addend value. // N.B. RESULT_ZERO = 0, RESULT_NEGATIVE = 1, RESULT_POSITIVE = 2. // sra v0, 63, t0 // replicate the sign bit to every bit addl t0, 2, t0 // if t0 = 0 return 2, if -1 return 1 cmovne v0, t0, v0 // if v0 = 0 return 0 ret zero, (ra) // return // // We expect the store conditional will usually succeed the first time so it // is faster to branch forward (predicted not taken) to here and then branch // backward (predicted taken) to where we wanted to go. // 20: br zero, 10b // go try spin lock again .end ExInterlockedIncrementLong SBTTL("Interlocked Insert Head List") //++ // // PLIST_ENTRY // ExInterlockedInsertHeadList ( // IN PLIST_ENTRY ListHead, // IN PLIST_ENTRY ListEntry, // IN PKSPIN_LOCK Lock // ) // // Routine Description: // // This function inserts an entry at the head of a doubly linked list // so that access to the list is synchronized in a multiprocessor system. // // Arguments: // // ListHead (a0) - Supplies a pointer to the head of the doubly linked // list into which an entry is to be inserted. // // ListEntry (a1) - Supplies a pointer to the entry to be inserted at the // head of the list. // // Lock (a2) - Supplies a pointer to a spin lock to be used to synchronize // access to the list. // // Return Value: // // Pointer to entry that was at the head of the list or NULL if the list // was empty. // //-- LEAF_ENTRY(ExInterlockedInsertHeadList) 10: DISABLE_INTERRUPTS // (PALcode) v0 is clobbered #if !defined(NT_UP) ldl_l t0, 0(a2) // get current lock value - locked bne t0, 20f // if ne, spin lock still owned mov a2, t0 // set ownership value (lock address) stl_c t0, 0(a2) // set spin lock owned - conditionally beq t0, 20f // if lock_flag eq zero, store failed mb // synchronize subsequent reads after // the spinlock is acquired #endif // // Set the return value in t0 for now since PALcode may use v0. // ldl t0, LsFlink(a0) // get address of next entry (return value also) stl t0, LsFlink(a1) // store next link in entry stl a0, LsBlink(a1) // store previous link in entry stl a1, LsBlink(t0) // store previous link in next stl a1, LsFlink(a0) // store next link in head #if !defined(NT_UP) mb // sychronize updates to the list // befor the spinlock is released stl zero, 0(a2) // set spin lock not owned #endif ENABLE_INTERRUPTS // (PALcode) v0 is clobbered xor t0, a0, v0 // if t0=a0, list empty, set v0 to NULL cmovne v0, t0, v0 // else return previous entry at head ret zero, (ra) // return #if !defined(NT_UP) // // We expect the store conditional will usually succeed the first time so it // is faster to branch forward (predicted not taken) to here and then branch // backward (predicted taken) to where we wanted to go. // 20: ENABLE_INTERRUPTS // (PALcode) v0 is clobbered 22: ldl t0, 0(a2) // read current lock value beq t0, 10b // try spinlock again if available br zero, 22b // spin in cache until available #endif .end ExInterlockedInsertHeadList SBTTL("Interlocked Insert Tail List") //++ // // PLIST_ENTRY // ExInterlockedInsertTailList ( // IN PLIST_ENTRY ListHead, // IN PLIST_ENTRY ListEntry, // IN PKSPIN_LOCK Lock // ) // // Routine Description: // // This function inserts an entry at the tail of a doubly linked list // so that access to the list is synchronized in a multiprocessor system. // // Arguments: // // ListHead (a0) - Supplies a pointer to the head of the doubly linked // list into which an entry is to be inserted. // // ListEntry (a1) - Supplies a pointer to the entry to be inserted at the // tail of the list. // // Lock (a2) - Supplies a pointer to a spin lock to be used to synchronize // access to the list. // // Return Value: // // Pointer to entry that was at the tail of the list or NULL if the list // was empty. // //-- LEAF_ENTRY(ExInterlockedInsertTailList) 10: DISABLE_INTERRUPTS // (PALcode) v0 is clobbered #if !defined(NT_UP) ldl_l t0, 0(a2) // get current lock value - locked bne t0, 20f // if ne, spin lock still owned mov a2, t0 // set ownership value (lock address) stl_c t0, 0(a2) // set spin lock owned - conditionally beq t0, 20f // if lock_flag eq zero, store failed mb // sychronize subsequent reads after // the spinlock is acquired #endif // // Set the return value in t0 for now since PALcode may use v0. // ldl t0, LsBlink(a0) // get address of previous entry (return value also) stl a0, LsFlink(a1) // store next link in entry stl t0, LsBlink(a1) // store previous link in entry stl a1, LsBlink(a0) // store previous link in next stl a1, LsFlink(t0) // store next link in head #if !defined(NT_UP) mb // sychronize updates to the list // before the spinlock is released stl zero, 0(a2) // set spin lock not owned #endif ENABLE_INTERRUPTS // (PALcode) v0 is clobbered xor t0, a0, v0 // if t0=a0, list empty, set v0 to NULL cmovne v0, t0, v0 // else return previous entry at tail ret zero, (ra) // return #if !defined(NT_UP) // // We expect the store conditional will usually succeed the first time so it // is faster to branch forward (predicted not taken) to here and then branch // backward (predicted taken) to where we wanted to go. // 20: ENABLE_INTERRUPTS // (PALcode) v0 is clobbered 22: ldl t0, 0(a2) // read current lock value beq t0, 10b // try spinlock again if available br zero, 22b // spin in cache until available #endif .end ExInterlockedInsertTailList SBTTL("Interlocked Remove Head List") //++ // // PLIST_ENTRY // ExInterlockedRemoveHeadList ( // IN PLIST_ENTRY ListHead, // IN PKSPIN_LOCK Lock // ) // // Routine Description: // // This function removes an entry from the head of a doubly linked list // so that access to the list is synchronized in a multiprocessor system. // If there are no entries in the list, then a value of NULL is returned. // Otherwise, the address of the entry that is removed is returned as the // function value. // // Arguments: // // ListHead (a0) - Supplies a pointer to the head of the doubly linked // list from which an entry is to be removed. // // Lock (a1) - Supplies a pointer to a spin lock to be used to synchronize // access to the list. // // Return Value: // // The address of the entry removed from the list, or NULL if the list is // empty. // //-- LEAF_ENTRY(ExInterlockedRemoveHeadList) 10: DISABLE_INTERRUPTS // (PALcode) v0 is clobbered #if !defined(NT_UP) ldl_l t0, 0(a1) // get current lock value - locked bne t0, 30f // if ne, spin lock still owned mov a1, t0 // set ownership value (lock address) stl_c t0, 0(a1) // set spin lock owned - conditionally beq t0, 30f // if lock_flag eq zero, store failed mb // synchronize subsequent reads after // the spinlock is acquired #endif // // Set the return value in t0 for now since PALcode may use v0. // ldl t2, LsFlink(a0) // get address of next entry xor t2, a0, t0 // if t2=a0, list empty, set t0 to NULL beq t0, 20f // if eq [NULL], list is empty ldl t1, LsFlink(t2) // get address of next entry stl t1, LsFlink(a0) // store address of next in head stl a0, LsBlink(t1) // store address of previous in next mov t2, t0 // return the address of entry removed 20: // #if !defined(NT_UP) mb // synchronize writes to the list // before the spinlock is released stl zero, 0(a1) // set spin lock not owned #endif ENABLE_INTERRUPTS // (PALcode) v0 is clobbered mov t0, v0 // set return value ret zero, (ra) // return #if !defined(NT_UP) // // We expect the store conditional will usually succeed the first time so it // is faster to branch forward (predicted not taken) to here and then branch // backward (predicted taken) to where we wanted to go. // 30: ENABLE_INTERRUPTS // (PALcode) v0 is clobbered 32: ldl t0, 0(a1) // read current lock value beq t0, 10b // try spinlock again if available br zero, 32b // spin in cache until available #endif .end ExInterlockedRemoveHeadList SBTTL("Interlocked Pop Entry List") //++ // // PSINGLE_LIST_ENTRY // ExInterlockedPopEntryList ( // IN PSINGLE_LIST_ENTRY ListHead, // IN PKSPIN_LOCK Lock // ) // // Routine Description: // // This function removes an entry from the head of a singly linked list // so that access to the list is synchronized in a multiprocessor system. // If there are no entries in the list, then a value of NULL is returned. // Otherwise, the address of the entry that is removed is returned as the // function value. // // Arguments: // // ListHead (a0) - Supplies a pointer to the head of the singly linked // list from which an entry is to be removed. // // Lock (a1) - Supplies a pointer to a spin lock to be used to synchronize // access to the list. // // Return Value: // // The address of the entry removed from the list, or NULL if the list is // empty. // //-- LEAF_ENTRY(ExInterlockedPopEntryList) 10: DISABLE_INTERRUPTS // (PALcode) v0 is clobbered #if !defined(NT_UP) ldl_l t0, 0(a1) // get current lock value - locked bne t0, 30f // if ne, spin lock still owned mov a1, t0 // set ownership value (lock address) stl_c t0, 0(a1) // set spin lock owned - conditionally beq t0, 30f // if lock_flag eq zero, store failed mb // synchronize subsequent reads after // the spinlock is acquired #endif // // Set the return value in t0 for now since PALcode may use v0. // ldl t0, 0(a0) // get address of next entry (return value also) beq t0, 20f // if eq [NULL], list is empty ldl t1, 0(t0) // get address of next entry stl t1, 0(a0) // store address of next in head 20: // #if !defined(NT_UP) mb // synchronize writes to the list // before the spinlock is released stl zero, 0(a1) // set spin lock not owned #endif ENABLE_INTERRUPTS // (PALcode) v0 is clobbered mov t0, v0 // set return value ret zero, (ra) // return #if !defined(NT_UP) // // We expect the store conditional will usually succeed the first time so it // is faster to branch forward (predicted not taken) to here and then branch // backward (predicted taken) to where we wanted to go. // 30: ENABLE_INTERRUPTS // (PALcode) v0 is clobbered 32: ldl t0, 0(a1) // read current lock value beq t0, 10b // try spinlock again if available br zero, 32b // spin in cache until available #endif .end ExInterlockedPopEntryList SBTTL("Interlocked Push Entry List") //++ // // PSINGLE_LIST_ENTRY // ExInterlockedPushEntryList ( // IN PSINGLE_LIST_ENTRY ListHead, // IN PSINGLE_LIST_ENTRY ListEntry, // IN PKSPIN_LOCK Lock // ) // // Routine Description: // // This function inserts an entry at the head of a singly linked list // so that access to the list is synchronized in a multiprocessor system. // // Arguments: // // ListHead (a0) - Supplies a pointer to the head of the singly linked // list into which an entry is to be inserted. // // ListEntry (a1) - Supplies a pointer to the entry to be inserted at the // head of the list. // // Lock (a2) - Supplies a pointer to a spin lock to be used to synchronize // access to the list. // // Return Value: // // Previous contents of ListHead. NULL implies list went from empty // to not empty. // //-- LEAF_ENTRY(ExInterlockedPushEntryList) 10: DISABLE_INTERRUPTS // (PALcode) v0 is clobbered #if !defined(NT_UP) ldl_l t0, 0(a2) // get current lock value - locked bne t0, 20f // if ne, spin lock still owned mov a2, t0 // set ownership value (lock address) stl_c t0, 0(a2) // set spin lock owned - conditionally beq t0, 20f // if lock_flag eq zero, store failed mb // synchronize subsequent reads after // the spinlock is acquired #endif // // Set the return value in t0 for now since PALcode may use v0. // ldl t0, 0(a0) // get address of first entry (return value also) stl t0, 0(a1) // set address of next in new entry stl a1, 0(a0) // set address of first entry #if !defined(NT_UP) mb // synchronize writes to the list // before the spinlock is released stl zero, 0(a2) // set spin lock not owned #endif ENABLE_INTERRUPTS // (PALcode) v0 is clobbered mov t0, v0 // set return value ret zero, (ra) // return #if !defined(NT_UP) // // We expect the store conditional will usually succeed the first time so it // is faster to branch forward (predicted not taken) to here and then branch // backward (predicted taken) to where we wanted to go. // 20: ENABLE_INTERRUPTS // (PALcode) v0 is clobbered 22: ldl t0, 0(a2) // read current lock value beq t0, 10b // try spinlock again if available br zero, 22b // spin in cache until available #endif .end ExInterlockedPushEntryList SBTTL("Interlocked Compare Exchange") //++ // // PVOID // InterlockedCompareExchange ( // IN OUT PVOID *Destination, // IN PVOID Exchange, // IN PVOID Comperand // ) // // Routine Description: // // This function performs an interlocked compare of the destination // value with the comperand value. If the destination value is equal // to the comperand value, then the exchange value is stored in the // destination. Otherwise, no opeation is performed. // // Arguments: // // Destination (a0) - Supplies a pointer to the destination value. // // Exchange (a1) - Supplies the exchange. // // Comperand (a2) - Supplies the comperand value. // // Return Value: // // The initial destination value is returned as the function value. // //-- LEAF_ENTRY(InterlockedCompareExchange) 10: #if !defined(NT_UP) mb #endif ldl_l v0, 0(a0) // get current addend value bis a1, zero, t0 // copy exchange value for store cmpeq v0, a2, t1 // if ne, operands mismatch beq t1, 20f stl_c t0, 0(a0) // store updated addend value beq t0,25f // if eq, store conditional failed #if !defined(NT_UP) mb #endif 20: ret zero, (ra) // return // // We expect the store conditional will usually succeed the first time so it // is faster to branch forward (predicted not taken) to here and then branch // backward (predicted taken) to where we wanted to go. // 25: br zero, 10b // go try spin lock again .end InterlockedCompareExchange SBTTL("Interlocked Exchange Add") //++ // // LONG // ExInterlockedAdd ( // IN PLONG Addend, // IN ULONG Increment // ) // // Routine Description: // // This function performs an interlocked add of an increment value to an // addend variable of type unsigned long. The initial value of the addend // variable is returned as the function value. // // Arguments: // // Addend (a0) - Supplies a pointer to a variable whose value is to be // adjusted by the increment value. // // Increment (a1) - Supplies the increment value to be added to the // addend variable. // // Return Value: // // The initial value of the addend variable. // //-- LEAF_ENTRY(InterlockedExchangeAdd) 10: #if !defined(NT_UP) mb #endif ldl_l v0, 0(a0) // get current addend value - locked addl v0, a1, t0 // increment addend value stl_c t0, 0(a0) // store updated value - conditionally beq t0, 20f // if lock_flag eq zero, store failed #if !defined(NT_UP) mb #endif ret zero, (ra) // return // // We expect the store conditional will usually succeed the first time so it // is faster to branch forward (predicted not taken) to here and then branch // backward (predicted taken) to where we wanted to go. // 20: br zero, 10b // go try spin lock again .end InterlockedExchangeAdd SBTTL("Interlocked Pop Entry Sequenced List") //++ // // PSINGLE_LIST_ENTRY // ExpInterlockedPopEntrySList ( // IN PSLIST_HEADER ListHead // ) // // Routine Description: // // This function removes an entry from the front of a sequenced singly // linked list so that access to the list is synchronized in a MP system. // If there are no entries in the list, then a value of NULL is returned. // Otherwise, the address of the entry that is removed is returned as the // function value. // // Arguments: // // ListHead (a0) - Supplies a pointer to the sequenced listhead from which // an entry is to be removed. // // Return Value: // // The address of the entry removed from the list, or NULL if the list is // empty. // //-- LEAF_ENTRY(ExpInterlockedPopEntrySList) 10: ldq t0, 0(a0) // get next entry address and sequence addl t0, zero, v0 // sign extend next entry address beq v0, 30f // if eq, list is empty srl t0, 32, t1 // shift sequence to low 32-bits #if !defined(NT_UP) mb // ensure we see updates in the same order #endif // // N.B. It is possible for the following instruction to fault in the rare // case where the first entry in the list is allocated on another // processor and free between the time the free pointer is read above // and the following instruction. When this happens, the access fault // code continues execution by skipping the following instruction. // This results in the compare failing and the entire operation is // retried. // ALTERNATE_ENTRY(ExpInterlockedPopEntrySListFault) ldl t5, 0(v0) // get address of successor entry zapnot t5, 0xf ,t2 // clear high 32-bits for merge ldq_l t3, 0(a0) // reload next entry address and sequence ldil t5, 0xffff // decrement list depth and addl t1, t5, t1 // increment sequence number sll t1, 32, t1 // merge successor address and sequence cmpeq t0, t3, t4 // if ne, listhead has changed beq t4, 15f // bis t1, t2, t1 // stq_c t1, 0(a0) // store next entry address and sequence beq t1, 15f // if eq, store conditional failed 30: #if !defined(NT_UP) mb // ensure consistent view of memory #endif ret zero, (ra) 15: br zero, 10b // retry .end ExpInterlockedPopEntrySList SBTTL("Interlocked Push Entry Sequenced List") //++ // // PSINGLE_LIST_ENTRY // ExpInterlockedPushEntrySList ( // IN PSLIST_HEADER ListHead, // IN PSINGLE_LIST_ENTRY ListEntry // ) // // Routine Description: // // This function inserts an entry at the head of a sequenced singly linked // list so that access to the list is synchronized in an MP system. // // Arguments: // // ListHead (a0) - Supplies a pointer to the sequenced listhead into which // an entry is to be inserted. // // ListEntry (a1) - Supplies a pointer to the entry to be inserted at the // head of the list. // // Return Value: // // Previous contents of ListHead. NULL implies list went from empty // to not empty. // //-- LEAF_ENTRY(ExpInterlockedPushEntrySList) 10: ldq t0, 0(a0) // get next entry address and sequence addl t0, zero, v0 // sign extend next entry address srl t0, 32, t1 // shift sequence to low 32-bits stl v0, 0(a1) // set next link in new first entry #if !defined(NT_UP) mb // ensure all writes (including next link) go #endif zapnot a1, 0xf, t2 // zero extend new first entry ldq_l t3, 0(a0) // reload next entry address and sequence ldah t5, 1(zero) // get sequence adjustment value addl t1, 1, t1 // increment list depth addl t1, t5, t1 // increment sequence number sll t1, 32, t1 // merge new first entry address and sequence cmpeq t0, t3, t4 // if ne, listhead has changed beq t4, 15f // bis t1, t2, t2 stq_c t2, 0(a0) // store next entry address and sequence beq t2, 15f // if eq, store conditional failed ret zero, (ra) 15: br zero, 10b // retry .end ExpInterlockedPushEntrySList SBTTL("Interlocked Compare Exchange 64-bits") //++ // // ULONGLONG // ExpInterlockedCompareExchange64 ( // IN PULONGLONG Destination, // IN PULONGLONG Exchange, // IN PULONGLONG Comperand // ) // // Routine Description: // // This function performs an interlocked compare and exchange of 64-bits. // // Arguments: // // Destination (a0) - Supplies a pointer to the destination variable. // // Exchange (a1) - Supplies a pointer to the exchange value. // // Comperand (a2) - Supplies a pointer to the comperand value. // // Return Value: // // The current destination value are returned as the function value. // //-- LEAF_ENTRY(ExpInterlockedCompareExchange64) ldq t0, 0(a1) // get exchange value ldq t1, 0(a2) // get comperand value 10: ldq_l v0, 0(a0) // get current destination value bis t0, zero, t2 // set exchange value cmpeq v0, t1, t3 // check if current and comperand match beq t3, 20f // if eq, current and comperand mismatch stq_c t2, 0(a0) // store exchange value beq t2, 30f // if eq, store conditional failed 20: ret zero, (ra) 30: br zero, 10b // retry .end ExpInterlockedCompareExchange64