path: root/private/ntos/rtl/alpha/mvmem.s

//      TITLE("Compare, Move, Zero, and Fill Memory Support")
//++
//
// Copyright (c) 1992  Digital Equipment Corporation
//
// Module Name:
//
//    mvmem.s
//
// Abstract:
//
//    This module implements functions to compare, move, zero, and fill
//    blocks of memory. If the memory is aligned, then these functions
//    are very efficient.
//
//    N.B. These routines MUST preserve all floating state since they are
//        frequently called from interrupt service routines that normally
//        do not save or restore floating state.
//
// Author:
//
//    Joe Notarangelo  21-May-1992
//
// Environment:
//
//    User or Kernel mode.
//
// Revision History:
//
//    Monty VanderBilt 14-Feb-1996 Avoid memory loads and branch takens between 
//                                 load lock and store conditional instructions
//                                 to conform with all alpha architecture rules.
//    Monty VanderBilt 27-Feb-1996 Added RtlZeroBytes and RtlFillBytes to support
//                                 byte granularity access when necessary.
//--

#include "ksalpha.h"

        SBTTL("Compare Memory")
//++
//
// ULONG
// RtlCompareMemory (
//    IN PVOID Source1,
//    IN PVOID Source2,
//    IN ULONG Length
//    )
//
// Routine Description:
//
//    This function compares two blocks of memory and returns the number
//    of bytes that compared equal.
//
// Arguments:
//
//    Source1 (a0) - Supplies a pointer to the first block of memory to
//       compare.
//
//    Source2 (a1) - Supplies a pointer to the second block of memory to
//       compare.
//
//    Length (a2) - Supplies the length, in bytes, of the memory to be
//       compared.
//
// Return Value:
//
//    The number of bytes that compared equal is returned as the function
//    value. If all bytes compared equal, then the length of the orginal
//    block of memory is returned.
//
//--


        LEAF_ENTRY(RtlCompareMemory)

        bis     a2, zero, v0            // save length of comparison
        beq     a2, 90f                 // (JAE) quit if nothing to compare
        xor     a0, a1, t0              // check for compatible alignment
        and     t0, 0x7, t0             // low bits only
        bne     t0, CompareUnaligned    // if ne, incompatible alignment

//
// Compare memory aligned
//

CompareAligned:                         //

//
// compare memory until sources are aligned
//
        and     a0, 0x7, t0             // get low bits
        bne     t0, 10f                 // if ne, sources not aligned yet
        br      zero, 30f               // already aligned, predicted


10:
        ldq_u   t1, 0(a0)               // get unaligned quad at source 1
        ldq_u   t2, 0(a1)               // get unaligned quad at source 2

20:
        extbl   t1, t0, t4              // byte at t0 in source 1 quad
        extbl   t2, t0, t5              // byte at t0 in source 2 quad
        xor     t4, t5, t3              // t1 = t2 ?
        bne     t3, 110f                // not equal, miscompare
        subq    a2, 1, a2               // decrement bytes to compare
        beq     a2, 90f                 // if eq, compare success
        addq    t0, 1, t0               // increment pointer within quad
        cmpeq   t0, 8, t3               // t0 = 8?, if so first quadword done
        beq     t3, 20b                 // continue while t0 < 8


        addq    a0, 8, a0               // increment to next quadword
        addq    a1, 8, a1               // increment source 2 to next also
        bic     a0, 7, a0               // align source 1 quadword
        bic     a1, 7, a1               // align source 2 quadword


//
// aligned block compare, compare blocks of 64 bytes
//

30:
        srl     a2, 6, t0               // t0 = number of 64 byte blocks
        beq     t0, 50f                 // if eq, no 64 byte blocks

//
// N.B. loads from each of the sources were separated in case these
//      blocks are fighting for the cache
//
        .set    noat
40:
        ldq     t1, 0(a0)               // t1 = source 1, quad 0
        ldq     t2, 8(a0)               // t2 = source 1, quad 1
        ldq     t3, 16(a0)              // t3 = source 1, quad 2
        addq    a1, 64, a1              // increment source 2 pointer
        ldq     t4, 24(a0)              // t4 = source 1, quad 3

        ldq     t5, -64(a1)             // t5 = source 2, quad 0
        ldq     a4, -56(a1)             // a4 = source 2, quad 1
        ldq     a5, -48(a1)             // a5 = source 2, quad 2
        xor     t1, t5, $at             // quad 0 match?
        bne     $at, 200f               // if ne[false], miscompare
        ldq     t5, -40(a1)             // t5 = source 2, quad 3
        ldq     t1, 32(a0)              // t1 = source 1, quad 4
        xor     t2, a4, $at             // quad 1 match?
        bne     $at, 122f               // if ne[false], miscompare
        ldq     t2, 40(a0)              // t2 = source 1, quad 5
        xor     t3, a5, $at             // quad 2 match?
        bne     $at, 124f               // if ne[false], miscompare
        ldq     t3, 48(a0)              // t3 = source 1, quad 6
        xor     t4, t5, $at             // quad 3 match?
        bne     $at, 126f               // if ne[false], miscompare
        ldq     t4, 56(a0)              // t4 = source 1, quad 7

        ldq     t5, -32(a1)             // t5 = source 2, quad 4
        addq    a0, 64, a0              // increment source 1 pointer
        ldq     a4, -24(a1)             // a4 = source 2, quad 5
        subq    t0, 1, t0               // decrement blocks to compare
        ldq     a5, -16(a1)             // a5 = source 2, quad 6
        xor     t1, t5, $at             // quad 4 match?
        bne     $at, 130f               // if ne[false], miscompare
        ldq     t5, -8(a1)              // t5 = source 2, quad 7
        xor     t2, a4, $at             // quad 5 match?
        bne     $at, 132f               // if ne[false], miscompare
        xor     t3, a5, $at             // quad 6 match?
        bne     $at, 134f               // if ne[false], miscompare
        xor     t4, t5, $at             // quad 7 match?
        bne     $at, 136f               // if ne[false], miscompare
        subq    a2, 64, a2              // decrement bytes to compare
        bne     t0, 40b                 // if ne, more blocks to compare
        .set    at


//
// Compare quadwords
//

50:
        srl     a2, 3, t0               // t0 = number of quadwords to compare
        beq     t0, 70f                 // if eq, no quadwords to compare

        .set    noat
60:
        ldq     t1, 0(a0)               // t1 = quad from source 1
        lda     a0, 8(a0)               // increment source 1 pointer
        ldq     t2, 0(a1)               // t2 = quad from source 2
        lda     a1, 8(a1)               // increment source 2 pointer
        xor     t1, t2, $at             // are quadwords equal?
        bne     $at, 200f               // if ne, miscompare
        subq    t0, 1, t0               // decrement quads to compare
        subq    a2, 8, a2               // decrement bytes to compare
        bne     t0, 60b                 // if ne, more quads to compare

        .set    at

//
// Compare bytes in last quadword
//

//      a2 =  number of bytes to compare, less than 8, greater than zero
//      a0, a1, quad-aligned to last quadword

        beq     a2, 80f                 // if eq, all bytes compared

        .set    noat
70:
        ldq     t1, 0(a0)               // t1 = quad at source 1
        ldq     t2, 0(a1)               // t2 = quad at source 2
        bis     zero, 0xff, t0          // zap mask
        sll     t0, a2, t0              //
        zap     t1, t0, t1              // zero bytes not compared
        zap     t2, t0, t2              // same for source 2
        xor     t1, t2, $at             // compare quadwords
        bne     $at, 200f               // if ne, miscompare

        .set    at
//
// Successful compare
//      v0 already contains full length
//

80:
        ret     zero, (ra)              // return


//
// Sources have incompatible alignment
//
CompareUnaligned:


//
// Compare until source 1 (a0) is aligned
//

        and     a0, 0x7, t0             // get byte position of pointer
        beq     t0, 30f                 // if eq, already aligned

        ldq_u   t1, 0(a0)               // get unaligned quad at a0

10:
        ldq_u   t2, 0(a1)               // get unaligned quad at a1
        extbl   t1, t0, t4              // get byte to compare from source 1
        extbl   t2, a1, t2              // get byte to compare from source 2
        xor     t4, t2, t3              // do bytes match?
        bne     t3, 110f                // if ne, miscompare
        subq    a2, 1, a2               // decrement bytes to compare
        beq     a2, 90f                 // (JAE) quit if nothing left to compare
        addq    t0, 1, t0               // increment byte within source 1
        addq    a1, 1, a1               // increment source 2 pointer
        cmpeq   t0, 8, t3               // finished with source 1 quad?
        beq     t3, 10b                 // if eq[false], more to compare

        addq    a0, 7, a0               // point to next source 1 quad
        bic     a0, 7, a0               // align to quadword


//
// Compare 64-byte blocks
//

30:
        srl     a2, 6, t0               // t0 = number of blocks to compare
        beq     t0, 50f                 // if eq, no blocks to move

        ldq_u   t1, 0(a1)               // get source 2 unaligned quad 1

        .set    noat
40:
        ldq_u   t2, 7(a1)               // get source 2 unaligned quad 2
        addq    a0, 64, a0              // increment source 1 pointer
        ldq_u   t3, 15(a1)              // get source 2 unaligned quad 3
        extql   t1, a1, t1              // bytes from unaligned quad 1
        extqh   t2, a1, $at             // bytes from unaligned quad 2
        ldq_u   t4, 23(a1)              // get source 2 unaligned quad 4
        bis     t1, $at, t1             // t1 = quadword 1 (source 2)
        ldq_u   t5, 31(a1)              // get source 2 unaligned quad 5
        extql   t2, a1, t2              // bytes from unaligned quad 2
        extqh   t3, a1, $at             // bytes from unaligned quad 3
        ldq     a3, -64(a0)             // a3 = quadword 1 (source 1)
        bis     t2, $at, t2             // t2 = quadword 2 (source 2)
        ldq     a4, -56(a0)             // a4 = quadword 2 (source 1)
        extql   t3, a1, t3              // bytes from unaligned quad 3
        extqh   t4, a1, $at             // bytes from unaligned quad 4
        ldq     a5, -48(a0)             // a5 = quadword 3 (source 1)
        bis     t3, $at, t3             // t3 = quadword 3 (source 2)
        extql   t4, a1, t4              // bytes from unaligned quad 4
        extqh   t5, a1, $at             // bytes from unaligned quad 5
        subq    t0, 1, t0               // decrement blocks to compare
        bis     t4, $at, t4             // t4 = quadword 4 (source 2)

        xor     t1, a3, $at             // match on quadword 1?
        ldq     a3, -40(a0)             // a3 = quadword 4 (source 1)
        bne     $at, 200f               // if ne, miscompare quad 1
        xor     t2, a4, $at             // match on quadword 2?
        ldq_u   t2, 39(a1)              // get source 2 unaligned quad 6
        bne     $at, 122f               // if ne, miscompare quad 2
        xor     t3, a5, $at             // match on quadword 3?
        ldq_u   t3, 47(a1)              // get source 2 unaligned quad 7
        bne     $at, 124f               // if ne, miscompare quad 3
        xor     t4, a3, $at             // match on quadword 4?
        ldq_u   t4, 55(a1)              // get source 2 unaligned quad 8
        bne     $at, 126f               // if ne, miscompare quad 4
        ldq_u   t1, 63(a1)              // get source 2 unaligned quad 9

        ldq     a3, -32(a0)             // a3 = quadword 5 (source 1)
        extql   t5, a1, t5              // bytes from unaligned quad 5
        extqh   t2, a1, $at             // bytes from unaligned quad 6
        ldq     a4, -24(a0)             // a4 = quadword 6 (source 1)
        ldq     a5, -16(a0)             // a5 = quadword 7 (source 1)
        bis     t5, $at, t5             // t5 = quadword 5 (source 2)

        xor     t5, a3, $at             // match on quadword 5?
        ldq     a3, -8(a0)              // a3 = quadword 8 (source 1)
        bne     $at, 130f               // if ne, miscompare quad 5
        extql   t2, a1, t2              // bytes from unaligned quad 6
        extqh   t3, a1, $at             // bytes from unaligned quad 7
        extql   t3, a1, t3              // bytes from unaligned quad 7
        bis     t2, $at, t2             // t2 = quadword 6 (source 2)
        xor     t2, a4, $at             // match on quadword 6?
        bne     $at, 132f               // if ne, miscompare quad 6
        extqh   t4, a1, $at             // bytes from unaligned quad 8
        extql   t4, a1, t4              // bytes from unaligned quad 8
        bis     t3, $at, t3             // t3 = quadword 7 (source 2)
        xor     t3, a5, $at             // match on quadword 7?
        bne     $at, 134f               // if ne, miscompare quad 7
        extqh   t1, a1, $at             // bytes from unaligned quad 9
        addq    a1, 64, a1              // increment source 2 pointer
        bis     t4, $at, t4             // t4 = quadword 8 (source 2)
        xor     t4, a3, $at             // match on quadword 8?
        bne     $at, 136f               // if ne, miscompare quad 8
        subq    a2, 64, a2              // decrement number of bytes to compare
        bne     t0, 40b                 // if ne, more blocks to compare

        .set    at

//
// Compare quadwords
//


50:
        srl     a2, 3, t0               // t0 = number of quads to compare
        beq     t0, 70f                 // if eq, no quads to compare
        ldq_u   t1, 0(a1)               // get unaligned quad 1 (source 2)

        .set    noat
60:
        ldq_u   t2, 7(a1)               // get unaligned quad 2 (source 2)
        ldq     t3, 0(a0)               // t3 = quadword 1 (source 1)
        extql   t1, a1, t1              // get bytes from unaligned quad 1
        extqh   t2, a1, $at             // get bytes from unaligned quad 2
        addq    a1, 8, a1               // increment source 2 pointer
        bis     t1, $at, t1             // t1 = quadword 1 (source 2)
        xor     t1, t3, $at             // match on quadword?
        bne     $at, 200f               // if ne, miscompare
        subq    t0, 1, t0               // decrement quadwords to compare
        addq    a0, 8, a0               // increment source 1 pointer
        subq    a2, 8, a2               // decrement bytes to compare
        bis     t2, zero, t1            // save low quadword for next loop
        bne     t0, 60b                 // if ne, more quads to compare

        .set    at

//
// Compare bytes for final quadword
//

70:
        beq     a2, 90f                 // if eq, comparison complete

        ldq     t1, 0(a0)               // get quadword from source 1
        bis     zero, zero, t0          // t0 = byte position to compare

        .set    noat
80:
        ldq_u   t2, 0(a1)               // get unaligned quad from source 2
        extbl   t1, t0, t3              // t3 = byte from source 1
        extbl   t2, a1, t2              // t2 = byte from source 2
        xor     t3, t2, $at             // match on byte?
        bne     $at, 100f               // if ne, miscompare on byte
        addq    t0, 1, t0               // increment byte position
        addq    a1, 1, a1               // increment source 2 pointer
        subq    a2, 1, a2               // decrement bytes to compare
        bne     a2, 80b                 // if ne, more bytes to compare

        .set    at
//
// Successful full comparison
//

90:
        ret     zero, (ra)              // return, v0 already set


//
// Miscompare on last quadword
//

100:
        subq    v0, a2, v0              // subtract bytes not compared
        ret     zero, (ra)              // return

//
// Miscompare on first quadword, unaligned case
//
//  v0 = total bytes to compare
//  a2 = bytes remaining to compare
//

110:
        subq    v0, a2, v0              // bytes compared successfully
        ret     zero, (ra)              // return

//
// Miscompare on 64-byte block compare
//

122:
        subq    a2, 8, a2               // miscompare on quad 2
        br      zero, 200f              // finish in common code

124:
        subq    a2, 16, a2              // miscompare on quad 3
        br      zero, 200f              // finish in common code

126:
        subq    a2, 24, a2              // miscompare on quad 4
        br      zero, 200f              // finish in common code

130:
        subq    a2, 32, a2              // miscompare on quad 5
        br      zero, 200f              // finish in common code

132:
        subq    a2, 40, a2              // miscompare on quad 6
        br      zero, 200f              // finish in common code

134:
        subq    a2, 48, a2              // miscompare on quad 7
        br      zero, 200f              // finish in common code

136:
        subq    a2, 56, a2              // miscompare on quad 8
        br      zero, 200f              // finish in common code

//
// Miscompare, determine number of bytes that successfully compared
//      $at = xor of relevant quads from sources, must be non-zero
//      a2 = number of bytes left to compare
//
        .set    noat
200:
        cmpbge  zero, $at, $at          // $at = mask of non-zero bytes

        //
        // look for the first bit cleared in $at, this is the
        //      number of the first byte which differed
        //
        bis     zero, zero, t0          // bit position to look for clear

210:
        blbc    $at, 220f               // if low clear, found difference
        srl     $at, 1, $at             // check next bit
        addq    t0, 1, t0               // count bit position checked
        br      zero, 210b

220:
        subq    v0, a2, v0              // subtract bytes yet to compare
        addq    v0, t0, v0              // add bytes that matched on last quad

        ret     zero, (ra)

        .set    at

        .end    RtlCompareMemory



        SBTTL("Move Memory")
//++
//
// VOID
// RtlMoveMemory (
//    IN PVOID Destination,
//    IN PVOID Source,
//    IN ULONG Length
//    )
//
// Routine Description:
//
//    This function moves memory either forward or backward, aligned or
//    unaligned, in 64-byte blocks, followed by 8-byte blocks, followed
//    by any remaining bytes.
//
// Arguments:
//
//    Destination (a0) - Supplies a pointer to the destination address of
//       the move operation.
//
//    Source (a1) - Supplies a pointer to the source address of the move
//       operation.
//
//    Length (a2) - Supplies the length, in bytes, of the memory to be moved.
//
// Return Value:
//
//    None.
//
//--

        LEAF_ENTRY(RtlMoveMemory)

        beq     a2, 80f                 // if eq, no bytes to move
//
// If the source address is less than the destination address and source
// address plus the length of the move is greater than the destination
// address, then the source and destination overlap such that the move
// must be performed backwards.
//

        cmpult  a0, a1, t0              // is destination less than source
        bne     t0, MoveForward         // if eq [true] no overlap possible
        addq    a1, a2, t0              // compute source ending address
        cmpult  t0, a0, t1              // is source end less than dest.
        beq     t1, MoveBackward        // if eq [false], overlap

//
// Move memory forward aligned and unaligned.
//

MoveForward:                            //
        xor     a0, a1, t0              // compare alignment bits
        and     t0, 0x7, t0             // isloate alignment comparison
        bne     t0, MoveForwardUnaligned  // if ne, incompatible alignment

//
// Move memory forward aligned.
//

MoveForwardAligned:                     //

//
// Move bytes until source and destination are quadword aligned
//

        and     a0, 0x7, t0             // t0 = unaligned bits
        bne     t0, 5f                  // if ne, not quad aligned
        br      zero, 20f               // predicted taken

5:
        ldq_u   t2, 0(a0)               // get unaligned quad from dest.
        ldq_u   t1, 0(a1)               // get unaligned quadword from source
10:
        beq     a2, 15f                 // if eq, all bytes moved
        extbl   t1, t0, t3              // t3 = byte from source
        insbl   t3, t0, t3              // t3 = byte from source, in position
        mskbl   t2, t0, t2              // clear position in dest. quad
        bis     t2, t3, t2              // merge in byte from source
        subq    a2, 1, a2               // decrement bytes to move
        addq    t0, 1, t0               // increment byte within quad
        cmpeq   t0, 8, t3               // finished the quadword?
        beq     t3, 10b                 // if eq [false], do next byte
15:
        stq_u   t2, 0(a0)               // store merged destination bytes

        addq    a0, 7, a0               // move to next quadword
        bic     a0, 7, a0               // aligned quadword

        addq    a1, 7, a1               // move to next quadword
        bic     a1, 7, a1               // aligned quadword

//
// Check for 64-byte block moves
//

20:
        srl     a2, 6, t0               // t0 = number of 64 byte blocks
        beq     t0, 40f                 // if eq no blocks to move
        and     a2, 64-1, a2            // a2 = residual bytes

30:
        ldq     t1, 0(a1)               // load 64 bytes from source
        addq    a0, 64, a0              // increment destination pointer
        ldq     v0, 56(a1)              //
        ldq     a3, 32(a1)              //
        stq     t1, -64(a0)             // write to destination
        ldq     t2, 8(a1)               //   into volatile registers
        ldq     t3, 16(a1)              //
        ldq     t4, 24(a1)              //
        subq    t0, 1, t0               // decrement number of blocks
        stq     t2, -56(a0)             //
        ldq     a4, 40(a1)              //
        stq     t3, -48(a0)             //
        ldq     a5, 48(a1)              //
        stq     t4, -40(a0)             //
        addq    a1, 64, a1              // increment source pointer
        stq     a3, -32(a0)             //
        stq     a4, -24(a0)             //
        stq     a5, -16(a0)             //
        stq     v0, -8(a0)              //
        bne     t0, 30b                 // if ne, more blocks to copy

//
// Copy quadwords
//

40:
        srl     a2, 3, t0               // t0 = number of quadwords to move
        beq     t0, 60f                 // if eq no quadwords to move
        and     a2, 8-1, a2             // a2 = residual bytes

50:
        ldq     t1, 0(a1)               // load quadword from source
        addq    a1, 8, a1               // increment source pointer
        stq     t1, 0(a0)               // store quadword to destination
        addq    a0, 8, a0               // increment destination pointer
        subq    t0, 1, t0               // decrement number of quadwords
        bne     t0, 50b                 // if ne, more quadwords to move

//
// Move final residual bytes
//

60:
        beq     a2, 80f                 // if eq, no more bytes to move
        ldq     t1, 0(a1)               // get last source quadword
        ldq     t2, 0(a0)               // get last dest. quadword
        bis     zero, zero, t0          // t0 = next byte number to move

70:
        extbl   t1, t0, t3              // extract byte from source
        insbl   t3, t0, t3              // t3 = source byte, in position
        mskbl   t2, t0, t2              // clear byte position for dest.
        bis     t2, t3, t2              // merge in source byte
        addq    t0, 1, t0               // increment byte position
        subq    a2, 1, a2               // decrement bytes to move
        bne     a2, 70b                 // if ne => more bytes to move

        stq     t2, 0(a0)               // store merged data

//
// Finish aligned MoveForward
//

80:
        ret     zero, (ra)              // return



//
// Move memory forward unaligned.
//

MoveForwardUnaligned:                   //


//
// Move bytes until the destination is aligned
//

        and     a0, 0x7, t0             // t0 = unaligned bits
        beq     t0, 100f                // if eq, destination quad aligned

        ldq_u   t2, 0(a0)               // get unaligned quad from dest

90:
        beq     a2, 95f                 // if eq no more bytes to move
        ldq_u   t1, 0(a1)               // get unaligned quad from source
        extbl   t1, a1, t1              // extract source byte
        insbl   t1, t0, t1              // t1 = source byte, in position
        mskbl   t2, t0, t2              // clear byte position in dest.
        bis     t2, t1, t2              // merge in source byte
        addq    t0, 1, t0               // increment byte position
        addq    a1, 1, a1               // increment source pointer
        subq    a2, 1, a2               // decrement bytes to move
        cmpeq   t0, 8, t3               // t0 = 8? => quad finished
        beq     t3, 90b                 // if eq [false], more bytes to move
95:
        stq_u   t2, 0(a0)               // store merged quadword
        addq    a0, 7, a0               // increment to next quad
        bic     a0, 7, a0               // align next quadword

//
// Check for 64-byte blocks to move
//

100:
        srl     a2, 6, t0               // t0 = number of blocks to move
        beq     t0, 120f                // if eq no blocks to move
        and     a2, 64-1, a2            // a2 = residual bytes to move


        ldq_u   t1, 0(a1)               // t1 = first unaligned quad

110:
                                        // get source data and merge it
                                        //  as we go
        ldq_u   t2, 7(a1)               // t2 = second unaligned quad
        extql   t1, a1, t1              // extract applicable bytes from t1
        extqh   t2, a1, v0              // extract applicable bytes from t2
        bis     t1, v0, t1              // t1 = quad #1
        ldq_u   t3, 15(a1)              // t3 = third unaligned quad
        extql   t2, a1, t2              // extract applicable bytes from t2
        extqh   t3, a1, v0              // extract applicable bytes from t3
        stq     t1, 0(a0)               // store quad #1
        bis     t2, v0, t2              // t2 = quad #2
        ldq_u   t4, 23(a1)              // t4 = fourth unaligned quad
        extql   t3, a1, t3              // extract applicable bytes from t3
        extqh   t4, a1, v0              // extract applicable bytes from t4
        stq     t2, 8(a0)               // store quad #2
        bis     t3, v0, t3              // t3 = quad #3
        ldq_u   t5, 31(a1)              // t5 = fifth unaligned quad
        extql   t4, a1, t4              // extract applicable bytes from t4
        extqh   t5, a1, v0              // extract applicable bytes from t5
        stq     t3, 16(a0)              // store quad #3
        bis     t4, v0, t4              // t4 = quad #4
        ldq_u   a3, 39(a1)              // a3 = sixth unaligned quad
        extql   t5, a1, t5              // extract applicable bytes from t5
        extqh   a3, a1, v0              // extract applicable bytes from a3
        stq     t4, 24(a0)              // store quad #4
        bis     t5, v0, t5              // t5 = quad #5
        ldq_u   a4, 47(a1)              // a4 = seventh unaligned quad
        extql   a3, a1, a3              // extract applicable bytes from a3
        extqh   a4, a1, v0              // extract applicable bytes from a4
        stq     t5, 32(a0)              // store quad #5
        bis     a3, v0, a3              // a3 = quad #6
        ldq_u   a5, 55(a1)              // a5 = eighth unaligned quad
        extql   a4, a1, a4              // extract applicable bytes from a4
        extqh   a5, a1, v0              // extract applicable bytes from a5
        stq     a3, 40(a0)              // store quad #6
        bis     a4, v0, a4              // a4 = quad #7
        ldq_u   t1, 63(a1)              // t1 = ninth unaligned = 1st of next
        extql   a5, a1, a5              // extract applicable bytes from a5
        extqh   t1, a1, v0              // extract applicable bytes from t1
        stq     a4, 48(a0)              // store quad #7
        bis     a5, v0, a5              // a5 = quad #8
        addq    a1, 64, a1              // increment source pointer
        stq     a5, 56(a0)              // store quad #8
        addq    a0, 64, a0              // increment destination pointer
        subq    t0, 1, t0               // decrement number of blocks
        bne     t0, 110b                // if ne, more blocks to move

//
// Move unaligned source quads to aligned destination quads
//

120:
        srl     a2, 3, t0               // t0 = number of quads to move
        beq     t0, 140f                // if eq no quads to move
        and     a2, 8-1, a2             // a2 = residual bytes


        ldq_u   t1, 0(a1)               // t1 = first unaligned quad
130:
        ldq_u   t2, 7(a1)               // t2 = second unaligned quad
        addq    a0, 8, a0               // increment destination pointer
        extql   t1, a1, t1              // extract applicable bytes from t1
        extqh   t2, a1, v0              // extract applicable bytes from t2
        bis     t1, v0, t1              // t1 = quadword of data
        stq     t1, -8(a0)              // store data to destination
        addq    a1, 8, a1               // increment source pointer
        subq    t0, 1, t0               // decrement quads to move
        bis     t2, zero, t1            // t1 = first of next unaligned pair
        bne     t0, 130b                // if ne, more quads to move

//
// Move remaining bytes to final quadword
//


140:
        beq     a2, 160f                // if eq no more bytes to move
        ldq     t2, 0(a0)               // t2 = destination quadword
        bis     zero, zero, t3          // t3 = position for next insertion

150:
        ldq_u   t1, 0(a1)               // get unaligned source quad
        extbl   t1, a1, t1              // t1 = source byte
        insbl   t1, t3, t1              // t1 = source byte, in position
        mskbl   t2, t3, t2              // clear byte in destination
        bis     t2, t1, t2              // merge in source byte
        addq    a1, 1, a1               // increment source pointer
        subq    a2, 1, a2               // decrement bytes to move
        addq    t3, 1, t3               // increment destination position
        bne     a2, 150b                // more bytes to move

        stq     t2, 0(a0)               // store merged data

//
// Finish unaligned MoveForward
//

160:
        ret     zero, (ra)              // return


//
// Move memory backward.
//

MoveBackward:                           //

        addq    a0, a2, a0              // compute ending destination address
        addq    a1, a2, a1              // compute ending source address
        subq    a0, 1, a0               // point to last destination byte
        subq    a1, 1, a1               // point to last source byte
        xor     a0, a1, t0              // compare alignment bits
        and     t0, 0x7, t0             // isolate alignment comparison
        bne     t0, MoveBackwardUnaligned   // if ne, incompatible alignment

//
// Move memory backward aligned.
//

MoveBackwardAligned:                    //

//
// Move bytes until source and destination are quadword aligned
//

        and     a0, 0x7, t0             // t0 = unaligned bits
        cmpeq   t0, 7, t1               // last byte position 7?
        beq     t1, 5f                  // if eq [false], not quad aligned
        subq    a0, 7, a0               // point to beginning of last quad
        subq    a1, 7, a1               // point to beginning of last quad
        br      zero, 30f               // predicted taken

5:
        ldq_u   t1, 0(a0)               // get unaligned quad from dest.
        ldq_u   t2, 0(a1)               // get unaligned quad from source

10:
        beq     a2, 20f                 // if eq, all bytes moved
        extbl   t2, t0, t3              // t3 = byte from source
        insbl   t3, t0, t3              // t3 = byte from source, in position
        mskbl   t1, t0, t1              // clear position in destination
        bis     t1, t3, t1              // merge in byte from source
        subq    a2, 1, a2               // decrement bytes to move
        subq    t0, 1, t0               // decrement byte within quadword
        cmplt   t0, zero, t3            // finished the quadword?
        beq     t3, 10b                 // if eq [false], do next byte

20:
        stq_u   t1, 0(a0)               // store merged destination bytes

        subq    a0, 8, a0               // move to previous quadword
        bic     a0, 7, a0               // aligned quadword

        subq    a1, 8, a1               // move to previous quadword
        bic     a1, 7, a1               // aligned quadword

//
// Check for 64-byte block moves
//

30:

        srl     a2, 6, t0               // t0 = number of 64 byte blocks
        beq     t0, 50f                 // if eq, no blocks to move
        and     a2, 64-1, a2            // a2 = residual bytes

40:
        ldq     t1, 0(a1)               // load 64 bytes from source into
        subq    a0, 64, a0              // decrement destination pointer
        ldq     v0, -56(a1)             //
        ldq     a3, -32(a1)             //
        stq     t1, 64(a0)              // write to destination
        ldq     t2, -8(a1)              //   into volatile registers
        ldq     a5, -48(a1)             //
        ldq     a4, -40(a1)             //
        stq     t2, 56(a0)              //
        ldq     t3, -16(a1)             //
        ldq     t4, -24(a1)             //
        subq    a1, 64, a1              // decrement source pointer
        stq     t3, 48(a0)              //
        stq     t4, 40(a0)              //
        stq     a3, 32(a0)              //
        subq    t0, 1, t0               // decrement number of blocks
        stq     a4, 24(a0)              //
        stq     a5, 16(a0)              //
        stq     v0, 8(a0)               //
        bne     t0, 40b                 // if ne, more blocks to copy

//
// Copy quadwords
//

50:
        srl     a2, 3, t0               // t0 = number of quadwords to move
        beq     t0, 70f                 // if eq no quadwords to move
        and     a2, 8-1, a2             // a2 = residual bytes

60:
        ldq     t1, 0(a1)               // load quadword from source
        subq    a1, 8, a1               // decrement source pointer
        stq     t1, 0(a0)               // store quadword to destination
        subq    a0, 8, a0               // decrement destination pointer
        subq    t0, 1, t0               // decrement quadwords to move
        bne     t0, 60b                 // if ne, more quadwords to move

//
// Move final residual bytes
//

70:
        beq     a2, 90f                 // if eq, no more bytes to move
        ldq     t1, 0(a1)               // get last source quadword
        ldq     t2, 0(a0)               // get last destination quadword
        bis     zero, 7, t0             // t0 = next byte number to move

80:
        extbl   t1, t0, t3              // extract byte from source
        insbl   t3, t0, t3              // t3 = source byte, in position
        mskbl   t2, t0, t2              // clear byte position for dest.
        bis     t2, t3, t2              // merge in source byte
        subq    t0, 1, t0               // decrement byte position
        subq    a2, 1, a2               // decrement bytes to move
        bne     a2, 80b                 // if ne, more bytes to move

        stq     t2, 0(a0)               // write destination data
//
// Finish aligned MoveBackward
//

90:

        ret     zero, (ra)              // return


//
// Move memory backward unaligned.
//

MoveBackwardUnaligned:                  //


//
// Move bytes until the destination is aligned
//

        and     a0, 0x7, t0             // t0 = unaligned bits
        cmpeq   t0, 7, t1               // last byte of a quadword
        beq     t1, 95f                 // if eq[false], not aligned
        subq    a0, 7, a0               // align pointer to beginning of quad
        br      zero, 120f              //

95:
        ldq_u   t2, 0(a0)               // get unaligned quad from dest.

100:
        beq     a2, 110f                // if eq, no more bytes to move
        ldq_u   t1, 0(a1)               // get unaligned quad from source
        extbl   t1, a1, t1              // extract source byte
        insbl   t1, t0, t1              // t1 = source byte in position
        mskbl   t2, t0, t2              // clear byte position in dest.
        bis     t2, t1, t2              // merge source byte
        subq    t0, 1, t0               // decrement byte position
        subq    a1, 1, a1               // decrement source pointer
        subq    a2, 1, a2               // decrement number of bytes to move
        cmplt   t0, zero, t3            // t0 < 0? => quad finished
        beq     t3, 100b                // if eq [false], more bytes to move

110:
        stq_u   t2, 0(a0)               // store merged quadword

        subq    a0, 8, a0               // decrement dest. to previous quad
        bic     a0, 7, a0               // align previous quadword

//
// Check for 64-byte blocks to move
//

120:

        srl     a2, 6, t0               // t0 = number of blocks to move
        subq    a1, 7, a1               // point to beginning of last quad
        beq     t0, 140f                // if eq no blocks to move
        and     a2, 64-1, a2            // a2 = residual bytes to move

        ldq_u   t1, 7(a1)               // t1 = first unaligned quad

130:
                                        // get source data and merge it
                                        //  as we go
        ldq_u   t2, 0(a1)               // t2 = second unaligned quad
        extqh   t1, a1, t1              // extract applicable bytes from t1
        extql   t2, a1, v0              // extract applicable bytes from t2
        bis     t1, v0, t1              // t1 = quad #1
        ldq_u   t3, -8(a1)              // t3 = third unaligned quad
        extqh   t2, a1, t2              // extract applicable bytes from t2
        extql   t3, a1, v0              // extract applicable bytes from t3
        stq     t1, 0(a0)               // store quad #1
        bis     t2, v0, t2              // t2 = quad #2
        ldq_u   t4, -16(a1)             // t4 = fourth unaligned quad
        extqh   t3, a1, t3              // extract applicable bytes from t3
        extql   t4, a1, v0              // extract applicable bytes from t4
        stq     t2, -8(a0)              // store quad #2
        bis     t3, v0, t3              // t3 = quad #3
        ldq_u   t5, -24(a1)             // t5 = fifth unaligned quad
        extqh   t4, a1, t4              // extract applicable bytes from t4
        extql   t5, a1, v0              // extract applicable bytes from t5
        stq     t3, -16(a0)             // store quad #3
        bis     t4, v0, t4              // t4 = quad #4
        ldq_u   a3, -32(a1)             // a3 = sixth unaligned quad
        extqh   t5, a1, t5              // extract applicable bytes from t5
        extql   a3, a1, v0              // extract applicable bytes from a3
        stq     t4, -24(a0)             // store quad #4
        bis     t5, v0, t5              // t5 = quad #5
        ldq_u   a4, -40(a1)             // a4 = seventh unaligned quad
        extqh   a3, a1, a3              // extract applicable bytes from a3
        extql   a4, a1, v0              // extract applicable bytes from a4
        stq     t5, -32(a0)             // store quad #5
        bis     a3, v0, a3              // a3 = quad #6
        ldq_u   a5, -48(a1)             // a5 = eighth unaligned quad
        extqh   a4, a1, a4              // extract applicable bytes from a4
        extql   a5, a1, v0              // extract applicable bytes from a5
        stq     a3, -40(a0)             // store quad #6
        bis     a4, v0, a4              // a4 = quad #7
        ldq_u   t1, -56(a1)             // t1 = ninth unaligned = 1st of next
        extqh   a5, a1, a5              // extract applicable bytes from a5
        extql   t1, a1, v0              // extract applicable bytes from t1
        stq     a4, -48(a0)             // store quad #7
        bis     a5, v0, a5              // a5 = quad #8
        subq    a1, 64, a1              // increment source pointer
        stq     a5, -56(a0)             // store quad #8
        subq    a0, 64, a0              // increment destination pointer
        subq    t0, 1, t0               // decrement number of blocks
        bne     t0, 130b                // if ne, more blocks to move


//
// Move unaligned source quads to aligned destination quads
//

140:
        srl     a2, 3, t0               // t0 = number of quads to move
        beq     t0, 160f                // if eq no quads to move
        and     a2, 8-1, a2             // a2 = residual bytes

        ldq_u   t1, 7(a1)               // t1 = first unaligned quad

150:
        ldq_u   t2, 0(a1)               // t2 = second unaligned quad
        subq    a0, 8, a0               // decrement destination pointer
        extqh   t1, a1, t1              // extract applicable bytes from t1
        extql   t2, a1, v0              // extract applicable bytes from t2
        bis     t1, v0, t1              // t1 = quadword of data
        stq     t1, 8(a0)               // store data to destination
        subq    a1, 8, a1               // decrement source pointer
        subq    t0, 1, t0               // decrement quads to move
        bis     t2, zero, t1            // t1 = first of next unaligned pair
        bne     t0, 150b                // if ne, more quads to move

//
// Move remaining bytes to final quadword
//

160:
        beq     a2, 180f                // if eq, no more bytes to move
        ldq     t2, 0(a0)               // t2 = destination quadword
        bis     zero, 7, t0             // t0 = position for next insertion

170:
        subq    a1, 1, a1               // decrement source pointer
        ldq_u   t1, 8(a1)               // get unaligned source quad
        extbl   t1, a1, t1              // t1 = source byte
        insbl   t1, t0, t1              // t1 = source byte, in position
        mskbl   t2, t0, t2              // clear byte position
        bis     t2, t1, t2              // merge in source byte
        subq    t0, 1, t0               // decrement byte position for dest.
        subq    a2, 1, a2               // decrement bytes to move
        bne     a2, 170b                // if ne, more bytes to move

        stq     t2, 0(a0)               //

//
// Finish unaligned MoveBackward
//

180:
        ret     zero, (ra)              // return

        .end    RtlMoveMemory

        SBTTL("Zero Memory")
//++
//
// VOID
// RtlZeroMemory (
//    IN PVOID Destination,
//    IN ULONG Length
//    )
//
// Routine Description:
//
//    This function zeros memory by first aligning the destination address to
//    a quadword boundary, and then zeroing 64-byte blocks, followed by 8-byte
//    blocks, followed by any remaining bytes.
//
// Arguments:
//
//    Destination (a0) - Supplies a pointer to the memory to zero.
//
//    Length (a1) - Supplies the length, in bytes, of the memory to be zeroed.
//
// Return Value:
//
//    None.
//
//--

        LEAF_ENTRY(RtlZeroMemory)

        bis     zero, zero, a2          // set fill pattern
        br      zero, RtlpFillMemory    //


        SBTTL("Fill Memory")
//++
//
// VOID
// RtlFillMemory (
//    IN PVOID Destination,
//    IN ULONG Length,
//    IN UCHAR Fill
//    )
//
// Routine Description:
//
//    This function fills memory by first aligning the destination address to
//    a longword boundary, and then filling 32-byte blocks, followed by 4-byte
//    blocks, followed by any remaining bytes.
//
// Arguments:
//
//    Destination (a0) - Supplies a pointer to the memory to fill.
//
//    Length (a1) - Supplies the length, in bytes, of the memory to be filled.
//
//    Fill (a2) - Supplies the fill byte.
//
//    N.B. The alternate entry memset expects the length and fill arguments
//         to be reversed.  It also returns the Destination pointer
//
// Return Value:
//
//    None.
//
//--

        ALTERNATE_ENTRY(memset)

        bis     a0, zero, v0            // set return value
        bis     a1, zero, a3            // swap length and fill arguments
        bis     a2, zero, a1            //
        bis     a3, zero, a2            //

        ALTERNATE_ENTRY(RtlFillMemory)

        and     a2, 0xff, a2            // clear excess bits
        sll     a2, 8, t0               // duplicate fill byte
        bis     a2, t0, a2              // generate fill word
        sll     a2, 16, t0              // duplicate fill word
        bis     a2, t0, a2              // generate fill longword
        sll     a2, 32, t0              // duplicate fill longword
        bis     a2, t0, a2              // generate fill quadword

.align 3                                // ensure quadword aligned target
//
// Fill memory with the pattern specified in register a2.
//

RtlpFillMemory:                         //

//
// Align destination to quadword
//

        beq     a1, 80f                 // anything to fill? (paranoia)
        and     a0, 8-1, t0             // t0 = unaligned bits
        bne     t0, 5f                  // if ne, then not quad aligned
        br      zero, 20f               // if eq, then quad aligned

5:
        ldq_u   t1, 0(a0)               // get unaligned quadword
                                        //   for first group of bytes
10:
        beq     a1, 15f                 // if eq no more bytes to fill
        insbl   a2, t0, t2              // get fill byte into position
        mskbl   t1, t0, t1              // clear byte for fill
        bis     t1, t2, t1              // put in fill byte
        addq    t0, 1, t0               // increment to next byte position
        subq    a1, 1, a1               // decrement bytes to fill
        cmpeq   t0, 8, t2               // t0 = 8?
        beq     t2, 10b                 // if eq [false] more bytes to do

15:
        stq_u   t1, 0(a0)               // store modified bytes
        addq    a0, 7, a0               // move a0 to next quadword
        bic     a0, 7, a0               // align a0 to quadword

//
// Check for 64-byte blocks
//

20:
        srl     a1, 6, t0               // t0 = number of 64 byte blocks
        beq     t0, 40f                 // if eq then no 64 byte blocks
        and     a1, 64-1, a1            // a1 = residual bytes to fill

30:
        stq     a2, 0(a0)               // store 64 bytes
        stq     a2, 8(a0)               //
        stq     a2, 16(a0)              //
        stq     a2, 24(a0)              //
        stq     a2, 32(a0)              //
        stq     a2, 40(a0)              //
        stq     a2, 48(a0)              //
        stq     a2, 56(a0)              //

        subq    t0, 1, t0               // decrement blocks remaining
        addq    a0, 64, a0              // increment destination pointer
        bne     t0, 30b                 // more blocks to write



//
// Fill aligned quadwords
//

40:
        srl     a1, 3, t0               // t0 = number of quadwords
        bne     t0, 55f                 // if ne quadwords left to fill
        br      zero, 60f               // if eq no quadwords left

55:
        and     a1, 8-1, a1             // a1 = residual bytes to fill

50:
        stq     a2, 0(a0)               // store quadword
        subq    t0, 1, t0               // decrement quadwords remaining
        addq    a0, 8, a0               // next quadword
        bne     t0, 50b                 // more quadwords to write


//
// Fill bytes for last quadword
//

60:
        bne     a1, 65f                 // if ne bytes remain to be filled
        br      zero, 80f               // if eq no more bytes to fill

65:
        ldq     t1, 0(a0)               // get last quadword
        bis     zero, zero, t0          // t0 = byte position to start fill

70:
        beq     a1, 75f                 // if eq, no more bytes to fill
        insbl   a2, t0, t2              // get fill byte into position
        mskbl   t1, t0, t1              // clear fill byte position
        bis     t1, t2, t1              // insert fill byte
        addq    t0, 1, t0               // increment byte within quad
        subq    a1, 1, a1               // decrement bytes to fill
        cmpeq   t0, 8, t3               // t0 = 8? => finished quad
        beq     t3, 70b                 // if eq [false] more bytes to fill

75:
        stq     t1, 0(a0)               // write merged quadword

//
// Finish up
//

80:
        ret     zero, (ra)              // return


        .end    RtlZeroMemory

        SBTTL("Fill Memory Ulong")
//++
//
// VOID
// RtlFillMemoryUlong (
//    IN PVOID Destination,
//    IN ULONG Length,
//    IN ULONG Pattern
//    )
//
// Routine Description:
//
//    This function fills memory with the specified longowrd pattern by
//    filling 64-byte blocks followed by 8-byte blocks and finally
//    4-byte blocks.
//
//    N.B. This routine assumes that the destination address is aligned
//         on a longword boundary and that the length is an even multiple
//         of longwords.
//
// Arguments:
//
//    Destination (a0) - Supplies a pointer to the memory to fill.
//
//    Length (a1) - Supplies the length, in bytes, of the memory to be filled.
//
//    Pattern (a2) - Supplies the fill pattern.
//
// Return Value:
//
//    None.
//
//--

        LEAF_ENTRY(RtlFillMemoryUlong)

        bic     a1, 3, a1               // make sure length is an even number
                                        // of longwords
        sll     a2, 32, a3              // a3 = long pattern in upper 32 bits
        srl     a3, 32, t0              // clear upper bits, pattern in lower 32
        bis     a3, t0, a3              // a3 = quad version of fill pattern

//
// Make destination address quad-aligned
//

        and     a0, 4, t0               // is a0 quad aligned?
        beq     t0, 10f                 // if eq, then a0 quad aligned
        stl     a2, 0(a0)               // fill first longword
        addq    a0, 4, a0               // quad align a0
        subq    a1, 4, a1               // bytes remaining to store

//
// Check for 64-byte blocks to fill
//

10:
        srl     a1, 6, t0               // t0 = # 64-byte blocks to fill
        beq     t0, 30f                 // if eq no 64 byte blocks
        and     a1, 64-1, a1            // a1 = residual bytes

20:
        stq     a3, 0(a0)               // store 64 bytes
        stq     a3, 8(a0)               //
        stq     a3, 16(a0)              //
        stq     a3, 24(a0)              //
        stq     a3, 32(a0)              //
        stq     a3, 40(a0)              //
        stq     a3, 48(a0)              //
        stq     a3, 56(a0)              //
        subq    t0, 1, t0               // t0 = blocks remaining
        addq    a0, 64, a0              // increment address pointer
        bne     t0, 20b                 // if ne more blocks to fill

//
// Fill 8 bytes at a time while we can, a1 = bytes remaining
//

30:
        srl     a1, 3, t0               // t0 = # quadwords to fill
        beq     t0, 50f                 // if eq no quadwords left
        and     a1, 8-1, a1             // a1 = residual bytes
40:
        stq     a3, 0(a0)               // store quadword
        subq    t0, 1, t0               // t0 = quadwords remaining
        addq    a0, 8, a0               // increment address pointer
        bne     t0, 40b                 // if ne more quadwords to fill

//
// Fill last 4 bytes
//

50:
        beq     a1, 60f                 // if eq no longwords remain
        stl     a2, 0(a0)               // fill last longword

//
// Finish up
//

60:
        ret     zero, (ra)              // return to caller


        .end    RtlFillMemoryUlong

        SBTTL("Copy Memory With Byte Granularity")
//++
//
// VOID
// RtlCopyBytes (
//    IN PVOID Destination,
//    IN PVOID Source,
//    IN ULONG Length
//    )
//
// Routine Description:
//
//    This function copies non-overlapping memory, aligned or unaligned, in
//    64-byte blocks, followed by 8-byte blocks, followed by any remaining
//    bytes.  Unlike RtlCopyMemory or RtlMoveMemory the copy is done such
//    that byte granularity is assured for all platforms.
//
// Arguments:
//
//    Destination (a0) - Supplies a pointer to the destination address of
//       the move operation.
//
//    Source (a1) - Supplies a pointer to the source address of the move
//       operation.
//
//    Length (a2) - Supplies the length, in bytes, of the memory to be moved.
//
// Return Value:
//
//    None.
//
//--

        LEAF_ENTRY(RtlCopyBytes)

//
// Move memory forward aligned and unaligned.
//

        xor     a0, a1, t0              // compare alignment bits
        and     t0, 0x7, t0             // isolate alignment comparison
        bne     t0, CopyForwardUnaligned // if ne, incompatible alignment

//
// Source and Destination buffers have the same alignment. Move
// bytes until done or source and destination are quadword aligned
//

        and     a0, 0x7, t0             // t0 = unaligned bits
        bne     t0, 5f                  // if ne, not quad aligned
        br      zero, 20f               // predicted taken
5:
        bis     zero, zero, t1          // t4 = destination byte zap mask
        bis     zero, 1, t2
        sll     t2, t0, t2              // t2 = next bit to set in zap mask
10:
        beq     a2, 15f                 // if eq, all bits set
        bis     t1, t2, t1              // set bit in zap mask
        sll     t2, 1, t2               // set next higher bit for zap mask
        subq    a2, 1, a2               // decrement bytes to move
        addq    t0, 1, t0               // increment byte within quad
        cmpeq   t0, 8, t3               // finished the quadword?
        beq     t3, 10b                 // if eq [false], do next byte
15:
        ldq_u   t2, 0(a1)               // get unaligned quadword from source
        zapnot  t2, t1, t2              // clear source bytes
        bic     a0, 7, a3               // a3 = quadword base of destination
retry1:
        ldq_l   t0, 0(a3)               // load destination quadword
        zap     t0, t1, t0              // clear destination bytes
        or      t0, t2, t0              // merge in bytes from source
        stq_c   t0, 0(a3)               // store merged quadword conditional
        beq     t0, retry1f             // if eq, retry failed interlock

        addq    a0, 7, a0               // move to next quadword
        bic     a0, 7, a0               // aligned quadword

        addq    a1, 7, a1               // move to next quadword
        bic     a1, 7, a1               // aligned quadword

//
// Check for 64-byte block moves
//

20:
        srl     a2, 6, t0               // t0 = number of 64 byte blocks
        beq     t0, 40f                 // if eq no blocks to move
        and     a2, 64-1, a2            // a2 = residual bytes

30:
        ldq     t1, 0(a1)               // load 64 bytes from source
        addq    a0, 64, a0              // increment destination pointer
        ldq     v0, 56(a1)              //
        ldq     a3, 32(a1)              //
        stq     t1, -64(a0)             // write to destination
        ldq     t2, 8(a1)               //   into volatile registers
        ldq     t3, 16(a1)              //
        ldq     t4, 24(a1)              //
        subq    t0, 1, t0               // decrement number of blocks
        stq     t2, -56(a0)             //
        ldq     a4, 40(a1)              //
        stq     t3, -48(a0)             //
        ldq     a5, 48(a1)              //
        stq     t4, -40(a0)             //
        addq    a1, 64, a1              // increment source pointer
        stq     a3, -32(a0)             //
        stq     a4, -24(a0)             //
        stq     a5, -16(a0)             //
        stq     v0, -8(a0)              //
        bne     t0, 30b                 // if ne, more blocks to copy

//
// Copy quadwords
//

40:
        srl     a2, 3, t0               // t0 = number of quadwords to move
        beq     t0, 60f                 // if eq no quadwords to move
        and     a2, 8-1, a2             // a2 = residual bytes

50:
        ldq     t1, 0(a1)               // load quadword from source
        addq    a1, 8, a1               // increment source pointer
        stq     t1, 0(a0)               // store quadword to destination
        addq    a0, 8, a0               // increment destination pointer
        subq    t0, 1, t0               // decrement number of quadwords
        bne     t0, 50b                 // if ne, more quadwords to move

//
// Move final residual bytes
//

60:
        beq     a2, 80f                 // if eq, no more bytes to move
        mov     a2, t0                  // t0 = number of bytes to move
        mov     -1, t1                  // t1 = bit mask
        sll     t0, 3, t0               // # of bytes to # of bits
        srl     t1, t0, t1              // clear t0 bits
        sll     t1, t0, t0              // move it back
        ldq     t1, 0(a1)               // get last source quadword
        bic     t1, t0, t1              // clear bytes not copied
        not     t0, t0                  // complement to clear destination
retry2:
        ldq_l   t2, 0(a0)               // get last destination quadword locked
        bic     t2, t0, t2              // clear bytes to be copied
        bis     t2, t1, t2              // move bytes from source
        stq_c   t2, 0(a0)               // store merged quadword conditional
        beq     t2, retry2f             // if eq, retry failed interlock

//
// Finish aligned MoveForward
//

80:
        ret     zero, (ra)              // return

//
// Move memory forward unaligned.
//

CopyForwardUnaligned:                   //

//
// Move bytes until the destination is aligned
//

        and     a0, 0x7, t0             // t0 = unaligned bits
        beq     t0, 100f                // if eq, destination quad aligned
        bis     zero, zero, t1          // t4 = destination byte zap mask
        bis     zero, 1, t2
        sll     t2, t0, t2              // t2 = next bit to set in zap mask
        mov     zero, t4                // assemble destination bytes here
90:
        beq     a2, 95f                 // if eq no more bytes to move
        bis     t1, t2, t1              // set bit in zap mask
        sll     t2, 1, t2               // set next higher bit for zap mask
        ldq_u   t5, 0(a1)               // get unaligned quad from source
        extbl   t5, a1, t5              // extract source byte
        insbl   t5, t0, t5              // t5 = source byte, in position
        or      t4, t5, t4              // merge in source byte
        addq    t0, 1, t0               // increment byte position
        addq    a1, 1, a1               // increment source pointer
        subq    a2, 1, a2               // decrement bytes to move
        cmpeq   t0, 8, t3               // t0 = 8? => quad finished
        beq     t3, 90b                 // if eq [false], more bytes to move
95:
        bic     a0, 0x7, a3             // a3 = quadword base of destination
retry3:
        ldq_l   t0, 0(a3)               // load destination quadword
        zap     t0, t1, t0              // clear destination bytes
        or      t0, t4, t0              // merge in bytes from source
        stq_c   t0, 0(a3)               // store merged quadword conditional
        beq     t0, retry3f             // if eq, retry failed interlock

        addq    a0, 7, a0               // increment to next quad
        bic     a0, 7, a0               // align next quadword

//
// Check for 64-byte blocks to move
//

100:
        srl     a2, 6, t0               // t0 = number of blocks to move
        beq     t0, 120f                // if eq no blocks to move
        and     a2, 64-1, a2            // a2 = residual bytes to move

        ldq_u   t1, 0(a1)               // t1 = first unaligned quad
110:
                                        // get source data and merge it
                                        //  as we go
        ldq_u   t2, 7(a1)               // t2 = second unaligned quad
        extql   t1, a1, t1              // extract applicable bytes from t1
        extqh   t2, a1, v0              // extract applicable bytes from t2
        bis     t1, v0, t1              // t1 = quad #1
        ldq_u   t3, 15(a1)              // t3 = third unaligned quad
        extql   t2, a1, t2              // extract applicable bytes from t2
        extqh   t3, a1, v0              // extract applicable bytes from t3
        stq     t1, 0(a0)               // store quad #1
        bis     t2, v0, t2              // t2 = quad #2
        ldq_u   t4, 23(a1)              // t4 = fourth unaligned quad
        extql   t3, a1, t3              // extract applicable bytes from t3
        extqh   t4, a1, v0              // extract applicable bytes from t4
        stq     t2, 8(a0)               // store quad #2
        bis     t3, v0, t3              // t3 = quad #3
        ldq_u   t5, 31(a1)              // t5 = fifth unaligned quad
        extql   t4, a1, t4              // extract applicable bytes from t4
        extqh   t5, a1, v0              // extract applicable bytes from t5
        stq     t3, 16(a0)              // store quad #3
        bis     t4, v0, t4              // t4 = quad #4
        ldq_u   a3, 39(a1)              // a3 = sixth unaligned quad
        extql   t5, a1, t5              // extract applicable bytes from t5
        extqh   a3, a1, v0              // extract applicable bytes from a3
        stq     t4, 24(a0)              // store quad #4
        bis     t5, v0, t5              // t5 = quad #5
        ldq_u   a4, 47(a1)              // a4 = seventh unaligned quad
        extql   a3, a1, a3              // extract applicable bytes from a3
        extqh   a4, a1, v0              // extract applicable bytes from a4
        stq     t5, 32(a0)              // store quad #5
        bis     a3, v0, a3              // a3 = quad #6
        ldq_u   a5, 55(a1)              // a5 = eighth unaligned quad
        extql   a4, a1, a4              // extract applicable bytes from a4
        extqh   a5, a1, v0              // extract applicable bytes from a5
        stq     a3, 40(a0)              // store quad #6
        bis     a4, v0, a4              // a4 = quad #7
        ldq_u   t1, 63(a1)              // t1 = ninth unaligned = 1st of next
        extql   a5, a1, a5              // extract applicable bytes from a5
        extqh   t1, a1, v0              // extract applicable bytes from t1
        stq     a4, 48(a0)              // store quad #7
        bis     a5, v0, a5              // a5 = quad #8
        addq    a1, 64, a1              // increment source pointer
        stq     a5, 56(a0)              // store quad #8
        addq    a0, 64, a0              // increment destination pointer
        subq    t0, 1, t0               // decrement number of blocks
        bne     t0, 110b                // if ne, more blocks to move

//
// Move unaligned source quads to aligned destination quads
//

120:
        srl     a2, 3, t0               // t0 = number of quads to move
        beq     t0, 140f                // if eq no quads to move
        and     a2, 8-1, a2             // a2 = residual bytes


        ldq_u   t1, 0(a1)               // t1 = first unaligned quad
130:
        ldq_u   t2, 7(a1)               // t2 = second unaligned quad
        addq    a0, 8, a0               // increment destination pointer
        extql   t1, a1, t1              // extract applicable bytes from t1
        extqh   t2, a1, v0              // extract applicable bytes from t2
        bis     t1, v0, t1              // t1 = quadword of data
        stq     t1, -8(a0)              // store data to destination
        addq    a1, 8, a1               // increment source pointer
        subq    t0, 1, t0               // decrement quads to move
        bis     t2, zero, t1            // t1 = first of next unaligned pair
        bne     t0, 130b                // if ne, more quads to move

//
// Move remaining bytes to final quadword
//

140:    
        beq     a2, 160f                // if eq no more bytes to move

        mov     zero, t3                // t3 = position for next insertion
        mov     zero, t4                // assemble destination bytes here
        mov     a2, t0                  // t0 = number of bytes to move
        mov     -1, t1                  // t1 = bit mask
        sll     t0, 3, t0               // # of bytes to # of bits
        srl     t1, t0, t1              // clear t0 bits
        sll     t1, t0, t0              // move it back
        not     t0, t0                  // complement for destination clear mask
150:
        ldq_u   t1, 0(a1)               // get unaligned source quad
        extbl   t1, a1, t1              // t1 = source byte
        insbl   t1, t3, t1              // t1 = source byte, in position
        bis     t4, t1, t4              // merge in source byte
        addq    a1, 1, a1               // increment source pointer
        subq    a2, 1, a2               // decrement bytes to move
        addq    t3, 1, t3               // increment destination position
        bne     a2, 150b                // more bytes to move
retry4:
        ldq_l   t2, 0(a0)               // get last destination quadword locked
        bic     t2, t0, t2              // clear bytes to be copied
        bis     t2, t4, t2              // move bytes from source
        stq_c   t2, 0(a0)               // store merged quadword conditional
        beq     t2, retry4f             // if eq, retry failed interlock
           
//
// Finish unaligned MoveForward
//

160:
        ret     zero, (ra)              // return

//
// Out of line branches for failed store conditional.
// Don't need to restore anything, just try again.
//

retry1f:
        br      retry1
retry2f:
        br      retry2
retry3f:
        br      retry3
retry4f:
        br      retry4

        .end    RtlCopyBytes

        SBTTL("Zero Bytes")
//++
//
// VOID
// RtlZeroBytes (
//    IN PVOID Destination,
//    IN ULONG Length
//    )
//
// Routine Description:
//
//    This function zeros memory by first aligning the destination address to
//    a quadword boundary, and then zeroing 64-byte blocks, followed by 8-byte
//    blocks, followed by any remaining bytes. Unlike RtlZeroMemory the copy is
//    done such that byte granularity is assured for all platforms.
//
// Arguments:
//
//    Destination (a0) - Supplies a pointer to the memory to zero.
//
//    Length (a1) - Supplies the length, in bytes, of the memory to be zeroed.
//
// Return Value:
//
//    None.
//
//--

        LEAF_ENTRY(RtlZeroBytes)

        bis     zero, zero, a2          // set fill pattern
        br      zero, RtlpFillBytes     //


        SBTTL("Fill Bytes")
//++
//
// VOID
// RtlFillBytes (
//    IN PVOID Destination,
//    IN ULONG Length,
//    IN UCHAR Fill
//    )
//
// Routine Description:
//
//    This function fills memory by first aligning the destination address to
//    a longword boundary, and then filling 32-byte blocks, followed by 4-byte
//    blocks, followed by any remaining bytes. Unlike RtlFillMemory the copy is
//    done such that byte granularity is assured for all platforms.
//
// Arguments:
//
//    Destination (a0) - Supplies a pointer to the memory to fill.
//
//    Length (a1) - Supplies the length, in bytes, of the memory to be filled.
//
//    Fill (a2) - Supplies the fill byte.
//
//    N.B. The alternate entry memset expects the length and fill arguments
//         to be reversed.  It also returns the Destination pointer
//
// Return Value:
//
//    None.
//
//--

        ALTERNATE_ENTRY(RtlFillBytes)

        and     a2, 0xff, a2            // clear excess bits
        sll     a2, 8, t0               // duplicate fill byte
        bis     a2, t0, a2              // generate fill word
        sll     a2, 16, t0              // duplicate fill word
        bis     a2, t0, a2              // generate fill longword
        sll     a2, 32, t0              // duplicate fill longword
        bis     a2, t0, a2              // generate fill quadword

.align 3                                // ensure quadword aligned target
//
// Fill memory with the pattern specified in register a2.
//

RtlpFillBytes:                          //

//
// Align destination to quadword
//

        beq     a1, 80f                 // anything to fill? (paranoia)
        and     a0, 8-1, t0             // t0 = unaligned bits
        bne     t0, 5f                  // if ne, then not quad aligned
        br      zero, 20f               // if eq, then quad aligned

5:
        bis     zero, zero, t1          // t4 = destination byte zap mask
        bis     zero, 1, t2
        sll     t2, t0, t2              // t2 = next bit to set in zap mask
10:
        beq     a1, 15f                 // if eq, all bits set
        bis     t1, t2, t1              // set bit in zap mask
        sll     t2, 1, t2               // set next higher bit for zap mask
        subq    a1, 1, a1               // decrement bytes to fill
        addq    t0, 1, t0               // increment byte within quad
        cmpeq   t0, 8, t3               // finished the quadword?
        beq     t3, 10b                 // if eq [false], do next byte
15:
        zapnot  a2, t1, t2              // clear fill bytes
        bic     a0, 7, a3               // a3 = quadword base of destination
retry5:
        ldq_l   t0, 0(a3)               // load destination quadword
        zap     t0, t1, t0              // clear destination bytes
        or      t0, t2, t0              // merge in fill bytes
        stq_c   t0, 0(a3)               // store merged quadword conditional
        beq     t0, retry5f            // if eq, retry failed interlock

        addq    a0, 7, a0               // move a0 to next quadword
        bic     a0, 7, a0               // align a0 to quadword

//
// Check for 64-byte blocks
//

20:
        srl     a1, 6, t0               // t0 = number of 64 byte blocks
        beq     t0, 40f                 // if eq then no 64 byte blocks
        and     a1, 64-1, a1            // a1 = residual bytes to fill

30:
        stq     a2, 0(a0)               // store 64 bytes
        stq     a2, 8(a0)               //
        stq     a2, 16(a0)              //
        stq     a2, 24(a0)              //
        stq     a2, 32(a0)              //
        stq     a2, 40(a0)              //
        stq     a2, 48(a0)              //
        stq     a2, 56(a0)              //

        subq    t0, 1, t0               // decrement blocks remaining
        addq    a0, 64, a0              // increment destination pointer
        bne     t0, 30b                 // more blocks to write



//
// Fill aligned quadwords
//

40:
        srl     a1, 3, t0               // t0 = number of quadwords
        bne     t0, 55f                 // if ne quadwords left to fill
        br      zero, 60f               // if eq no quadwords left

55:
        and     a1, 8-1, a1             // a1 = residual bytes to fill

50:
        stq     a2, 0(a0)               // store quadword
        subq    t0, 1, t0               // decrement quadwords remaining
        addq    a0, 8, a0               // next quadword
        bne     t0, 50b                 // more quadwords to write

//
// Fill bytes for last quadword
//

60:
        beq     a1, 80f                 // if eq no more bytes to fill

        mov     a1, t0                  // t0 = number of bytes to move
        mov     -1, t1                  // t1 = bit mask
        sll     t0, 3, t0               // # of bytes to # of bits
        srl     t1, t0, t1              // clear t0 bits
        sll     t1, t0, t0              // move it back
        bic     a2, t0, t1              // clear fill bytes not copied
        not     t0, t0                  // complement to clear destination
retry6:
        ldq_l   t2, 0(a0)               // get last destination quadword locked
        bic     t2, t0, t2              // clear bytes to be copied
        bis     t2, t1, t2              // move bytes from source
        stq_c   t2, 0(a0)               // store merged quadword conditional
        beq     t2, retry6f             // if eq, retry failed interlock

//
// Finish up
//

80:
        ret     zero, (ra)              // return

//
// Out of line branches for failed store conditional.
// Don't need to restore anything, just try again.
//

retry5f:
        br      retry5
retry6f:
        br      retry6

        .end    RtlZeroBytes