917 lines
36 KiB
ArmAsm
917 lines
36 KiB
ArmAsm
/* $Id: VIScsumcopyusr.S,v 1.2 2000/02/20 23:21:40 davem Exp $
|
|
* VIScsumcopyusr.S: High bandwidth IP checksumming with simultaneous
|
|
* copying utilizing the UltraSparc Visual Instruction Set.
|
|
*
|
|
* Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
|
|
* Copyright (C) 2000 David S. Miller (davem@redhat.com)
|
|
*
|
|
* Based on older sparc32/sparc64 checksum.S, which is:
|
|
*
|
|
* Copyright(C) 1995 Linus Torvalds
|
|
* Copyright(C) 1995 Miguel de Icaza
|
|
* Copyright(C) 1996,1997 David S. Miller
|
|
* derived from:
|
|
* Linux/Alpha checksum c-code
|
|
* Linux/ix86 inline checksum assembly
|
|
* RFC1071 Computing the Internet Checksum (esp. Jacobsons m68k code)
|
|
* David Mosberger-Tang for optimized reference c-code
|
|
* BSD4.4 portable checksum routine
|
|
*/
|
|
|
|
#ifdef __sparc_v9__
|
|
#define STACKOFF 0x7ff+128
|
|
#else
|
|
#define STACKOFF 64
|
|
#endif
|
|
|
|
#ifdef __KERNEL__
|
|
#include <asm/head.h>
|
|
#include <asm/asi.h>
|
|
#include <asm/page.h>
|
|
#include <asm/visasm.h>
|
|
#include <asm/thread_info.h>
|
|
#define ASI_BLK_XOR 0
|
|
#define ASI_BLK_XOR1 (ASI_BLK_P ^ (ASI_BLK_P >> 3) ^ ASI_P)
|
|
#define ASI_BLK_OR (ASI_BLK_P & ~ASI_P)
|
|
#else
|
|
#define ASI_P 0x80
|
|
#define ASI_BLK_P 0xf0
|
|
#define FRPS_FEF 0x04
|
|
#define FPRS_DU 0x02
|
|
#define FPRS_DL 0x01
|
|
#define ASI_BLK_XOR (ASI_BLK_P ^ ASI_P)
|
|
#endif
|
|
|
|
#define src o0
|
|
#define dst o1
|
|
#define len o2
|
|
#define sum o3
|
|
#define x1 g1
|
|
#define x2 g2
|
|
#define x3 o4
|
|
#define x4 g4
|
|
#define x5 g5
|
|
#define x6 g7
|
|
#define x7 g3
|
|
#define x8 o5
|
|
|
|
/* Dobrou noc, SunSoft engineers. Spete sladce.
|
|
* This has a couple of tricks in and those
|
|
* tricks are UltraLinux trade secrets :))
|
|
* Once AGAIN, the SunSoft engineers are caught
|
|
* asleep at the keyboard :)).
|
|
* The main loop does about 20 superscalar cycles
|
|
* per 64bytes checksummed/copied.
|
|
*/
|
|
|
|
#define LDBLK(O0) \
|
|
ldda [%src] ASI_BLK_P, %O0 /* Load Group */
|
|
|
|
#define STBLK \
|
|
stda %f48, [%dst] %asi /* Store */
|
|
|
|
#ifdef __KERNEL__
|
|
#define STBLK_XORASI(tmpreg1,tmpreg2) \
|
|
stda %f48, [%dst] %asi /* Store */; \
|
|
rd %asi, %tmpreg1; \
|
|
srl %tmpreg1, 3, %tmpreg2; \
|
|
xor %tmpreg1, ASI_BLK_XOR1, %tmpreg1; \
|
|
wr %tmpreg1, %tmpreg2, %asi;
|
|
#else
|
|
#define STBLK_XORASI(tmpreg1,tmpreg2) \
|
|
stda %f48, [%dst] %asi /* Store */; \
|
|
rd %asi, %tmpreg1; \
|
|
wr %tmpreg1, ASI_BLK_XOR, %asi;
|
|
#endif
|
|
|
|
#define ST(fx,off) \
|
|
stda %fx, [%dst + off] %asi /* Store */
|
|
|
|
#define SYNC \
|
|
membar #Sync
|
|
|
|
|
|
#define DO_THE_TRICK(f0,f2,f4,f6,f8,f10,f12,f14,F0,F2,F4,F6,F8,F10,F12,F14,DUMMY1,A0,A2,A4,A6,A8,A10,A12,A14,B14,DUMMY2,LOAD,STORE1,STORE2,STORE3,STORE4,STORE5,STORE6,STORE7,STORE8,DUMMY3,BRANCH...) \
|
|
LOAD /* Load (Group) */; \
|
|
faligndata %A14, %F0, %A14 /* FPA Group */; \
|
|
inc %x5 /* IEU0 */; \
|
|
STORE1 /* Store (optional) */; \
|
|
faligndata %F0, %F2, %A0 /* FPA Group */; \
|
|
srl %x5, 1, %x5 /* IEU0 */; \
|
|
add %sum, %x4, %sum /* IEU1 */; \
|
|
fpadd32 %F0, %f0, %F0 /* FPA Group */; \
|
|
inc %x6 /* IEU0 */; \
|
|
STORE2 /* Store (optional) */; \
|
|
faligndata %F2, %F4, %A2 /* FPA Group */; \
|
|
srl %x6, 1, %x6 /* IEU0 */; \
|
|
add %sum, %x5, %sum /* IEU1 */; \
|
|
fpadd32 %F2, %f2, %F2 /* FPA Group */; \
|
|
add %src, 64, %src /* IEU0 */; \
|
|
fcmpgt32 %f0, %F0, %x1 /* FPM */; \
|
|
add %dst, 64, %dst /* IEU1 Group */; \
|
|
inc %x7 /* IEU0 */; \
|
|
STORE3 /* Store (optional) */; \
|
|
faligndata %F4, %F6, %A4 /* FPA */; \
|
|
fpadd32 %F4, %f4, %F4 /* FPA Group */; \
|
|
add %sum, %x6, %sum /* IEU1 */; \
|
|
fcmpgt32 %f2, %F2, %x2 /* FPM */; \
|
|
srl %x7, 1, %x7 /* IEU0 Group */; \
|
|
inc %x8 /* IEU1 */; \
|
|
STORE4 /* Store (optional) */; \
|
|
faligndata %F6, %F8, %A6 /* FPA */; \
|
|
fpadd32 %F6, %f6, %F6 /* FPA Group */; \
|
|
srl %x8, 1, %x8 /* IEU0 */; \
|
|
fcmpgt32 %f4, %F4, %x3 /* FPM */; \
|
|
add %sum, %x7, %sum /* IEU0 Group */; \
|
|
inc %x1 /* IEU1 */; \
|
|
STORE5 /* Store (optional) */; \
|
|
faligndata %F8, %F10, %A8 /* FPA */; \
|
|
fpadd32 %F8, %f8, %F8 /* FPA Group */; \
|
|
srl %x1, 1, %x1 /* IEU0 */; \
|
|
fcmpgt32 %f6, %F6, %x4 /* FPM */; \
|
|
add %sum, %x8, %sum /* IEU0 Group */; \
|
|
inc %x2 /* IEU1 */; \
|
|
STORE6 /* Store (optional) */; \
|
|
faligndata %F10, %F12, %A10 /* FPA */; \
|
|
fpadd32 %F10, %f10, %F10 /* FPA Group */; \
|
|
srl %x2, 1, %x2 /* IEU0 */; \
|
|
fcmpgt32 %f8, %F8, %x5 /* FPM */; \
|
|
add %sum, %x1, %sum /* IEU0 Group */; \
|
|
inc %x3 /* IEU1 */; \
|
|
STORE7 /* Store (optional) */; \
|
|
faligndata %F12, %F14, %A12 /* FPA */; \
|
|
fpadd32 %F12, %f12, %F12 /* FPA Group */; \
|
|
srl %x3, 1, %x3 /* IEU0 */; \
|
|
fcmpgt32 %f10, %F10, %x6 /* FPM */; \
|
|
add %sum, %x2, %sum /* IEU0 Group */; \
|
|
inc %x4 /* IEU1 */; \
|
|
STORE8 /* Store (optional) */; \
|
|
fmovd %F14, %B14 /* FPA */; \
|
|
fpadd32 %F14, %f14, %F14 /* FPA Group */; \
|
|
srl %x4, 1, %x4 /* IEU0 */; \
|
|
fcmpgt32 %f12, %F12, %x7 /* FPM */; \
|
|
add %sum, %x3, %sum /* IEU0 Group */; \
|
|
subcc %len, 64, %len /* IEU1 */; \
|
|
BRANCH /* CTI */; \
|
|
fcmpgt32 %f14, %F14, %x8 /* FPM Group */;
|
|
|
|
#define END_THE_TRICK(f0,f2,f4,f6,f8,f10,f12,f14,FA,FB,S0,S1,S2,S3,T0,T1,U0,fz) \
|
|
inc %x5 /* IEU0 Group */; \
|
|
fpadd32 %f2, %f0, %S0 /* FPA */; \
|
|
add %sum, %x4, %sum /* IEU1 */; \
|
|
srl %x5, 1, %x5 /* IEU0 Group */; \
|
|
fpadd32 %f6, %f4, %S1 /* FPA */; \
|
|
inc %x6 /* IEU1 */; \
|
|
fpadd32 %f10, %f8, %S2 /* FPA Group */; \
|
|
add %sum, %x5, %sum /* IEU0 */; \
|
|
fcmpgt32 %f0, %S0, %x1 /* FPM */; \
|
|
fpadd32 %f14, %f12, %S3 /* FPA Group */; \
|
|
srl %x6, 1, %x6 /* IEU0 */; \
|
|
fcmpgt32 %f4, %S1, %x2 /* FPM */; \
|
|
add %sum, %x6, %sum /* IEU0 Group */; \
|
|
fzero %fz /* FPA */; \
|
|
fcmpgt32 %f8, %S2, %x3 /* FPM */; \
|
|
inc %x7 /* IEU0 Group */; \
|
|
inc %x8 /* IEU1 */; \
|
|
srl %x7, 1, %x7 /* IEU0 Group */; \
|
|
inc %x1 /* IEU1 */; \
|
|
fpadd32 %S0, %S1, %T0 /* FPA */; \
|
|
fpadd32 %S2, %S3, %T1 /* FPA Group */; \
|
|
add %sum, %x7, %sum /* IEU0 */; \
|
|
fcmpgt32 %f12, %S3, %x4 /* FPM */; \
|
|
srl %x8, 1, %x8 /* IEU0 Group */; \
|
|
inc %x2 /* IEU1 */; \
|
|
srl %x1, 1, %x1 /* IEU0 Group */; \
|
|
add %sum, %x8, %sum /* IEU1 */; \
|
|
add %sum, %x1, %sum /* IEU0 Group */; \
|
|
fcmpgt32 %S0, %T0, %x5 /* FPM */; \
|
|
srl %x2, 1, %x2 /* IEU0 Group */; \
|
|
fcmpgt32 %S2, %T1, %x6 /* FPM */; \
|
|
inc %x3 /* IEU0 Group */; \
|
|
add %sum, %x2, %sum /* IEU1 */; \
|
|
srl %x3, 1, %x3 /* IEU0 Group */; \
|
|
inc %x4 /* IEU1 */; \
|
|
fpadd32 %T0, %T1, %U0 /* FPA Group */; \
|
|
add %sum, %x3, %sum /* IEU0 */; \
|
|
fcmpgt32 %fz, %f2, %x7 /* FPM */; \
|
|
srl %x4, 1, %x4 /* IEU0 Group */; \
|
|
fcmpgt32 %fz, %f6, %x8 /* FPM */; \
|
|
inc %x5 /* IEU0 Group */; \
|
|
add %sum, %x4, %sum /* IEU1 */; \
|
|
srl %x5, 1, %x5 /* IEU0 Group */; \
|
|
fcmpgt32 %fz, %f10, %x1 /* FPM */; \
|
|
inc %x6 /* IEU0 Group */; \
|
|
add %sum, %x5, %sum /* IEU1 */; \
|
|
fmovd %FA, %FB /* FPA Group */; \
|
|
fcmpgt32 %fz, %f14, %x2 /* FPM */; \
|
|
srl %x6, 1, %x6 /* IEU0 Group */; \
|
|
ba,pt %xcc, ett /* CTI */; \
|
|
inc %x7 /* IEU1 */;
|
|
|
|
#define END_THE_TRICK1(f0,f2,f4,f6,f8,f10,f12,f14,FA,FB) \
|
|
END_THE_TRICK(f0,f2,f4,f6,f8,f10,f12,f14,FA,FB,f48,f50,f52,f54,f56,f58,f60,f62)
|
|
|
|
#define END_THE_TRICK2(S0,S1,S2,S3,T0,T1,U0,U1,V0,fz) \
|
|
fpadd32 %U0, %U1, %V0 /* FPA Group */; \
|
|
srl %x7, 1, %x7 /* IEU0 */; \
|
|
add %sum, %x6, %sum /* IEU1 */; \
|
|
std %V0, [%sp + STACKOFF] /* Store Group */; \
|
|
inc %x8 /* IEU0 */; \
|
|
sub %sum, %x7, %sum /* IEU1 */; \
|
|
srl %x8, 1, %x8 /* IEU0 Group */; \
|
|
fcmpgt32 %fz, %S1, %x3 /* FPM */; \
|
|
inc %x1 /* IEU0 Group */; \
|
|
fcmpgt32 %fz, %S3, %x4 /* FPM */; \
|
|
srl %x1, 1, %x1 /* IEU0 Group */; \
|
|
sub %sum, %x8, %sum /* IEU1 */; \
|
|
ldx [%sp + STACKOFF], %x8 /* Load Group */; \
|
|
inc %x2 /* IEU0 */; \
|
|
sub %sum, %x1, %sum /* IEU1 */; \
|
|
srl %x2, 1, %x2 /* IEU0 Group */; \
|
|
fcmpgt32 %fz, %T1, %x5 /* FPM */; \
|
|
inc %x3 /* IEU0 Group */; \
|
|
fcmpgt32 %T0, %U0, %x6 /* FPM */; \
|
|
srl %x3, 1, %x3 /* IEU0 Group */; \
|
|
sub %sum, %x2, %sum /* IEU1 */; \
|
|
inc %x4 /* IEU0 Group */; \
|
|
sub %sum, %x3, %sum /* IEU1 */; \
|
|
srl %x4, 1, %x4 /* IEU0 Group */; \
|
|
fcmpgt32 %fz, %U1, %x7 /* FPM */; \
|
|
inc %x5 /* IEU0 Group */; \
|
|
fcmpgt32 %U0, %V0, %x1 /* FPM */; \
|
|
srl %x5, 1, %x5 /* IEU0 Group */; \
|
|
sub %sum, %x4, %sum /* IEU1 */; \
|
|
sub %sum, %x5, %sum /* IEU0 Group */; \
|
|
fcmpgt32 %fz, %V0, %x2 /* FPM */; \
|
|
inc %x6 /* IEU0 Group */; \
|
|
inc %x7 /* IEU1 */; \
|
|
srl %x6, 1, %x6 /* IEU0 Group */; \
|
|
inc %x1 /* IEU1 */; \
|
|
srl %x7, 1, %x7 /* IEU0 Group */; \
|
|
add %sum, %x6, %sum /* IEU1 */; \
|
|
srl %x1, 1, %x1 /* IEU0 Group */; \
|
|
sub %sum, %x7, %sum /* IEU1 */; \
|
|
inc %x2 /* IEU0 Group */; \
|
|
add %sum, %x1, %sum /* IEU1 */; \
|
|
srl %x2, 1, %x2 /* IEU0 Group */; \
|
|
sub %sum, %x2, %sum /* IEU0 Group */; \
|
|
addcc %sum, %x8, %sum /* IEU1 Group */; \
|
|
bcs,a,pn %xcc, 33f /* CTI */; \
|
|
add %sum, 1, %sum /* IEU0 (Group) */; \
|
|
33: /* That's it */;
|
|
|
|
.text
|
|
.globl csum_partial_copy_user_vis
|
|
.align 32
|
|
/* %asi should be either ASI_P or ASI_AIUS for csum_partial_copy resp.
|
|
* csum_partial_copy_from_user
|
|
* This assumes that !((%src^%dst)&3) && !((%src|%dst)&1) && %len >= 256
|
|
*/
|
|
csum_partial_copy_user_vis:
|
|
andcc %dst, 7, %g0 /* IEU1 Group */
|
|
be,pt %icc, 4f /* CTI */
|
|
and %dst, 0x38, %o4 /* IEU0 */
|
|
mov 1, %g5 /* IEU0 Group */
|
|
andcc %dst, 2, %g0 /* IEU1 */
|
|
be,pt %icc, 1f /* CTI */
|
|
and %dst, 4, %g7 /* IEU0 Group */
|
|
lduh [%src], %g2 /* Load */
|
|
sub %len, 2, %len /* IEU0 Group */
|
|
add %dst, 2, %dst /* IEU1 */
|
|
andcc %dst, 4, %g7 /* IEU1 Group */
|
|
sll %g5, 16, %g5 /* IEU0 */
|
|
stha %g2, [%dst - 2] %asi /* Store Group */
|
|
sll %g2, 16, %g2 /* IEU0 */
|
|
add %src, 2, %src /* IEU1 */
|
|
addcc %g2, %sum, %sum /* IEU1 Group */
|
|
bcs,a,pn %icc, 1f /* CTI */
|
|
add %sum, %g5, %sum /* IEU0 */
|
|
1: lduw [%src], %g2 /* Load */
|
|
brz,a,pn %g7, 4f /* CTI+IEU1 Group */
|
|
and %dst, 0x38, %o4 /* IEU0 */
|
|
add %dst, 4, %dst /* IEU0 Group */
|
|
sub %len, 4, %len /* IEU1 */
|
|
addcc %g2, %sum, %sum /* IEU1 Group */
|
|
bcs,a,pn %icc, 1f /* CTI */
|
|
add %sum, 1, %sum /* IEU0 */
|
|
1: and %dst, 0x38, %o4 /* IEU0 Group */
|
|
stwa %g2, [%dst - 4] %asi /* Store */
|
|
add %src, 4, %src /* IEU1 */
|
|
4:
|
|
#ifdef __KERNEL__
|
|
VISEntry
|
|
#endif
|
|
mov %src, %g7 /* IEU1 Group */
|
|
fzero %f48 /* FPA */
|
|
alignaddr %src, %g0, %src /* Single Group */
|
|
subcc %g7, %src, %g7 /* IEU1 Group */
|
|
be,pt %xcc, 1f /* CTI */
|
|
mov 0x40, %g1 /* IEU0 */
|
|
lduw [%src], %g2 /* Load Group */
|
|
subcc %sum, %g2, %sum /* IEU1 Group+load stall*/
|
|
bcs,a,pn %icc, 1f /* CTI */
|
|
sub %sum, 1, %sum /* IEU0 */
|
|
1: srl %sum, 0, %sum /* IEU0 Group */
|
|
clr %g5 /* IEU1 */
|
|
brz,pn %o4, 3f /* CTI+IEU1 Group */
|
|
sub %g1, %o4, %g1 /* IEU0 */
|
|
ldd [%src], %f0 /* Load */
|
|
clr %o4 /* IEU0 Group */
|
|
andcc %dst, 8, %g0 /* IEU1 */
|
|
be,pn %icc, 1f /* CTI */
|
|
ldd [%src + 8], %f2 /* Load Group */
|
|
add %src, 8, %src /* IEU0 */
|
|
sub %len, 8, %len /* IEU1 */
|
|
fpadd32 %f0, %f48, %f50 /* FPA */
|
|
addcc %dst, 8, %dst /* IEU1 Group */
|
|
faligndata %f0, %f2, %f16 /* FPA */
|
|
fcmpgt32 %f48, %f50, %o4 /* FPM Group */
|
|
fmovd %f2, %f0 /* FPA Group */
|
|
ldd [%src + 8], %f2 /* Load */
|
|
stda %f16, [%dst - 8] %asi /* Store */
|
|
fmovd %f50, %f48 /* FPA */
|
|
1: andcc %g1, 0x10, %g0 /* IEU1 Group */
|
|
be,pn %icc, 1f /* CTI */
|
|
and %g1, 0x20, %g1 /* IEU0 */
|
|
fpadd32 %f0, %f48, %f50 /* FPA */
|
|
ldd [%src + 16], %f4 /* Load Group */
|
|
add %src, 16, %src /* IEU0 */
|
|
add %dst, 16, %dst /* IEU1 */
|
|
faligndata %f0, %f2, %f16 /* FPA */
|
|
fcmpgt32 %f48, %f50, %g5 /* FPM Group */
|
|
sub %len, 16, %len /* IEU0 */
|
|
inc %o4 /* IEU1 */
|
|
stda %f16, [%dst - 16] %asi /* Store Group */
|
|
fpadd32 %f2, %f50, %f48 /* FPA */
|
|
srl %o4, 1, %o5 /* IEU0 */
|
|
faligndata %f2, %f4, %f18 /* FPA Group */
|
|
stda %f18, [%dst - 8] %asi /* Store */
|
|
fcmpgt32 %f50, %f48, %o4 /* FPM Group */
|
|
add %o5, %sum, %sum /* IEU0 */
|
|
ldd [%src + 8], %f2 /* Load */
|
|
fmovd %f4, %f0 /* FPA */
|
|
1: brz,a,pn %g1, 4f /* CTI+IEU1 Group */
|
|
rd %asi, %g2 /* LSU Group + 4 bubbles*/
|
|
inc %g5 /* IEU0 */
|
|
fpadd32 %f0, %f48, %f50 /* FPA */
|
|
ldd [%src + 16], %f4 /* Load Group */
|
|
srl %g5, 1, %g5 /* IEU0 */
|
|
add %dst, 32, %dst /* IEU1 */
|
|
faligndata %f0, %f2, %f16 /* FPA */
|
|
fcmpgt32 %f48, %f50, %o5 /* FPM Group */
|
|
inc %o4 /* IEU0 */
|
|
ldd [%src + 24], %f6 /* Load */
|
|
srl %o4, 1, %o4 /* IEU0 Group */
|
|
add %g5, %sum, %sum /* IEU1 */
|
|
ldd [%src + 32], %f8 /* Load */
|
|
fpadd32 %f2, %f50, %f48 /* FPA */
|
|
faligndata %f2, %f4, %f18 /* FPA Group */
|
|
sub %len, 32, %len /* IEU0 */
|
|
stda %f16, [%dst - 32] %asi /* Store */
|
|
fcmpgt32 %f50, %f48, %g3 /* FPM Group */
|
|
inc %o5 /* IEU0 */
|
|
add %o4, %sum, %sum /* IEU1 */
|
|
fpadd32 %f4, %f48, %f50 /* FPA */
|
|
faligndata %f4, %f6, %f20 /* FPA Group */
|
|
srl %o5, 1, %o5 /* IEU0 */
|
|
fcmpgt32 %f48, %f50, %g5 /* FPM Group */
|
|
add %o5, %sum, %sum /* IEU0 */
|
|
stda %f18, [%dst - 24] %asi /* Store */
|
|
fpadd32 %f6, %f50, %f48 /* FPA */
|
|
inc %g3 /* IEU0 Group */
|
|
stda %f20, [%dst - 16] %asi /* Store */
|
|
add %src, 32, %src /* IEU1 */
|
|
faligndata %f6, %f8, %f22 /* FPA */
|
|
fcmpgt32 %f50, %f48, %o4 /* FPM Group */
|
|
srl %g3, 1, %g3 /* IEU0 */
|
|
stda %f22, [%dst - 8] %asi /* Store */
|
|
add %g3, %sum, %sum /* IEU0 Group */
|
|
3: rd %asi, %g2 /* LSU Group + 4 bubbles*/
|
|
#ifdef __KERNEL__
|
|
4: sethi %hi(vis0s), %g7 /* IEU0 Group */
|
|
or %g2, ASI_BLK_OR, %g2 /* IEU1 */
|
|
#else
|
|
4: rd %pc, %g7 /* LSU Group + 4 bubbles*/
|
|
#endif
|
|
inc %g5 /* IEU0 Group */
|
|
and %src, 0x38, %g3 /* IEU1 */
|
|
membar #StoreLoad /* LSU Group */
|
|
srl %g5, 1, %g5 /* IEU0 */
|
|
inc %o4 /* IEU1 */
|
|
sll %g3, 8, %g3 /* IEU0 Group */
|
|
sub %len, 0xc0, %len /* IEU1 */
|
|
addcc %g5, %sum, %sum /* IEU1 Group */
|
|
srl %o4, 1, %o4 /* IEU0 */
|
|
add %g7, %g3, %g7 /* IEU0 Group */
|
|
add %o4, %sum, %sum /* IEU1 */
|
|
#ifdef __KERNEL__
|
|
jmpl %g7 + %lo(vis0s), %g0 /* CTI+IEU1 Group */
|
|
#else
|
|
jmpl %g7 + (vis0s - 4b), %g0 /* CTI+IEU1 Group */
|
|
#endif
|
|
fzero %f32 /* FPA */
|
|
|
|
.align 2048
|
|
vis0s: wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
|
|
ldda [%src] ASI_BLK_P, %f0 /* Load Group */
|
|
add %src, 64, %src /* IEU0 Group */
|
|
ldda [%src] ASI_BLK_P, %f16 /* Load Group */
|
|
add %src, 64, %src /* IEU0 Group */
|
|
fmovd %f48, %f62 /* FPA Group f0 available*/
|
|
faligndata %f0, %f2, %f48 /* FPA Group f2 available*/
|
|
fcmpgt32 %f32, %f2, %x1 /* FPM Group f4 available*/
|
|
fpadd32 %f0, %f62, %f0 /* FPA */
|
|
fcmpgt32 %f32, %f4, %x2 /* FPM Group f6 available*/
|
|
faligndata %f2, %f4, %f50 /* FPA */
|
|
fcmpgt32 %f62, %f0, %x3 /* FPM Group f8 available*/
|
|
faligndata %f4, %f6, %f52 /* FPA */
|
|
fcmpgt32 %f32, %f6, %x4 /* FPM Group f10 available*/
|
|
inc %x1 /* IEU0 */
|
|
faligndata %f6, %f8, %f54 /* FPA */
|
|
fcmpgt32 %f32, %f8, %x5 /* FPM Group f12 available*/
|
|
srl %x1, 1, %x1 /* IEU0 */
|
|
inc %x2 /* IEU1 */
|
|
faligndata %f8, %f10, %f56 /* FPA */
|
|
fcmpgt32 %f32, %f10, %x6 /* FPM Group f14 available*/
|
|
srl %x2, 1, %x2 /* IEU0 */
|
|
add %sum, %x1, %sum /* IEU1 */
|
|
faligndata %f10, %f12, %f58 /* FPA */
|
|
fcmpgt32 %f32, %f12, %x7 /* FPM Group */
|
|
inc %x3 /* IEU0 */
|
|
add %sum, %x2, %sum /* IEU1 */
|
|
faligndata %f12, %f14, %f60 /* FPA */
|
|
fcmpgt32 %f32, %f14, %x8 /* FPM Group */
|
|
srl %x3, 1, %x3 /* IEU0 */
|
|
inc %x4 /* IEU1 */
|
|
fmovd %f14, %f62 /* FPA */
|
|
srl %x4, 1, %x4 /* IEU0 Group */
|
|
add %sum, %x3, %sum /* IEU1 */
|
|
vis0: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f48,f50,f52,f54,f56,f58,f60,f62,f62,
|
|
,LDBLK(f32), STBLK,,,,,,,,
|
|
,bcs,pn %icc, vis0e1)
|
|
DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f48,f50,f52,f54,f56,f58,f60,f62,f62,
|
|
,LDBLK(f0), STBLK,,,,,,,,
|
|
,bcs,pn %icc, vis0e2)
|
|
DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f48,f50,f52,f54,f56,f58,f60,f62,f62,
|
|
,LDBLK(f16), STBLK,,,,,,,,
|
|
,bcc,pt %icc, vis0)
|
|
vis0e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f48,f50,f52,f54,f56,f58,f60,f62,f32,
|
|
,SYNC, STBLK_XORASI(x1,x2),ST(f48,64),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),ST(f60,48),
|
|
,add %dst, 56, %dst; add %len, 192 - 8*8, %len; ba,pt %icc, e2)
|
|
vis0e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f48,f50,f52,f54,f56,f58,f60,f62,f0,
|
|
,SYNC, STBLK_XORASI(x1,x2),ST(f48,64),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),ST(f60,48),
|
|
,add %dst, 56, %dst; add %len, 192 - 8*8, %len; ba,pt %icc, e3)
|
|
vis0e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f48,f50,f52,f54,f56,f58,f60,f62,f16,
|
|
,SYNC, STBLK_XORASI(x1,x2),ST(f48,64),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),ST(f60,48),
|
|
,add %dst, 56, %dst; add %len, 192 - 8*8, %len; ba,pt %icc, e1)
|
|
.align 2048
|
|
vis1s: wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
|
|
sub %src, 8, %src /* IEU0 Group */
|
|
ldda [%src] ASI_BLK_P, %f0 /* Load Group */
|
|
add %src, 64, %src /* IEU0 Group */
|
|
ldda [%src] ASI_BLK_P, %f16 /* Load Group */
|
|
add %src, 64, %src /* IEU0 Group */
|
|
fmovd %f0, %f58 /* FPA Group */
|
|
fmovd %f48, %f0 /* FPA Group */
|
|
fcmpgt32 %f32, %f2, %x2 /* FPM Group */
|
|
faligndata %f2, %f4, %f48 /* FPA */
|
|
fcmpgt32 %f32, %f4, %x3 /* FPM Group */
|
|
faligndata %f4, %f6, %f50 /* FPA */
|
|
fcmpgt32 %f32, %f6, %x4 /* FPM Group */
|
|
faligndata %f6, %f8, %f52 /* FPA */
|
|
fcmpgt32 %f32, %f8, %x5 /* FPM Group */
|
|
inc %x2 /* IEU1 */
|
|
faligndata %f8, %f10, %f54 /* FPA */
|
|
fcmpgt32 %f32, %f10, %x6 /* FPM Group */
|
|
srl %x2, 1, %x2 /* IEU0 */
|
|
faligndata %f10, %f12, %f56 /* FPA */
|
|
fcmpgt32 %f32, %f12, %x7 /* FPM Group */
|
|
inc %x3 /* IEU0 */
|
|
add %sum, %x2, %sum /* IEU1 */
|
|
faligndata %f12, %f14, %f58 /* FPA */
|
|
fcmpgt32 %f32, %f14, %x8 /* FPM Group */
|
|
srl %x3, 1, %x3 /* IEU0 */
|
|
inc %x4 /* IEU1 */
|
|
fmovd %f14, %f60 /* FPA */
|
|
srl %x4, 1, %x4 /* IEU0 Group */
|
|
add %sum, %x3, %sum /* IEU1 */
|
|
vis1: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f62,f48,f50,f52,f54,f56,f58,f60,f60,
|
|
,LDBLK(f32), ,STBLK,,,,,,,
|
|
,bcs,pn %icc, vis1e1)
|
|
DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f62,f48,f50,f52,f54,f56,f58,f60,f60,
|
|
,LDBLK(f0), ,STBLK,,,,,,,
|
|
,bcs,pn %icc, vis1e2)
|
|
DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f62,f48,f50,f52,f54,f56,f58,f60,f60,
|
|
,LDBLK(f16), ,STBLK,,,,,,,
|
|
,bcc,pt %icc, vis1)
|
|
vis1e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f62,f48,f50,f52,f54,f56,f58,f60,f32,
|
|
,SYNC, ,STBLK_XORASI(x1,x2),ST(f48,0),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),
|
|
,add %dst, 48, %dst; add %len, 192 - 7*8, %len; ba,pt %icc, e2)
|
|
vis1e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f62,f48,f50,f52,f54,f56,f58,f60,f0,
|
|
,SYNC, ,STBLK_XORASI(x1,x2),ST(f48,0),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),
|
|
,add %dst, 48, %dst; add %len, 192 - 7*8, %len; ba,pt %icc, e3)
|
|
vis1e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f62,f48,f50,f52,f54,f56,f58,f60,f16,
|
|
,SYNC, ,STBLK_XORASI(x1,x2),ST(f48,0),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),
|
|
,add %dst, 48, %dst; add %len, 192 - 7*8, %len; ba,pt %icc, e1)
|
|
.align 2048
|
|
vis2s: wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
|
|
sub %src, 16, %src /* IEU0 Group */
|
|
ldda [%src] ASI_BLK_P, %f0 /* Load Group */
|
|
add %src, 64, %src /* IEU0 Group */
|
|
ldda [%src] ASI_BLK_P, %f16 /* Load Group */
|
|
add %src, 64, %src /* IEU0 Group */
|
|
fmovd %f0, %f56 /* FPA Group */
|
|
fmovd %f48, %f0 /* FPA Group */
|
|
sub %dst, 64, %dst /* IEU0 */
|
|
fpsub32 %f2, %f2, %f2 /* FPA Group */
|
|
fcmpgt32 %f32, %f4, %x3 /* FPM Group */
|
|
faligndata %f4, %f6, %f48 /* FPA */
|
|
fcmpgt32 %f32, %f6, %x4 /* FPM Group */
|
|
faligndata %f6, %f8, %f50 /* FPA */
|
|
fcmpgt32 %f32, %f8, %x5 /* FPM Group */
|
|
faligndata %f8, %f10, %f52 /* FPA */
|
|
fcmpgt32 %f32, %f10, %x6 /* FPM Group */
|
|
faligndata %f10, %f12, %f54 /* FPA */
|
|
fcmpgt32 %f32, %f12, %x7 /* FPM Group */
|
|
inc %x3 /* IEU0 */
|
|
faligndata %f12, %f14, %f56 /* FPA */
|
|
fcmpgt32 %f32, %f14, %x8 /* FPM Group */
|
|
srl %x3, 1, %x3 /* IEU0 */
|
|
inc %x4 /* IEU1 */
|
|
fmovd %f14, %f58 /* FPA */
|
|
srl %x4, 1, %x4 /* IEU0 Group */
|
|
add %sum, %x3, %sum /* IEU1 */
|
|
vis2: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f60,f62,f48,f50,f52,f54,f56,f58,f58,
|
|
,LDBLK(f32), ,,STBLK,,,,,,
|
|
,bcs,pn %icc, vis2e1)
|
|
DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f60,f62,f48,f50,f52,f54,f56,f58,f58,
|
|
,LDBLK(f0), ,,STBLK,,,,,,
|
|
,bcs,pn %icc, vis2e2)
|
|
DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f60,f62,f48,f50,f52,f54,f56,f58,f58,
|
|
,LDBLK(f16), ,,STBLK,,,,,,
|
|
,bcc,pt %icc, vis2)
|
|
vis2e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f60,f62,f48,f50,f52,f54,f56,f58,f32,
|
|
,SYNC, ,,STBLK_XORASI(x2,x3),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),ST(f56,96),
|
|
,add %dst, 104, %dst; add %len, 192 - 6*8, %len; ba,pt %icc, e2)
|
|
vis2e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f60,f62,f48,f50,f52,f54,f56,f58,f0,
|
|
,SYNC, ,,STBLK_XORASI(x2,x3),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),ST(f56,96),
|
|
,add %dst, 104, %dst; add %len, 192 - 6*8, %len; ba,pt %icc, e3)
|
|
vis2e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f60,f62,f48,f50,f52,f54,f56,f58,f16,
|
|
,SYNC, ,,STBLK_XORASI(x2,x3),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),ST(f56,96),
|
|
,add %dst, 104, %dst; add %len, 192 - 6*8, %len; ba,pt %icc, e1)
|
|
.align 2048
|
|
vis3s: wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
|
|
sub %src, 24, %src /* IEU0 Group */
|
|
ldda [%src] ASI_BLK_P, %f0 /* Load Group */
|
|
add %src, 64, %src /* IEU0 Group */
|
|
ldda [%src] ASI_BLK_P, %f16 /* Load Group */
|
|
add %src, 64, %src /* IEU0 Group */
|
|
fmovd %f0, %f54 /* FPA Group */
|
|
fmovd %f48, %f0 /* FPA Group */
|
|
sub %dst, 64, %dst /* IEU0 */
|
|
fpsub32 %f2, %f2, %f2 /* FPA Group */
|
|
fpsub32 %f4, %f4, %f4 /* FPA Group */
|
|
fcmpgt32 %f32, %f6, %x4 /* FPM Group */
|
|
faligndata %f6, %f8, %f48 /* FPA */
|
|
fcmpgt32 %f32, %f8, %x5 /* FPM Group */
|
|
faligndata %f8, %f10, %f50 /* FPA */
|
|
fcmpgt32 %f32, %f10, %x6 /* FPM Group */
|
|
faligndata %f10, %f12, %f52 /* FPA */
|
|
fcmpgt32 %f32, %f12, %x7 /* FPM Group */
|
|
faligndata %f12, %f14, %f54 /* FPA */
|
|
fcmpgt32 %f32, %f14, %x8 /* FPM Group */
|
|
fmovd %f14, %f56 /* FPA */
|
|
inc %x4 /* IEU0 */
|
|
srl %x4, 1, %x4 /* IEU0 Group */
|
|
vis3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f58,f60,f62,f48,f50,f52,f54,f56,f56,
|
|
,LDBLK(f32), ,,,STBLK,,,,,
|
|
,bcs,pn %icc, vis3e1)
|
|
DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f58,f60,f62,f48,f50,f52,f54,f56,f56,
|
|
,LDBLK(f0), ,,,STBLK,,,,,
|
|
,bcs,pn %icc, vis3e2)
|
|
DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f58,f60,f62,f48,f50,f52,f54,f56,f56,
|
|
,LDBLK(f16), ,,,STBLK,,,,,
|
|
,bcc,pt %icc, vis3)
|
|
vis3e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f58,f60,f62,f48,f50,f52,f54,f56,f32,
|
|
,SYNC, ,,,STBLK_XORASI(x3,x4),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),
|
|
,add %dst, 96, %dst; add %len, 192 - 5*8, %len; ba,pt %icc, e2)
|
|
vis3e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f58,f60,f62,f48,f50,f52,f54,f56,f0,
|
|
,SYNC, ,,,STBLK_XORASI(x3,x4),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),
|
|
,add %dst, 96, %dst; add %len, 192 - 5*8, %len; ba,pt %icc, e3)
|
|
vis3e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f58,f60,f62,f48,f50,f52,f54,f56,f16,
|
|
,SYNC, ,,,STBLK_XORASI(x3,x4),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),
|
|
,add %dst, 96, %dst; add %len, 192 - 5*8, %len; ba,pt %icc, e1)
|
|
.align 2048
|
|
vis4s: wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
|
|
sub %src, 32, %src /* IEU0 Group */
|
|
ldda [%src] ASI_BLK_P, %f0 /* Load Group */
|
|
add %src, 64, %src /* IEU0 Group */
|
|
ldda [%src] ASI_BLK_P, %f16 /* Load Group */
|
|
add %src, 64, %src /* IEU0 Group */
|
|
fmovd %f0, %f52 /* FPA Group */
|
|
fmovd %f48, %f0 /* FPA Group */
|
|
sub %dst, 64, %dst /* IEU0 */
|
|
fpsub32 %f2, %f2, %f2 /* FPA Group */
|
|
fpsub32 %f4, %f4, %f4 /* FPA Group */
|
|
fpsub32 %f6, %f6, %f6 /* FPA Group */
|
|
clr %x4 /* IEU0 */
|
|
fcmpgt32 %f32, %f8, %x5 /* FPM Group */
|
|
faligndata %f8, %f10, %f48 /* FPA */
|
|
fcmpgt32 %f32, %f10, %x6 /* FPM Group */
|
|
faligndata %f10, %f12, %f50 /* FPA */
|
|
fcmpgt32 %f32, %f12, %x7 /* FPM Group */
|
|
faligndata %f12, %f14, %f52 /* FPA */
|
|
fcmpgt32 %f32, %f14, %x8 /* FPM Group */
|
|
fmovd %f14, %f54 /* FPA */
|
|
vis4: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f56,f58,f60,f62,f48,f50,f52,f54,f54,
|
|
,LDBLK(f32), ,,,,STBLK,,,,
|
|
,bcs,pn %icc, vis4e1)
|
|
DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f56,f58,f60,f62,f48,f50,f52,f54,f54,
|
|
,LDBLK(f0), ,,,,STBLK,,,,
|
|
,bcs,pn %icc, vis4e2)
|
|
DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f56,f58,f60,f62,f48,f50,f52,f54,f54,
|
|
,LDBLK(f16), ,,,,STBLK,,,,
|
|
,bcc,pt %icc, vis4)
|
|
vis4e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f56,f58,f60,f62,f48,f50,f52,f54,f32,
|
|
,SYNC, ,,,,STBLK_XORASI(x4,x5),ST(f48,64),ST(f50,72),ST(f52,80),
|
|
,add %dst, 88, %dst; add %len, 192 - 4*8, %len; ba,pt %icc, e2)
|
|
vis4e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f56,f58,f60,f62,f48,f50,f52,f54,f0,
|
|
,SYNC, ,,,,STBLK_XORASI(x4,x5),ST(f48,64),ST(f50,72),ST(f52,80),
|
|
,add %dst, 88, %dst; add %len, 192 - 4*8, %len; ba,pt %icc, e3)
|
|
vis4e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f56,f58,f60,f62,f48,f50,f52,f54,f16,
|
|
,SYNC, ,,,,STBLK_XORASI(x4,x5),ST(f48,64),ST(f50,72),ST(f52,80),
|
|
,add %dst, 88, %dst; add %len, 192 - 4*8, %len; ba,pt %icc, e1)
|
|
.align 2048
|
|
vis5s: ldd [%src+0], %f10 /* Load Group */
|
|
ldd [%src+8], %f12 /* Load Group */
|
|
ldd [%src+16], %f14 /* Load Group */
|
|
add %src, 24, %src /* IEU0 Group */
|
|
wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
|
|
ldda [%src] ASI_BLK_P, %f16 /* Load Group */
|
|
add %src, 64, %src /* IEU0 Group */
|
|
fmovd %f48, %f0 /* FPA Group */
|
|
fmuld %f32, %f32, %f2 /* FPM */
|
|
clr %x4 /* IEU0 */
|
|
faddd %f32, %f32, %f4 /* FPA Group */
|
|
fmuld %f32, %f32, %f6 /* FPM */
|
|
clr %x5 /* IEU0 */
|
|
faddd %f32, %f32, %f8 /* FPA Group */
|
|
fcmpgt32 %f32, %f10, %x6 /* FPM Group */
|
|
sub %dst, 64, %dst /* IEU0 */
|
|
faligndata %f10, %f12, %f48 /* FPA */
|
|
fcmpgt32 %f32, %f12, %x7 /* FPM Group */
|
|
faligndata %f12, %f14, %f50 /* FPA */
|
|
fcmpgt32 %f32, %f14, %x8 /* FPM Group */
|
|
fmovd %f14, %f52 /* FPA */
|
|
vis5: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f54,f56,f58,f60,f62,f48,f50,f52,f52,
|
|
,LDBLK(f32), ,,,,,STBLK,,,
|
|
,bcs,pn %icc, vis5e1)
|
|
DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f54,f56,f58,f60,f62,f48,f50,f52,f52,
|
|
,LDBLK(f0), ,,,,,STBLK,,,
|
|
,bcs,pn %icc, vis5e2)
|
|
DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f54,f56,f58,f60,f62,f48,f50,f52,f52,
|
|
,LDBLK(f16), ,,,,,STBLK,,,
|
|
,bcc,pt %icc, vis5)
|
|
vis5e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f54,f56,f58,f60,f62,f48,f50,f52,f32,
|
|
,SYNC, ,,,,,STBLK_XORASI(x5,x6),ST(f48,64),ST(f50,72),
|
|
,add %dst, 80, %dst; add %len, 192 - 3*8, %len; ba,pt %icc, e2)
|
|
vis5e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f54,f56,f58,f60,f62,f48,f50,f52,f0,
|
|
,SYNC, ,,,,,STBLK_XORASI(x5,x6),ST(f48,64),ST(f50,72),
|
|
,add %dst, 80, %dst; add %len, 192 - 3*8, %len; ba,pt %icc, e3)
|
|
vis5e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f54,f56,f58,f60,f62,f48,f50,f52,f16,
|
|
,SYNC, ,,,,,STBLK_XORASI(x5,x6),ST(f48,64),ST(f50,72),
|
|
,add %dst, 80, %dst; add %len, 192 - 3*8, %len; ba,pt %icc, e1)
|
|
.align 2048
|
|
vis6s: ldd [%src+0], %f12 /* Load Group */
|
|
ldd [%src+8], %f14 /* Load Group */
|
|
add %src, 16, %src /* IEU0 Group */
|
|
wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
|
|
ldda [%src] ASI_BLK_P, %f16 /* Load Group */
|
|
add %src, 64, %src /* IEU0 Group */
|
|
fmovd %f48, %f0 /* FPA Group */
|
|
fmuld %f32, %f32, %f2 /* FPM */
|
|
clr %x4 /* IEU0 */
|
|
faddd %f32, %f32, %f4 /* FPA Group */
|
|
fmuld %f32, %f32, %f6 /* FPM */
|
|
clr %x5 /* IEU0 */
|
|
faddd %f32, %f32, %f8 /* FPA Group */
|
|
fmuld %f32, %f32, %f10 /* FPM */
|
|
clr %x6 /* IEU0 */
|
|
fcmpgt32 %f32, %f12, %x7 /* FPM Group */
|
|
sub %dst, 64, %dst /* IEU0 */
|
|
fcmpgt32 %f32, %f14, %x8 /* FPM Group */
|
|
faligndata %f12, %f14, %f48 /* FPA */
|
|
fmovd %f14, %f50 /* FPA Group */
|
|
vis6: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f52,f54,f56,f58,f60,f62,f48,f50,f50,
|
|
,LDBLK(f32), ,,,,,,STBLK,,
|
|
,bcs,pn %icc, vis6e1)
|
|
DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f52,f54,f56,f58,f60,f62,f48,f50,f50,
|
|
,LDBLK(f0), ,,,,,,STBLK,,
|
|
,bcs,pn %icc, vis6e2)
|
|
DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f52,f54,f56,f58,f60,f62,f48,f50,f50,
|
|
,LDBLK(f16), ,,,,,,STBLK,,
|
|
,bcc,pt %icc, vis6)
|
|
vis6e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f52,f54,f56,f58,f60,f62,f48,f50,f32,
|
|
,SYNC, ,,,,,,STBLK_XORASI(x6,x7),ST(f48,64),
|
|
,add %dst, 72, %dst; add %len, 192 - 2*8, %len; ba,pt %icc, e2)
|
|
vis6e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f52,f54,f56,f58,f60,f62,f48,f50,f0,
|
|
,SYNC, ,,,,,,STBLK_XORASI(x6,x7),ST(f48,64),
|
|
,add %dst, 72, %dst; add %len, 192 - 2*8, %len; ba,pt %icc, e3)
|
|
vis6e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f52,f54,f56,f58,f60,f62,f48,f50,f16,
|
|
,SYNC, ,,,,,,STBLK_XORASI(x6,x7),ST(f48,64),
|
|
,add %dst, 72, %dst; add %len, 192 - 2*8, %len; ba,pt %icc, e1)
|
|
.align 2048
|
|
vis7s: ldd [%src+0], %f14 /* Load Group */
|
|
add %src, 8, %src /* IEU0 Group */
|
|
wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
|
|
ldda [%src] ASI_BLK_P, %f16 /* Load Group */
|
|
add %src, 64, %src /* IEU0 Group */
|
|
fmovd %f48, %f0 /* FPA Group */
|
|
fmuld %f32, %f32, %f2 /* FPM */
|
|
clr %x4 /* IEU0 */
|
|
faddd %f32, %f32, %f4 /* FPA Group */
|
|
fmuld %f32, %f32, %f6 /* FPM */
|
|
clr %x5 /* IEU0 */
|
|
faddd %f32, %f32, %f8 /* FPA Group */
|
|
fmuld %f32, %f32, %f10 /* FPM */
|
|
clr %x6 /* IEU0 */
|
|
faddd %f32, %f32, %f12 /* FPA Group */
|
|
clr %x7 /* IEU0 */
|
|
fcmpgt32 %f32, %f14, %x8 /* FPM Group */
|
|
sub %dst, 64, %dst /* IEU0 */
|
|
fmovd %f14, %f48 /* FPA */
|
|
vis7: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f50,f52,f54,f56,f58,f60,f62,f48,f48,
|
|
,LDBLK(f32), ,,,,,,,STBLK,
|
|
,bcs,pn %icc, vis7e1)
|
|
DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f50,f52,f54,f56,f58,f60,f62,f48,f48,
|
|
,LDBLK(f0), ,,,,,,,STBLK,
|
|
,bcs,pn %icc, vis7e2)
|
|
DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f50,f52,f54,f56,f58,f60,f62,f48,f48,
|
|
,LDBLK(f16), ,,,,,,,STBLK,
|
|
,bcc,pt %icc, vis7)
|
|
vis7e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
|
|
,f50,f52,f54,f56,f58,f60,f62,f48,f32,
|
|
,SYNC, ,,,,,,,STBLK_XORASI(x7,x8),
|
|
,add %dst, 64, %dst; add %len, 192 - 1*8, %len; ba,pt %icc, e2)
|
|
vis7e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
|
|
,f50,f52,f54,f56,f58,f60,f62,f48,f0,
|
|
,SYNC, ,,,,,,,STBLK_XORASI(x7,x8),
|
|
,add %dst, 64, %dst; add %len, 192 - 1*8, %len; ba,pt %icc, e3)
|
|
vis7e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
|
|
,f50,f52,f54,f56,f58,f60,f62,f48,f16,
|
|
,SYNC, ,,,,,,,STBLK_XORASI(x7,x8),
|
|
,add %dst, 64, %dst; add %len, 192 - 1*8, %len; ba,pt %icc, e1)
|
|
e1: END_THE_TRICK1( f0,f2,f4,f6,f8,f10,f12,f14,f16,f6)
|
|
e2: END_THE_TRICK1( f16,f18,f20,f22,f24,f26,f28,f30,f32,f6)
|
|
e3: END_THE_TRICK1( f32,f34,f36,f38,f40,f42,f44,f46,f0,f6)
|
|
ett: rd %gsr, %x3 /* LSU Group+4bubbles */
|
|
andcc %x3, 7, %x3 /* IEU1 Group */
|
|
add %dst, 8, %dst /* IEU0 */
|
|
bne,pn %icc, 1f /* CTI */
|
|
fzero %f10 /* FPA */
|
|
brz,a,pn %len, 2f /* CTI+IEU1 Group */
|
|
stda %f6, [%dst - 8] %asi /* Store */
|
|
1: cmp %len, 8 /* IEU1 */
|
|
blu,pn %icc, 3f /* CTI */
|
|
sub %src, 64, %src /* IEU0 Group */
|
|
1: ldd [%src], %f2 /* Load Group */
|
|
fpadd32 %f10, %f2, %f12 /* FPA Group+load stall*/
|
|
add %src, 8, %src /* IEU0 */
|
|
add %dst, 8, %dst /* IEU1 */
|
|
faligndata %f6, %f2, %f14 /* FPA Group */
|
|
fcmpgt32 %f10, %f12, %x5 /* FPM Group */
|
|
stda %f14, [%dst - 16] %asi /* Store */
|
|
fmovd %f2, %f6 /* FPA */
|
|
fmovd %f12, %f10 /* FPA Group */
|
|
sub %len, 8, %len /* IEU1 */
|
|
fzero %f16 /* FPA Group - FPU nop */
|
|
fzero %f18 /* FPA Group - FPU nop */
|
|
inc %x5 /* IEU0 */
|
|
srl %x5, 1, %x5 /* IEU0 Group (regdep) */
|
|
cmp %len, 8 /* IEU1 */
|
|
bgeu,pt %icc, 1b /* CTI */
|
|
add %x5, %sum, %sum /* IEU0 Group */
|
|
3: brz,a,pt %x3, 2f /* CTI+IEU1 */
|
|
stda %f6, [%dst - 8] %asi /* Store Group */
|
|
sta %f7, [%dst - 8] %asi /* Store Group */
|
|
sub %dst, 4, %dst /* IEU0 */
|
|
add %len, 4, %len /* IEU1 */
|
|
2:
|
|
#ifdef __KERNEL__
|
|
sub %sp, 8, %sp /* IEU0 Group */
|
|
#endif
|
|
END_THE_TRICK2( f48,f50,f52,f54,f56,f58,f60,f10,f12,f62)
|
|
membar #Sync /* LSU Group */
|
|
#ifdef __KERNEL__
|
|
VISExit
|
|
add %sp, 8, %sp /* IEU0 Group */
|
|
#endif
|
|
23: brnz,pn %len, 26f /* CTI+IEU1 Group */
|
|
24: sllx %sum, 32, %g1 /* IEU0 */
|
|
25: addcc %sum, %g1, %src /* IEU1 Group */
|
|
srlx %src, 32, %src /* IEU0 Group (regdep) */
|
|
bcs,a,pn %xcc, 1f /* CTI */
|
|
add %src, 1, %src /* IEU1 */
|
|
#ifndef __KERNEL__
|
|
1: retl /* CTI Group brk forced*/
|
|
srl %src, 0, %src /* IEU0 */
|
|
#else
|
|
1: retl /* CTI Group brk forced*/
|
|
ldx [%g6 + TI_TASK], %g4 /* Load */
|
|
#endif
|
|
26: andcc %len, 8, %g0 /* IEU1 Group */
|
|
be,pn %icc, 1f /* CTI */
|
|
lduw [%src], %o4 /* Load */
|
|
lduw [%src+4], %g2 /* Load Group */
|
|
add %src, 8, %src /* IEU0 */
|
|
add %dst, 8, %dst /* IEU1 */
|
|
sllx %o4, 32, %g5 /* IEU0 Group */
|
|
stwa %o4, [%dst - 8] %asi /* Store */
|
|
or %g5, %g2, %g5 /* IEU0 Group */
|
|
stwa %g2, [%dst - 4] %asi /* Store */
|
|
addcc %g5, %sum, %sum /* IEU1 Group */
|
|
bcs,a,pn %xcc, 1f /* CTI */
|
|
add %sum, 1, %sum /* IEU0 */
|
|
1: andcc %len, 4, %g0 /* IEU1 Group */
|
|
be,a,pn %icc, 1f /* CTI */
|
|
clr %g2 /* IEU0 */
|
|
lduw [%src], %g7 /* Load */
|
|
add %src, 4, %src /* IEU0 Group */
|
|
add %dst, 4, %dst /* IEU1 */
|
|
sllx %g7, 32, %g2 /* IEU0 Group */
|
|
stwa %g7, [%dst - 4] %asi /* Store */
|
|
1: andcc %len, 2, %g0 /* IEU1 */
|
|
be,a,pn %icc, 1f /* CTI */
|
|
clr %g3 /* IEU0 Group */
|
|
lduh [%src], %g7 /* Load */
|
|
add %src, 2, %src /* IEU1 */
|
|
add %dst, 2, %dst /* IEU0 Group */
|
|
sll %g7, 16, %g3 /* IEU0 Group */
|
|
stha %g7, [%dst - 2] %asi /* Store */
|
|
1: andcc %len, 1, %g0 /* IEU1 */
|
|
be,a,pn %icc, 1f /* CTI */
|
|
clr %o5 /* IEU0 Group */
|
|
ldub [%src], %g7 /* Load */
|
|
sll %g7, 8, %o5 /* IEU0 Group */
|
|
stba %g7, [%dst] %asi /* Store */
|
|
1: or %g2, %g3, %g3 /* IEU1 */
|
|
or %o5, %g3, %g3 /* IEU0 Group (regdep) */
|
|
addcc %g3, %sum, %sum /* IEU1 Group (regdep) */
|
|
bcs,a,pn %xcc, 1f /* CTI */
|
|
add %sum, 1, %sum /* IEU0 */
|
|
1: ba,pt %xcc, 25b /* CTI Group */
|
|
sllx %sum, 32, %g1 /* IEU0 */
|
|
|
|
#ifdef __KERNEL__
|
|
end:
|
|
|
|
.section __ex_table
|
|
.align 4
|
|
.word csum_partial_copy_user_vis, 0, end, cpc_handler
|
|
#endif
|