Skip to content

loongarch: fix DGEES/DGEESX lapack-test failure #6058

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
HecaiYuan wants to merge 1 commit into
base: develop
Choose a base branch
from
+80 −181
Open

loongarch: fix DGEES/DGEESX lapack-test failure #6058

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
261 changes: 80 additions & 181 deletions kernel/loongarch64/dgemv_t_lsx.S
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/

#define ASSEMBLER

#include "common.h"

/* Param */
#define M $r4
#define N $r5
#define A $r7
Expand All @@ -41,52 +39,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define BUFFER $r16
#define ALPHA $f0

#define YORIG $r18
#define T0 $r19
#define T1 $r20
#define AO3 $r12
#define AO4 $r13
#define I $r14
#define J $r15
#define AO1 $r23
#define AO2 $r24
#define AO $r23
#define XORG $r24
#define IX $r25
#define IY $r26
#define II $r27
#define T0 $r19
#define T1 $r20
#define T2 $r28
#define T3 $r29
#define T4 $r30

/* LSX vectors */
#define U0 $vr11
#define U1 $vr12
#define U2 $vr2
#define U3 $vr3
#define U4 $vr4
#define U5 $vr5
#define U6 $vr6
#define U7 $vr7
#define U8 $vr8
#define U9 $vr9
#define VALPHA $vr10

#define a1 $f3
#define a2 $f4
#define a3 $f5
#define a4 $f6
#define a5 $f7
#define a6 $f8
#define a7 $f9
#define a8 $f10

#define ACC $vr11
#define VX $vr12
#define VA $vr13
#define VT $vr14
#define FAC $f17
#define FAX $f18
#define FAY $f19

PROLOGUE

LDARG INCY, $sp, 0
LDARG BUFFER, $sp, 8

addi.d $sp, $sp, -80

SDARG $r23, $sp, 0
SDARG $r24, $sp, 8
SDARG $r25, $sp, 16
Expand All @@ -97,8 +76,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
SDARG $r30, $sp, 64
ST ALPHA, $sp, 72

vldrepl.d VALPHA, $sp, 72

slli.d LDA, LDA, BASE_SHIFT
slli.d INCX, INCX, BASE_SHIFT
slli.d INCY, INCY, BASE_SHIFT
Expand All @@ -108,160 +85,82 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

move J, $r0
move IY, $r0

move AO1, A //a_ptr1

srai.d T0, N, 2 //n/4
beq J, T0, .L04

.L01: /* j<n/4 */
vxor.v U0, U0, U0
vxor.v U7, U7, U7
add.d AO2, AO1, LDA
add.d AO3, AO2, LDA
add.d AO4, AO3, LDA

move IX, $r0
move I, $r0
move XORG, X
move AO, A
li.d T1, 8

.L01: /* for each column J */
vxor.v ACC, ACC, ACC
MTC FAC, $r0
move X, XORG
move II, $r0

beq $r0, M, .L03

.L02: /* i<m */
vldx U1, X, IX
fldx.d $f2, AO1, II
fldx.d $f3, AO2, II
fldx.d $f4, AO3, II
fldx.d $f5, AO4, II

vshuf4i.d U1, U1, 0x00
vextrins.d U2, U3, 0x10
vextrins.d U4, U5, 0x10

vfmadd.d U0, U2, U1, U0 //temp1,2
vfmadd.d U7, U4, U1, U7 //temp3,4

add.d IX, IX, INCX

addi.d II, II, 8
addi.d I, I, 1
blt I, M, .L02
move I, $r0
srli.d T2, M, 1
slli.d T2, T2, 1 /* T2 = M & ~1 */

bne INCX, T1, .L_SCALAR

.L02:
beq I, T2, .L03
vld VX, X, 0
vld VA, AO, 0
vfmadd.d ACC, VA, VX, ACC
addi.d II, II, 16
addi.d I, I, 2
addi.d X, X, 16
addi.d AO, AO, 16
b .L02

.L03:
move T1, IY
add.d T2, T1, INCY
add.d T3, T2, INCY
add.d T4, T3, INCY

fldx.d $f3, Y, T1
fldx.d $f4, Y, T2
fldx.d $f5, Y, T3
fldx.d $f6, Y, T4

vextrins.d U3, U4, 0x10
vextrins.d U5, U6, 0x10

vfmadd.d U3, VALPHA, U0, U3
vfmadd.d U5, VALPHA, U7, U5

vextrins.d U4, U3, 0x01
vextrins.d U6, U5, 0x01

fstx.d $f3, Y, T1
fstx.d $f4, Y, T2
fstx.d $f5, Y, T3
fstx.d $f6, Y, T4

slli.d T1, LDA, 2
add.d AO1, AO1, T1
add.d IY, T4, INCY

addi.d J, J, 1
blt J, T0, .L01

.L04: /* if(n&2) */
andi T0, N, 2
beq $r0, T0, .L07

vxor.v U0, U0, U0

add.d AO2, AO1, LDA

andi T3, M, 1
beq $r0, T3, .L05
fld.d FAY, AO, 0
fld.d FAX, X, 0
fmadd.d FAC, FAY, FAX, FAC
.L05:
/* dot = (lane0+lane1) + tail */
vextrins.d VT, ACC, 0x01
vfadd.d VT, ACC, VT
vpickve2gr.d T0, VT, 0
movgr2fr.d FAX, T0
fadd.d FAC, FAX, FAC
/* y += alpha*dot */
fldx.d FAY, Y, IY
fmadd.d FAY, ALPHA, FAC, FAY
fstx.d FAY, Y, IY
/* next column: AO = A + (J+1)*LDA */
addi.d T3, J, 1
mul.d T3, T3, LDA
add.d AO, A, T3
add.d IY, IY, INCY
addi.d J, J, 1
blt J, N, .L01
b .L999

.L_SCALAR:
move IX, $r0
move I, $r0
move II, $r0

beq $r0, M, .L06

.L05: /* i<m */
vldx U1, X, IX
fldx.d $f2, AO1, II
fldx.d $f3, AO2, II

vshuf4i.d U1, U1, 0x00
vextrins.d U2, U3, 0x10

vfmadd.d U0, U2, U1, U0 //temp1,2

add.d IX, IX, INCX

addi.d II, II, 8
addi.d I, I, 1
blt I, M, .L05

.L06:
move T1, IY
add.d T2, T1, INCY

fldx.d a1, Y, T1
fldx.d a2, Y, T2

vextrins.d U3, U4, 0x10

vfmadd.d U3, VALPHA, U0, U3

vextrins.d U4, U3, 0x01

fstx.d a1, Y, T1
fstx.d a2, Y, T2

slli.d T0, LDA, 1
add.d AO1, AO1, T0
add.d IY, T2, INCY

.L07: /* if(n&1) */
andi T0, N, 1
beq $r0, T0, .L999

MTC a1, $r0

move IX, $r0
move I, $r0
move II, $r0

beq $r0, M, .L09

.L08: /* i<m */
fldx.d a3, X, IX
fldx.d a4, AO1, II

fmadd.d a1, a4, a3, a1 //temp1

add.d IX, IX, INCX

addi.d II, II, 8
addi.d I, I, 1
blt I, M, .L08

.L09:
fldx.d a3, Y, IY

fmadd.d a3, ALPHA, a1, a3

fstx.d a3, Y, IY

add.d AO1, AO1, LDA
add.d IY, IY, INCY
beq I, M, .L07
fldx.d FAX, X, IX
fldx.d FAY, AO, II
fmadd.d FAC, FAY, FAX, FAC
add.d IX, IX, INCX
addi.d II, II, 8
addi.d I, I, 1
b .L06
.L07:
fldx.d FAY, Y, IY
fmadd.d FAY, ALPHA, FAC, FAY
fstx.d FAY, Y, IY
addi.d T3, J, 1
mul.d T3, T3, LDA
add.d AO, A, T3
add.d IY, IY, INCY
addi.d J, J, 1
blt J, N, .L01
b .L999

.L999:
LDARG $r23, $sp, 0
Expand Down
Loading