@@ -510,20 +510,33 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
510
510
case BPF_ALU | BPF_DIV | BPF_X : /* (u32) dst /= (u32) src */
511
511
case BPF_ALU | BPF_MOD | BPF_X : /* (u32) dst %= (u32) src */
512
512
if (BPF_OP (code ) == BPF_MOD ) {
513
- EMIT (PPC_RAW_DIVWU (tmp1_reg , dst_reg , src_reg ));
513
+ if (off )
514
+ EMIT (PPC_RAW_DIVW (tmp1_reg , dst_reg , src_reg ));
515
+ else
516
+ EMIT (PPC_RAW_DIVWU (tmp1_reg , dst_reg , src_reg ));
517
+
514
518
EMIT (PPC_RAW_MULW (tmp1_reg , src_reg , tmp1_reg ));
515
519
EMIT (PPC_RAW_SUB (dst_reg , dst_reg , tmp1_reg ));
516
520
} else
517
- EMIT (PPC_RAW_DIVWU (dst_reg , dst_reg , src_reg ));
521
+ if (off )
522
+ EMIT (PPC_RAW_DIVW (dst_reg , dst_reg , src_reg ));
523
+ else
524
+ EMIT (PPC_RAW_DIVWU (dst_reg , dst_reg , src_reg ));
518
525
goto bpf_alu32_trunc ;
519
526
case BPF_ALU64 | BPF_DIV | BPF_X : /* dst /= src */
520
527
case BPF_ALU64 | BPF_MOD | BPF_X : /* dst %= src */
521
528
if (BPF_OP (code ) == BPF_MOD ) {
522
- EMIT (PPC_RAW_DIVDU (tmp1_reg , dst_reg , src_reg ));
529
+ if (off )
530
+ EMIT (PPC_RAW_DIVD (tmp1_reg , dst_reg , src_reg ));
531
+ else
532
+ EMIT (PPC_RAW_DIVDU (tmp1_reg , dst_reg , src_reg ));
523
533
EMIT (PPC_RAW_MULD (tmp1_reg , src_reg , tmp1_reg ));
524
534
EMIT (PPC_RAW_SUB (dst_reg , dst_reg , tmp1_reg ));
525
535
} else
526
- EMIT (PPC_RAW_DIVDU (dst_reg , dst_reg , src_reg ));
536
+ if (off )
537
+ EMIT (PPC_RAW_DIVD (dst_reg , dst_reg , src_reg ));
538
+ else
539
+ EMIT (PPC_RAW_DIVDU (dst_reg , dst_reg , src_reg ));
527
540
break ;
528
541
case BPF_ALU | BPF_MOD | BPF_K : /* (u32) dst %= (u32) imm */
529
542
case BPF_ALU | BPF_DIV | BPF_K : /* (u32) dst /= (u32) imm */
@@ -544,19 +557,31 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
544
557
switch (BPF_CLASS (code )) {
545
558
case BPF_ALU :
546
559
if (BPF_OP (code ) == BPF_MOD ) {
547
- EMIT (PPC_RAW_DIVWU (tmp2_reg , dst_reg , tmp1_reg ));
560
+ if (off )
561
+ EMIT (PPC_RAW_DIVW (tmp2_reg , dst_reg , tmp1_reg ));
562
+ else
563
+ EMIT (PPC_RAW_DIVWU (tmp2_reg , dst_reg , tmp1_reg ));
548
564
EMIT (PPC_RAW_MULW (tmp1_reg , tmp1_reg , tmp2_reg ));
549
565
EMIT (PPC_RAW_SUB (dst_reg , dst_reg , tmp1_reg ));
550
566
} else
551
- EMIT (PPC_RAW_DIVWU (dst_reg , dst_reg , tmp1_reg ));
567
+ if (off )
568
+ EMIT (PPC_RAW_DIVW (dst_reg , dst_reg , tmp1_reg ));
569
+ else
570
+ EMIT (PPC_RAW_DIVWU (dst_reg , dst_reg , tmp1_reg ));
552
571
break ;
553
572
case BPF_ALU64 :
554
573
if (BPF_OP (code ) == BPF_MOD ) {
555
- EMIT (PPC_RAW_DIVDU (tmp2_reg , dst_reg , tmp1_reg ));
574
+ if (off )
575
+ EMIT (PPC_RAW_DIVD (tmp2_reg , dst_reg , tmp1_reg ));
576
+ else
577
+ EMIT (PPC_RAW_DIVDU (tmp2_reg , dst_reg , tmp1_reg ));
556
578
EMIT (PPC_RAW_MULD (tmp1_reg , tmp1_reg , tmp2_reg ));
557
579
EMIT (PPC_RAW_SUB (dst_reg , dst_reg , tmp1_reg ));
558
580
} else
559
- EMIT (PPC_RAW_DIVDU (dst_reg , dst_reg , tmp1_reg ));
581
+ if (off )
582
+ EMIT (PPC_RAW_DIVD (dst_reg , dst_reg , tmp1_reg ));
583
+ else
584
+ EMIT (PPC_RAW_DIVDU (dst_reg , dst_reg , tmp1_reg ));
560
585
break ;
561
586
}
562
587
goto bpf_alu32_trunc ;
@@ -676,8 +701,14 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
676
701
/* special mov32 for zext */
677
702
EMIT (PPC_RAW_RLWINM (dst_reg , dst_reg , 0 , 0 , 31 ));
678
703
break ;
679
- }
680
- EMIT (PPC_RAW_MR (dst_reg , src_reg ));
704
+ } else if (off == 8 ) {
705
+ EMIT (PPC_RAW_EXTSB (dst_reg , src_reg ));
706
+ } else if (off == 16 ) {
707
+ EMIT (PPC_RAW_EXTSH (dst_reg , src_reg ));
708
+ } else if (off == 32 ) {
709
+ EMIT (PPC_RAW_EXTSW (dst_reg , src_reg ));
710
+ } else if (dst_reg != src_reg )
711
+ EMIT (PPC_RAW_MR (dst_reg , src_reg ));
681
712
goto bpf_alu32_trunc ;
682
713
case BPF_ALU | BPF_MOV | BPF_K : /* (u32) dst = imm */
683
714
case BPF_ALU64 | BPF_MOV | BPF_K : /* dst = (s64) imm */
@@ -699,11 +730,12 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
699
730
*/
700
731
case BPF_ALU | BPF_END | BPF_FROM_LE :
701
732
case BPF_ALU | BPF_END | BPF_FROM_BE :
733
+ case BPF_ALU64 | BPF_END | BPF_FROM_LE :
702
734
#ifdef __BIG_ENDIAN__
703
735
if (BPF_SRC (code ) == BPF_FROM_BE )
704
736
goto emit_clear ;
705
737
#else /* !__BIG_ENDIAN__ */
706
- if (BPF_SRC (code ) == BPF_FROM_LE )
738
+ if (BPF_CLASS ( code ) == BPF_ALU && BPF_SRC (code ) == BPF_FROM_LE )
707
739
goto emit_clear ;
708
740
#endif
709
741
switch (imm ) {
@@ -936,13 +968,19 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
936
968
*/
937
969
/* dst = *(u8 *)(ul) (src + off) */
938
970
case BPF_LDX | BPF_MEM | BPF_B :
971
+ case BPF_LDX | BPF_MEMSX | BPF_B :
939
972
case BPF_LDX | BPF_PROBE_MEM | BPF_B :
973
+ case BPF_LDX | BPF_PROBE_MEMSX | BPF_B :
940
974
/* dst = *(u16 *)(ul) (src + off) */
941
975
case BPF_LDX | BPF_MEM | BPF_H :
976
+ case BPF_LDX | BPF_MEMSX | BPF_H :
942
977
case BPF_LDX | BPF_PROBE_MEM | BPF_H :
978
+ case BPF_LDX | BPF_PROBE_MEMSX | BPF_H :
943
979
/* dst = *(u32 *)(ul) (src + off) */
944
980
case BPF_LDX | BPF_MEM | BPF_W :
981
+ case BPF_LDX | BPF_MEMSX | BPF_W :
945
982
case BPF_LDX | BPF_PROBE_MEM | BPF_W :
983
+ case BPF_LDX | BPF_PROBE_MEMSX | BPF_W :
946
984
/* dst = *(u64 *)(ul) (src + off) */
947
985
case BPF_LDX | BPF_MEM | BPF_DW :
948
986
case BPF_LDX | BPF_PROBE_MEM | BPF_DW :
@@ -952,7 +990,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
952
990
* load only if addr is kernel address (see is_kernel_addr()), otherwise
953
991
* set dst_reg=0 and move on.
954
992
*/
955
- if (BPF_MODE (code ) == BPF_PROBE_MEM ) {
993
+ if (BPF_MODE (code ) == BPF_PROBE_MEM || BPF_MODE ( code ) == BPF_PROBE_MEMSX ) {
956
994
EMIT (PPC_RAW_ADDI (tmp1_reg , src_reg , off ));
957
995
if (IS_ENABLED (CONFIG_PPC_BOOK3E_64 ))
958
996
PPC_LI64 (tmp2_reg , 0x8000000000000000ul );
@@ -965,30 +1003,47 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
965
1003
* Check if 'off' is word aligned for BPF_DW, because
966
1004
* we might generate two instructions.
967
1005
*/
968
- if (BPF_SIZE (code ) == BPF_DW && (off & 3 ))
1006
+ if ((BPF_SIZE (code ) == BPF_DW ||
1007
+ (BPF_SIZE (code ) == BPF_B && BPF_MODE (code ) == BPF_PROBE_MEMSX )) &&
1008
+ (off & 3 ))
969
1009
PPC_JMP ((ctx -> idx + 3 ) * 4 );
970
1010
else
971
1011
PPC_JMP ((ctx -> idx + 2 ) * 4 );
972
1012
}
973
1013
974
- switch (size ) {
975
- case BPF_B :
976
- EMIT (PPC_RAW_LBZ (dst_reg , src_reg , off ));
977
- break ;
978
- case BPF_H :
979
- EMIT (PPC_RAW_LHZ (dst_reg , src_reg , off ));
980
- break ;
981
- case BPF_W :
982
- EMIT (PPC_RAW_LWZ (dst_reg , src_reg , off ));
983
- break ;
984
- case BPF_DW :
985
- if (off % 4 ) {
986
- EMIT (PPC_RAW_LI (tmp1_reg , off ));
987
- EMIT (PPC_RAW_LDX (dst_reg , src_reg , tmp1_reg ));
988
- } else {
989
- EMIT (PPC_RAW_LD (dst_reg , src_reg , off ));
1014
+ if (BPF_MODE (code ) == BPF_MEMSX || BPF_MODE (code ) == BPF_PROBE_MEMSX ) {
1015
+ switch (size ) {
1016
+ case BPF_B :
1017
+ EMIT (PPC_RAW_LBZ (dst_reg , src_reg , off ));
1018
+ EMIT (PPC_RAW_EXTSB (dst_reg , dst_reg ));
1019
+ break ;
1020
+ case BPF_H :
1021
+ EMIT (PPC_RAW_LHA (dst_reg , src_reg , off ));
1022
+ break ;
1023
+ case BPF_W :
1024
+ EMIT (PPC_RAW_LWA (dst_reg , src_reg , off ));
1025
+ break ;
1026
+ }
1027
+ } else {
1028
+ switch (size ) {
1029
+ case BPF_B :
1030
+ EMIT (PPC_RAW_LBZ (dst_reg , src_reg , off ));
1031
+ break ;
1032
+ case BPF_H :
1033
+ EMIT (PPC_RAW_LHZ (dst_reg , src_reg , off ));
1034
+ break ;
1035
+ case BPF_W :
1036
+ EMIT (PPC_RAW_LWZ (dst_reg , src_reg , off ));
1037
+ break ;
1038
+ case BPF_DW :
1039
+ if (off % 4 ) {
1040
+ EMIT (PPC_RAW_LI (tmp1_reg , off ));
1041
+ EMIT (PPC_RAW_LDX (dst_reg , src_reg , tmp1_reg ));
1042
+ } else {
1043
+ EMIT (PPC_RAW_LD (dst_reg , src_reg , off ));
1044
+ }
1045
+ break ;
990
1046
}
991
- break ;
992
1047
}
993
1048
994
1049
if (size != BPF_DW && insn_is_zext (& insn [i + 1 ]))
@@ -1065,6 +1120,9 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
1065
1120
case BPF_JMP | BPF_JA :
1066
1121
PPC_JMP (addrs [i + 1 + off ]);
1067
1122
break ;
1123
+ case BPF_JMP32 | BPF_JA :
1124
+ PPC_JMP (addrs [i + 1 + imm ]);
1125
+ break ;
1068
1126
1069
1127
case BPF_JMP | BPF_JGT | BPF_K :
1070
1128
case BPF_JMP | BPF_JGT | BPF_X :
0 commit comments