// Code generated by command: go run index_pair_asm.go -pkg mem -out ../mem/index_pair_amd64.s -stubs ../mem/index_pair_amd64.go. DO NOT EDIT.

//go:build !purego

#include "textflag.h"

// func indexPair1(b []byte) int
// Requires: AVX, AVX2, BMI
TEXT ·indexPair1(SB), NOSPLIT, $0-32
	MOVQ b_base+0(FP), AX
	MOVQ b_len+8(FP), CX
	MOVQ AX, DX
	CMPQ CX, $0x00
	JLE  fail
	SUBQ $0x01, CX
	BTL  $0x08, github·com∕segmentio∕asm∕cpu·X86+0(SB)
	JCS  avx2

tail:
	CMPQ CX, $0x00
	JE   fail

generic:
	MOVB (AX), BL
	CMPB BL, 1(AX)
	JE   done
	ADDQ $0x01, AX
	SUBQ $0x01, CX
	CMPQ CX, $0x00
	JA   generic

fail:
	MOVQ $0xffffffffffffffff, AX
	MOVQ AX, ret+24(FP)
	RET

done:
	SUBQ DX, AX
	MOVQ AX, ret+24(FP)
	RET

avx2:
	CMPQ CX, $0x00000101
	JB   avx2_tail128
	XORQ BX, BX
	XORQ SI, SI
	XORQ DI, DI
	XORQ R8, R8
	XORQ R9, R9
	XORQ R10, R10
	XORQ R11, R11
	XORQ R12, R12

avx2_loop256:
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   128(AX), Y8
	VMOVDQU   160(AX), Y10
	VMOVDQU   192(AX), Y12
	VMOVDQU   224(AX), Y14
	VMOVDQU   1(AX), Y1
	VMOVDQU   33(AX), Y3
	VMOVDQU   65(AX), Y5
	VMOVDQU   97(AX), Y7
	VMOVDQU   129(AX), Y9
	VMOVDQU   161(AX), Y11
	VMOVDQU   193(AX), Y13
	VMOVDQU   225(AX), Y15
	VPCMPEQB  Y0, Y1, Y1
	VPCMPEQB  Y2, Y3, Y3
	VPCMPEQB  Y4, Y5, Y5
	VPCMPEQB  Y6, Y7, Y7
	VPCMPEQB  Y8, Y9, Y9
	VPCMPEQB  Y10, Y11, Y11
	VPCMPEQB  Y12, Y13, Y13
	VPCMPEQB  Y14, Y15, Y15
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y3, SI
	VPMOVMSKB Y5, DI
	VPMOVMSKB Y7, R8
	VPMOVMSKB Y9, R9
	VPMOVMSKB Y11, R10
	VPMOVMSKB Y13, R11
	VPMOVMSKB Y15, R12
	XORQ      R13, R13
	ORQ       BX, R13
	ORQ       SI, R13
	ORQ       DI, R13
	ORQ       R8, R13
	ORQ       R9, R13
	ORQ       R10, R13
	ORQ       R11, R13
	ORQ       R12, R13
	CMPQ      R13, $0x00
	JNE       avx2_done
	ADDQ      $0x00000100, AX
	SUBQ      $0x00000100, CX
	CMPQ      CX, $0x00000101
	JAE       avx2_loop256

avx2_tail128:
	CMPQ      CX, $0x81
	JB        avx2_tail64
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   1(AX), Y1
	VMOVDQU   33(AX), Y3
	VMOVDQU   65(AX), Y5
	VMOVDQU   97(AX), Y7
	VPCMPEQB  Y0, Y1, Y1
	VPCMPEQB  Y2, Y3, Y3
	VPCMPEQB  Y4, Y5, Y5
	VPCMPEQB  Y6, Y7, Y7
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y3, SI
	VPMOVMSKB Y5, DI
	VPMOVMSKB Y7, R8
	XORQ      R13, R13
	ORQ       BX, R13
	ORQ       SI, R13
	ORQ       DI, R13
	ORQ       R8, R13
	CMPQ      R13, $0x00
	JNE       avx2_done
	ADDQ      $0x00000080, AX
	SUBQ      $0x00000080, CX

avx2_tail64:
	CMPQ      CX, $0x41
	JB        avx2_tail32
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   1(AX), Y1
	VMOVDQU   33(AX), Y3
	VPCMPEQB  Y0, Y1, Y1
	VPCMPEQB  Y2, Y3, Y3
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y3, SI
	XORQ      R13, R13
	ORQ       BX, R13
	ORQ       SI, R13
	CMPQ      R13, $0x00
	JNE       avx2_done
	ADDQ      $0x00000040, AX
	SUBQ      $0x00000040, CX

avx2_tail32:
	CMPQ      CX, $0x21
	JB        avx2_tail16
	VMOVDQU   (AX), Y0
	VMOVDQU   1(AX), Y1
	VPCMPEQB  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	CMPQ      BX, $0x00
	JNE       avx2_done
	ADDQ      $0x00000020, AX
	SUBQ      $0x00000020, CX

avx2_tail16:
	CMPQ      CX, $0x11
	JB        avx2_tail
	VMOVDQU   (AX), X0
	VMOVDQU   1(AX), X1
	VPCMPEQB  X0, X1, X1
	VPMOVMSKB X1, BX
	CMPQ      BX, $0x00
	JNE       avx2_done
	ADDQ      $0x10, AX
	SUBQ      $0x10, CX

avx2_tail:
	VZEROUPPER
	JMP tail

avx2_done:
	VZEROUPPER
	CMPQ BX, $0x00
	JNE  avx2_done0
	CMPQ SI, $0x00
	JNE  avx2_done1
	CMPQ DI, $0x00
	JNE  avx2_done2
	CMPQ R8, $0x00
	JNE  avx2_done3
	CMPQ R9, $0x00
	JNE  avx2_done4
	CMPQ R10, $0x00
	JNE  avx2_done5
	CMPQ R11, $0x00
	JNE  avx2_done6
	CMPQ R12, $0x00
	JNE  avx2_done7

avx2_done0:
	TZCNTQ BX, BX
	ADDQ   BX, AX
	SUBQ   BX, CX
	JMP    done

avx2_done1:
	ADDQ   $0x00000020, AX
	SUBQ   $0x00000020, CX
	TZCNTQ SI, SI
	ADDQ   SI, AX
	SUBQ   SI, CX
	JMP    done

avx2_done2:
	ADDQ   $0x00000040, AX
	SUBQ   $0x00000040, CX
	TZCNTQ DI, DI
	ADDQ   DI, AX
	SUBQ   DI, CX
	JMP    done

avx2_done3:
	ADDQ   $0x00000060, AX
	SUBQ   $0x00000060, CX
	TZCNTQ R8, R8
	ADDQ   R8, AX
	SUBQ   R8, CX
	JMP    done

avx2_done4:
	ADDQ   $0x00000080, AX
	SUBQ   $0x00000080, CX
	TZCNTQ R9, R9
	ADDQ   R9, AX
	SUBQ   R9, CX
	JMP    done

avx2_done5:
	ADDQ   $0x000000a0, AX
	SUBQ   $0x000000a0, CX
	TZCNTQ R10, R10
	ADDQ   R10, AX
	SUBQ   R10, CX
	JMP    done

avx2_done6:
	ADDQ   $0x000000c0, AX
	SUBQ   $0x000000c0, CX
	TZCNTQ R11, R11
	ADDQ   R11, AX
	SUBQ   R11, CX
	JMP    done

avx2_done7:
	ADDQ   $0x000000e0, AX
	SUBQ   $0x000000e0, CX
	TZCNTQ R12, R12
	ADDQ   R12, AX
	SUBQ   R12, CX
	JMP    done

// func indexPair2(b []byte) int
// Requires: AVX, AVX2, BMI
TEXT ·indexPair2(SB), NOSPLIT, $0-32
	MOVQ b_base+0(FP), AX
	MOVQ b_len+8(FP), CX
	MOVQ AX, DX
	CMPQ CX, $0x00
	JLE  fail
	SUBQ $0x02, CX
	BTL  $0x08, github·com∕segmentio∕asm∕cpu·X86+0(SB)
	JCS  avx2

tail:
	CMPQ CX, $0x00
	JE   fail

generic:
	MOVW (AX), BX
	CMPW BX, 2(AX)
	JE   done
	ADDQ $0x02, AX
	SUBQ $0x02, CX
	CMPQ CX, $0x00
	JA   generic

fail:
	MOVQ $0xffffffffffffffff, AX
	MOVQ AX, ret+24(FP)
	RET

done:
	SUBQ DX, AX
	MOVQ AX, ret+24(FP)
	RET

avx2:
	CMPQ CX, $0x00000102
	JB   avx2_tail128
	XORQ BX, BX
	XORQ SI, SI
	XORQ DI, DI
	XORQ R8, R8
	XORQ R9, R9
	XORQ R10, R10
	XORQ R11, R11
	XORQ R12, R12

avx2_loop256:
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   128(AX), Y8
	VMOVDQU   160(AX), Y10
	VMOVDQU   192(AX), Y12
	VMOVDQU   224(AX), Y14
	VMOVDQU   2(AX), Y1
	VMOVDQU   34(AX), Y3
	VMOVDQU   66(AX), Y5
	VMOVDQU   98(AX), Y7
	VMOVDQU   130(AX), Y9
	VMOVDQU   162(AX), Y11
	VMOVDQU   194(AX), Y13
	VMOVDQU   226(AX), Y15
	VPCMPEQW  Y0, Y1, Y1
	VPCMPEQW  Y2, Y3, Y3
	VPCMPEQW  Y4, Y5, Y5
	VPCMPEQW  Y6, Y7, Y7
	VPCMPEQW  Y8, Y9, Y9
	VPCMPEQW  Y10, Y11, Y11
	VPCMPEQW  Y12, Y13, Y13
	VPCMPEQW  Y14, Y15, Y15
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y3, SI
	VPMOVMSKB Y5, DI
	VPMOVMSKB Y7, R8
	VPMOVMSKB Y9, R9
	VPMOVMSKB Y11, R10
	VPMOVMSKB Y13, R11
	VPMOVMSKB Y15, R12
	XORQ      R13, R13
	ORQ       BX, R13
	ORQ       SI, R13
	ORQ       DI, R13
	ORQ       R8, R13
	ORQ       R9, R13
	ORQ       R10, R13
	ORQ       R11, R13
	ORQ       R12, R13
	CMPQ      R13, $0x00
	JNE       avx2_done
	ADDQ      $0x00000100, AX
	SUBQ      $0x00000100, CX
	CMPQ      CX, $0x00000102
	JAE       avx2_loop256

avx2_tail128:
	CMPQ      CX, $0x82
	JB        avx2_tail64
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   2(AX), Y1
	VMOVDQU   34(AX), Y3
	VMOVDQU   66(AX), Y5
	VMOVDQU   98(AX), Y7
	VPCMPEQW  Y0, Y1, Y1
	VPCMPEQW  Y2, Y3, Y3
	VPCMPEQW  Y4, Y5, Y5
	VPCMPEQW  Y6, Y7, Y7
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y3, SI
	VPMOVMSKB Y5, DI
	VPMOVMSKB Y7, R8
	XORQ      R13, R13
	ORQ       BX, R13
	ORQ       SI, R13
	ORQ       DI, R13
	ORQ       R8, R13
	CMPQ      R13, $0x00
	JNE       avx2_done
	ADDQ      $0x00000080, AX
	SUBQ      $0x00000080, CX

avx2_tail64:
	CMPQ      CX, $0x42
	JB        avx2_tail32
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   2(AX), Y1
	VMOVDQU   34(AX), Y3
	VPCMPEQW  Y0, Y1, Y1
	VPCMPEQW  Y2, Y3, Y3
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y3, SI
	XORQ      R13, R13
	ORQ       BX, R13
	ORQ       SI, R13
	CMPQ      R13, $0x00
	JNE       avx2_done
	ADDQ      $0x00000040, AX
	SUBQ      $0x00000040, CX

avx2_tail32:
	CMPQ      CX, $0x22
	JB        avx2_tail16
	VMOVDQU   (AX), Y0
	VMOVDQU   2(AX), Y1
	VPCMPEQW  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	CMPQ      BX, $0x00
	JNE       avx2_done
	ADDQ      $0x00000020, AX
	SUBQ      $0x00000020, CX

avx2_tail16:
	CMPQ      CX, $0x12
	JB        avx2_tail
	VMOVDQU   (AX), X0
	VMOVDQU   2(AX), X1
	VPCMPEQW  X0, X1, X1
	VPMOVMSKB X1, BX
	CMPQ      BX, $0x00
	JNE       avx2_done
	ADDQ      $0x10, AX
	SUBQ      $0x10, CX

avx2_tail:
	VZEROUPPER
	JMP tail

avx2_done:
	VZEROUPPER
	CMPQ BX, $0x00
	JNE  avx2_done0
	CMPQ SI, $0x00
	JNE  avx2_done1
	CMPQ DI, $0x00
	JNE  avx2_done2
	CMPQ R8, $0x00
	JNE  avx2_done3
	CMPQ R9, $0x00
	JNE  avx2_done4
	CMPQ R10, $0x00
	JNE  avx2_done5
	CMPQ R11, $0x00
	JNE  avx2_done6
	CMPQ R12, $0x00
	JNE  avx2_done7

avx2_done0:
	TZCNTQ BX, BX
	ADDQ   BX, AX
	SUBQ   BX, CX
	JMP    done

avx2_done1:
	ADDQ   $0x00000020, AX
	SUBQ   $0x00000020, CX
	TZCNTQ SI, SI
	ADDQ   SI, AX
	SUBQ   SI, CX
	JMP    done

avx2_done2:
	ADDQ   $0x00000040, AX
	SUBQ   $0x00000040, CX
	TZCNTQ DI, DI
	ADDQ   DI, AX
	SUBQ   DI, CX
	JMP    done

avx2_done3:
	ADDQ   $0x00000060, AX
	SUBQ   $0x00000060, CX
	TZCNTQ R8, R8
	ADDQ   R8, AX
	SUBQ   R8, CX
	JMP    done

avx2_done4:
	ADDQ   $0x00000080, AX
	SUBQ   $0x00000080, CX
	TZCNTQ R9, R9
	ADDQ   R9, AX
	SUBQ   R9, CX
	JMP    done

avx2_done5:
	ADDQ   $0x000000a0, AX
	SUBQ   $0x000000a0, CX
	TZCNTQ R10, R10
	ADDQ   R10, AX
	SUBQ   R10, CX
	JMP    done

avx2_done6:
	ADDQ   $0x000000c0, AX
	SUBQ   $0x000000c0, CX
	TZCNTQ R11, R11
	ADDQ   R11, AX
	SUBQ   R11, CX
	JMP    done

avx2_done7:
	ADDQ   $0x000000e0, AX
	SUBQ   $0x000000e0, CX
	TZCNTQ R12, R12
	ADDQ   R12, AX
	SUBQ   R12, CX
	JMP    done

// func indexPair4(b []byte) int
// Requires: AVX, AVX2, BMI
TEXT ·indexPair4(SB), NOSPLIT, $0-32
	MOVQ b_base+0(FP), AX
	MOVQ b_len+8(FP), CX
	MOVQ AX, DX
	CMPQ CX, $0x00
	JLE  fail
	SUBQ $0x04, CX
	BTL  $0x08, github·com∕segmentio∕asm∕cpu·X86+0(SB)
	JCS  avx2

tail:
	CMPQ CX, $0x00
	JE   fail

generic:
	MOVL (AX), BX
	CMPL BX, 4(AX)
	JE   done
	ADDQ $0x04, AX
	SUBQ $0x04, CX
	CMPQ CX, $0x00
	JA   generic

fail:
	MOVQ $0xffffffffffffffff, AX
	MOVQ AX, ret+24(FP)
	RET

done:
	SUBQ DX, AX
	MOVQ AX, ret+24(FP)
	RET

avx2:
	CMPQ CX, $0x00000104
	JB   avx2_tail128
	XORQ BX, BX
	XORQ SI, SI
	XORQ DI, DI
	XORQ R8, R8
	XORQ R9, R9
	XORQ R10, R10
	XORQ R11, R11
	XORQ R12, R12

avx2_loop256:
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   128(AX), Y8
	VMOVDQU   160(AX), Y10
	VMOVDQU   192(AX), Y12
	VMOVDQU   224(AX), Y14
	VMOVDQU   4(AX), Y1
	VMOVDQU   36(AX), Y3
	VMOVDQU   68(AX), Y5
	VMOVDQU   100(AX), Y7
	VMOVDQU   132(AX), Y9
	VMOVDQU   164(AX), Y11
	VMOVDQU   196(AX), Y13
	VMOVDQU   228(AX), Y15
	VPCMPEQD  Y0, Y1, Y1
	VPCMPEQD  Y2, Y3, Y3
	VPCMPEQD  Y4, Y5, Y5
	VPCMPEQD  Y6, Y7, Y7
	VPCMPEQD  Y8, Y9, Y9
	VPCMPEQD  Y10, Y11, Y11
	VPCMPEQD  Y12, Y13, Y13
	VPCMPEQD  Y14, Y15, Y15
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y3, SI
	VPMOVMSKB Y5, DI
	VPMOVMSKB Y7, R8
	VPMOVMSKB Y9, R9
	VPMOVMSKB Y11, R10
	VPMOVMSKB Y13, R11
	VPMOVMSKB Y15, R12
	XORQ      R13, R13
	ORQ       BX, R13
	ORQ       SI, R13
	ORQ       DI, R13
	ORQ       R8, R13
	ORQ       R9, R13
	ORQ       R10, R13
	ORQ       R11, R13
	ORQ       R12, R13
	CMPQ      R13, $0x00
	JNE       avx2_done
	ADDQ      $0x00000100, AX
	SUBQ      $0x00000100, CX
	CMPQ      CX, $0x00000104
	JAE       avx2_loop256

avx2_tail128:
	CMPQ      CX, $0x84
	JB        avx2_tail64
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   4(AX), Y1
	VMOVDQU   36(AX), Y3
	VMOVDQU   68(AX), Y5
	VMOVDQU   100(AX), Y7
	VPCMPEQD  Y0, Y1, Y1
	VPCMPEQD  Y2, Y3, Y3
	VPCMPEQD  Y4, Y5, Y5
	VPCMPEQD  Y6, Y7, Y7
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y3, SI
	VPMOVMSKB Y5, DI
	VPMOVMSKB Y7, R8
	XORQ      R13, R13
	ORQ       BX, R13
	ORQ       SI, R13
	ORQ       DI, R13
	ORQ       R8, R13
	CMPQ      R13, $0x00
	JNE       avx2_done
	ADDQ      $0x00000080, AX
	SUBQ      $0x00000080, CX

avx2_tail64:
	CMPQ      CX, $0x44
	JB        avx2_tail32
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   4(AX), Y1
	VMOVDQU   36(AX), Y3
	VPCMPEQD  Y0, Y1, Y1
	VPCMPEQD  Y2, Y3, Y3
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y3, SI
	XORQ      R13, R13
	ORQ       BX, R13
	ORQ       SI, R13
	CMPQ      R13, $0x00
	JNE       avx2_done
	ADDQ      $0x00000040, AX
	SUBQ      $0x00000040, CX

avx2_tail32:
	CMPQ      CX, $0x24
	JB        avx2_tail16
	VMOVDQU   (AX), Y0
	VMOVDQU   4(AX), Y1
	VPCMPEQD  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	CMPQ      BX, $0x00
	JNE       avx2_done
	ADDQ      $0x00000020, AX
	SUBQ      $0x00000020, CX

avx2_tail16:
	CMPQ      CX, $0x14
	JB        avx2_tail
	VMOVDQU   (AX), X0
	VMOVDQU   4(AX), X1
	VPCMPEQD  X0, X1, X1
	VPMOVMSKB X1, BX
	CMPQ      BX, $0x00
	JNE       avx2_done
	ADDQ      $0x10, AX
	SUBQ      $0x10, CX

avx2_tail:
	VZEROUPPER
	JMP tail

avx2_done:
	VZEROUPPER
	CMPQ BX, $0x00
	JNE  avx2_done0
	CMPQ SI, $0x00
	JNE  avx2_done1
	CMPQ DI, $0x00
	JNE  avx2_done2
	CMPQ R8, $0x00
	JNE  avx2_done3
	CMPQ R9, $0x00
	JNE  avx2_done4
	CMPQ R10, $0x00
	JNE  avx2_done5
	CMPQ R11, $0x00
	JNE  avx2_done6
	CMPQ R12, $0x00
	JNE  avx2_done7

avx2_done0:
	TZCNTQ BX, BX
	ADDQ   BX, AX
	SUBQ   BX, CX
	JMP    done

avx2_done1:
	ADDQ   $0x00000020, AX
	SUBQ   $0x00000020, CX
	TZCNTQ SI, SI
	ADDQ   SI, AX
	SUBQ   SI, CX
	JMP    done

avx2_done2:
	ADDQ   $0x00000040, AX
	SUBQ   $0x00000040, CX
	TZCNTQ DI, DI
	ADDQ   DI, AX
	SUBQ   DI, CX
	JMP    done

avx2_done3:
	ADDQ   $0x00000060, AX
	SUBQ   $0x00000060, CX
	TZCNTQ R8, R8
	ADDQ   R8, AX
	SUBQ   R8, CX
	JMP    done

avx2_done4:
	ADDQ   $0x00000080, AX
	SUBQ   $0x00000080, CX
	TZCNTQ R9, R9
	ADDQ   R9, AX
	SUBQ   R9, CX
	JMP    done

avx2_done5:
	ADDQ   $0x000000a0, AX
	SUBQ   $0x000000a0, CX
	TZCNTQ R10, R10
	ADDQ   R10, AX
	SUBQ   R10, CX
	JMP    done

avx2_done6:
	ADDQ   $0x000000c0, AX
	SUBQ   $0x000000c0, CX
	TZCNTQ R11, R11
	ADDQ   R11, AX
	SUBQ   R11, CX
	JMP    done

avx2_done7:
	ADDQ   $0x000000e0, AX
	SUBQ   $0x000000e0, CX
	TZCNTQ R12, R12
	ADDQ   R12, AX
	SUBQ   R12, CX
	JMP    done

// func indexPair8(b []byte) int
// Requires: AVX, AVX2, BMI
TEXT ·indexPair8(SB), NOSPLIT, $0-32
	MOVQ b_base+0(FP), AX
	MOVQ b_len+8(FP), CX
	MOVQ AX, DX
	CMPQ CX, $0x00
	JLE  fail
	SUBQ $0x08, CX
	BTL  $0x08, github·com∕segmentio∕asm∕cpu·X86+0(SB)
	JCS  avx2

tail:
	CMPQ CX, $0x00
	JE   fail

generic:
	MOVQ (AX), BX
	CMPQ BX, 8(AX)
	JE   done
	ADDQ $0x08, AX
	SUBQ $0x08, CX
	CMPQ CX, $0x00
	JA   generic

fail:
	MOVQ $0xffffffffffffffff, AX
	MOVQ AX, ret+24(FP)
	RET

done:
	SUBQ DX, AX
	MOVQ AX, ret+24(FP)
	RET

avx2:
	CMPQ CX, $0x00000108
	JB   avx2_tail128
	XORQ BX, BX
	XORQ SI, SI
	XORQ DI, DI
	XORQ R8, R8
	XORQ R9, R9
	XORQ R10, R10
	XORQ R11, R11
	XORQ R12, R12

avx2_loop256:
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   128(AX), Y8
	VMOVDQU   160(AX), Y10
	VMOVDQU   192(AX), Y12
	VMOVDQU   224(AX), Y14
	VMOVDQU   8(AX), Y1
	VMOVDQU   40(AX), Y3
	VMOVDQU   72(AX), Y5
	VMOVDQU   104(AX), Y7
	VMOVDQU   136(AX), Y9
	VMOVDQU   168(AX), Y11
	VMOVDQU   200(AX), Y13
	VMOVDQU   232(AX), Y15
	VPCMPEQQ  Y0, Y1, Y1
	VPCMPEQQ  Y2, Y3, Y3
	VPCMPEQQ  Y4, Y5, Y5
	VPCMPEQQ  Y6, Y7, Y7
	VPCMPEQQ  Y8, Y9, Y9
	VPCMPEQQ  Y10, Y11, Y11
	VPCMPEQQ  Y12, Y13, Y13
	VPCMPEQQ  Y14, Y15, Y15
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y3, SI
	VPMOVMSKB Y5, DI
	VPMOVMSKB Y7, R8
	VPMOVMSKB Y9, R9
	VPMOVMSKB Y11, R10
	VPMOVMSKB Y13, R11
	VPMOVMSKB Y15, R12
	XORQ      R13, R13
	ORQ       BX, R13
	ORQ       SI, R13
	ORQ       DI, R13
	ORQ       R8, R13
	ORQ       R9, R13
	ORQ       R10, R13
	ORQ       R11, R13
	ORQ       R12, R13
	CMPQ      R13, $0x00
	JNE       avx2_done
	ADDQ      $0x00000100, AX
	SUBQ      $0x00000100, CX
	CMPQ      CX, $0x00000108
	JAE       avx2_loop256

avx2_tail128:
	CMPQ      CX, $0x88
	JB        avx2_tail64
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   8(AX), Y1
	VMOVDQU   40(AX), Y3
	VMOVDQU   72(AX), Y5
	VMOVDQU   104(AX), Y7
	VPCMPEQQ  Y0, Y1, Y1
	VPCMPEQQ  Y2, Y3, Y3
	VPCMPEQQ  Y4, Y5, Y5
	VPCMPEQQ  Y6, Y7, Y7
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y3, SI
	VPMOVMSKB Y5, DI
	VPMOVMSKB Y7, R8
	XORQ      R13, R13
	ORQ       BX, R13
	ORQ       SI, R13
	ORQ       DI, R13
	ORQ       R8, R13
	CMPQ      R13, $0x00
	JNE       avx2_done
	ADDQ      $0x00000080, AX
	SUBQ      $0x00000080, CX

avx2_tail64:
	CMPQ      CX, $0x48
	JB        avx2_tail32
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   8(AX), Y1
	VMOVDQU   40(AX), Y3
	VPCMPEQQ  Y0, Y1, Y1
	VPCMPEQQ  Y2, Y3, Y3
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y3, SI
	XORQ      R13, R13
	ORQ       BX, R13
	ORQ       SI, R13
	CMPQ      R13, $0x00
	JNE       avx2_done
	ADDQ      $0x00000040, AX
	SUBQ      $0x00000040, CX

avx2_tail32:
	CMPQ      CX, $0x28
	JB        avx2_tail16
	VMOVDQU   (AX), Y0
	VMOVDQU   8(AX), Y1
	VPCMPEQQ  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	CMPQ      BX, $0x00
	JNE       avx2_done
	ADDQ      $0x00000020, AX
	SUBQ      $0x00000020, CX

avx2_tail16:
	CMPQ      CX, $0x18
	JB        avx2_tail
	VMOVDQU   (AX), X0
	VMOVDQU   8(AX), X1
	VPCMPEQQ  X0, X1, X1
	VPMOVMSKB X1, BX
	CMPQ      BX, $0x00
	JNE       avx2_done
	ADDQ      $0x10, AX
	SUBQ      $0x10, CX

avx2_tail:
	VZEROUPPER
	JMP tail

avx2_done:
	VZEROUPPER
	CMPQ BX, $0x00
	JNE  avx2_done0
	CMPQ SI, $0x00
	JNE  avx2_done1
	CMPQ DI, $0x00
	JNE  avx2_done2
	CMPQ R8, $0x00
	JNE  avx2_done3
	CMPQ R9, $0x00
	JNE  avx2_done4
	CMPQ R10, $0x00
	JNE  avx2_done5
	CMPQ R11, $0x00
	JNE  avx2_done6
	CMPQ R12, $0x00
	JNE  avx2_done7

avx2_done0:
	TZCNTQ BX, BX
	ADDQ   BX, AX
	SUBQ   BX, CX
	JMP    done

avx2_done1:
	ADDQ   $0x00000020, AX
	SUBQ   $0x00000020, CX
	TZCNTQ SI, SI
	ADDQ   SI, AX
	SUBQ   SI, CX
	JMP    done

avx2_done2:
	ADDQ   $0x00000040, AX
	SUBQ   $0x00000040, CX
	TZCNTQ DI, DI
	ADDQ   DI, AX
	SUBQ   DI, CX
	JMP    done

avx2_done3:
	ADDQ   $0x00000060, AX
	SUBQ   $0x00000060, CX
	TZCNTQ R8, R8
	ADDQ   R8, AX
	SUBQ   R8, CX
	JMP    done

avx2_done4:
	ADDQ   $0x00000080, AX
	SUBQ   $0x00000080, CX
	TZCNTQ R9, R9
	ADDQ   R9, AX
	SUBQ   R9, CX
	JMP    done

avx2_done5:
	ADDQ   $0x000000a0, AX
	SUBQ   $0x000000a0, CX
	TZCNTQ R10, R10
	ADDQ   R10, AX
	SUBQ   R10, CX
	JMP    done

avx2_done6:
	ADDQ   $0x000000c0, AX
	SUBQ   $0x000000c0, CX
	TZCNTQ R11, R11
	ADDQ   R11, AX
	SUBQ   R11, CX
	JMP    done

avx2_done7:
	ADDQ   $0x000000e0, AX
	SUBQ   $0x000000e0, CX
	TZCNTQ R12, R12
	ADDQ   R12, AX
	SUBQ   R12, CX
	JMP    done

// func indexPair16(b []byte) int
// Requires: AVX, AVX2, BMI, SSE2, SSE4.1
TEXT ·indexPair16(SB), NOSPLIT, $0-32
	MOVQ b_base+0(FP), AX
	MOVQ b_len+8(FP), CX
	MOVQ AX, DX
	CMPQ CX, $0x00
	JLE  fail
	SUBQ $0x10, CX
	BTL  $0x08, github·com∕segmentio∕asm∕cpu·X86+0(SB)
	JCS  avx2

tail:
	CMPQ CX, $0x00
	JE   fail

generic:
	MOVOU    (AX), X0
	MOVOU    16(AX), X1
	PCMPEQQ  X0, X1
	PMOVMSKB X1, BX
	CMPL     BX, $0x0000ffff
	JE       done
	ADDQ     $0x10, AX
	SUBQ     $0x10, CX
	CMPQ     CX, $0x00
	JA       generic

fail:
	MOVQ $0xffffffffffffffff, AX
	MOVQ AX, ret+24(FP)
	RET

done:
	SUBQ DX, AX
	MOVQ AX, ret+24(FP)
	RET

avx2:
	CMPQ CX, $0x00000110
	JB   avx2_tail128
	XORQ BX, BX
	XORQ SI, SI
	XORQ DI, DI
	XORQ R8, R8
	XORQ R9, R9
	XORQ R10, R10
	XORQ R11, R11
	XORQ R12, R12

avx2_loop256:
	VMOVDQU    (AX), Y0
	VMOVDQU    32(AX), Y2
	VMOVDQU    64(AX), Y4
	VMOVDQU    96(AX), Y6
	VMOVDQU    128(AX), Y8
	VMOVDQU    160(AX), Y10
	VMOVDQU    192(AX), Y12
	VMOVDQU    224(AX), Y14
	VPERM2I128 $0x21, Y2, Y0, Y1
	VPERM2I128 $0x21, Y4, Y2, Y3
	VPERM2I128 $0x21, Y6, Y4, Y5
	VPERM2I128 $0x21, Y8, Y6, Y7
	VPERM2I128 $0x21, Y10, Y8, Y9
	VPERM2I128 $0x21, Y12, Y10, Y11
	VPERM2I128 $0x21, Y14, Y12, Y13
	VMOVDQU    240(AX), Y15
	VPCMPEQQ   Y0, Y1, Y1
	VPCMPEQQ   Y2, Y3, Y3
	VPCMPEQQ   Y4, Y5, Y5
	VPCMPEQQ   Y6, Y7, Y7
	VPCMPEQQ   Y8, Y9, Y9
	VPCMPEQQ   Y10, Y11, Y11
	VPCMPEQQ   Y12, Y13, Y13
	VPCMPEQQ   Y14, Y15, Y15
	VPERMQ     $0xb1, Y1, Y0
	VPAND      Y1, Y0, Y0
	VPMOVMSKB  Y0, BX
	VPERMQ     $0xb1, Y3, Y2
	VPAND      Y3, Y2, Y2
	VPMOVMSKB  Y2, SI
	VPERMQ     $0xb1, Y5, Y4
	VPAND      Y5, Y4, Y4
	VPMOVMSKB  Y4, DI
	VPERMQ     $0xb1, Y7, Y6
	VPAND      Y7, Y6, Y6
	VPMOVMSKB  Y6, R8
	VPERMQ     $0xb1, Y9, Y8
	VPAND      Y9, Y8, Y8
	VPMOVMSKB  Y8, R9
	VPERMQ     $0xb1, Y11, Y10
	VPAND      Y11, Y10, Y10
	VPMOVMSKB  Y10, R10
	VPERMQ     $0xb1, Y13, Y12
	VPAND      Y13, Y12, Y12
	VPMOVMSKB  Y12, R11
	VPERMQ     $0xb1, Y15, Y14
	VPAND      Y15, Y14, Y14
	VPMOVMSKB  Y14, R12
	XORQ       R13, R13
	ORQ        BX, R13
	ORQ        SI, R13
	ORQ        DI, R13
	ORQ        R8, R13
	ORQ        R9, R13
	ORQ        R10, R13
	ORQ        R11, R13
	ORQ        R12, R13
	CMPQ       R13, $0x00
	JNE        avx2_done
	ADDQ       $0x00000100, AX
	SUBQ       $0x00000100, CX
	CMPQ       CX, $0x00000110
	JAE        avx2_loop256

avx2_tail128:
	CMPQ       CX, $0x90
	JB         avx2_tail64
	VMOVDQU    (AX), Y0
	VMOVDQU    32(AX), Y2
	VMOVDQU    64(AX), Y4
	VMOVDQU    96(AX), Y6
	VPERM2I128 $0x21, Y2, Y0, Y1
	VPERM2I128 $0x21, Y4, Y2, Y3
	VPERM2I128 $0x21, Y6, Y4, Y5
	VMOVDQU    112(AX), Y7
	VPCMPEQQ   Y0, Y1, Y1
	VPCMPEQQ   Y2, Y3, Y3
	VPCMPEQQ   Y4, Y5, Y5
	VPCMPEQQ   Y6, Y7, Y7
	VPERMQ     $0xb1, Y1, Y0
	VPAND      Y1, Y0, Y0
	VPMOVMSKB  Y0, BX
	VPERMQ     $0xb1, Y3, Y2
	VPAND      Y3, Y2, Y2
	VPMOVMSKB  Y2, SI
	VPERMQ     $0xb1, Y5, Y4
	VPAND      Y5, Y4, Y4
	VPMOVMSKB  Y4, DI
	VPERMQ     $0xb1, Y7, Y6
	VPAND      Y7, Y6, Y6
	VPMOVMSKB  Y6, R8
	XORQ       R13, R13
	ORQ        BX, R13
	ORQ        SI, R13
	ORQ        DI, R13
	ORQ        R8, R13
	CMPQ       R13, $0x00
	JNE        avx2_done
	ADDQ       $0x00000080, AX
	SUBQ       $0x00000080, CX

avx2_tail64:
	CMPQ       CX, $0x50
	JB         avx2_tail32
	VMOVDQU    (AX), Y0
	VMOVDQU    32(AX), Y2
	VPERM2I128 $0x21, Y2, Y0, Y1
	VMOVDQU    48(AX), Y3
	VPCMPEQQ   Y0, Y1, Y1
	VPCMPEQQ   Y2, Y3, Y3
	VPERMQ     $0xb1, Y1, Y0
	VPAND      Y1, Y0, Y0
	VPMOVMSKB  Y0, BX
	VPERMQ     $0xb1, Y3, Y2
	VPAND      Y3, Y2, Y2
	VPMOVMSKB  Y2, SI
	XORQ       R13, R13
	ORQ        BX, R13
	ORQ        SI, R13
	CMPQ       R13, $0x00
	JNE        avx2_done
	ADDQ       $0x00000040, AX
	SUBQ       $0x00000040, CX

avx2_tail32:
	CMPQ      CX, $0x30
	JB        avx2_tail16
	VMOVDQU   (AX), Y0
	VMOVDQU   16(AX), Y1
	VPCMPEQQ  Y0, Y1, Y1
	VPERMQ    $0xb1, Y1, Y0
	VPAND     Y1, Y0, Y0
	VPMOVMSKB Y0, BX
	CMPQ      BX, $0x00
	JNE       avx2_done
	ADDQ      $0x00000020, AX
	SUBQ      $0x00000020, CX

avx2_tail16:
	VZEROUPPER
	JMP tail

avx2_done:
	VZEROUPPER
	CMPQ BX, $0x00
	JNE  avx2_done0
	CMPQ SI, $0x00
	JNE  avx2_done1
	CMPQ DI, $0x00
	JNE  avx2_done2
	CMPQ R8, $0x00
	JNE  avx2_done3
	CMPQ R9, $0x00
	JNE  avx2_done4
	CMPQ R10, $0x00
	JNE  avx2_done5
	CMPQ R11, $0x00
	JNE  avx2_done6
	CMPQ R12, $0x00
	JNE  avx2_done7

avx2_done0:
	TZCNTQ BX, BX
	ADDQ   BX, AX
	SUBQ   BX, CX
	JMP    done

avx2_done1:
	ADDQ   $0x00000020, AX
	SUBQ   $0x00000020, CX
	TZCNTQ SI, SI
	ADDQ   SI, AX
	SUBQ   SI, CX
	JMP    done

avx2_done2:
	ADDQ   $0x00000040, AX
	SUBQ   $0x00000040, CX
	TZCNTQ DI, DI
	ADDQ   DI, AX
	SUBQ   DI, CX
	JMP    done

avx2_done3:
	ADDQ   $0x00000060, AX
	SUBQ   $0x00000060, CX
	TZCNTQ R8, R8
	ADDQ   R8, AX
	SUBQ   R8, CX
	JMP    done

avx2_done4:
	ADDQ   $0x00000080, AX
	SUBQ   $0x00000080, CX
	TZCNTQ R9, R9
	ADDQ   R9, AX
	SUBQ   R9, CX
	JMP    done

avx2_done5:
	ADDQ   $0x000000a0, AX
	SUBQ   $0x000000a0, CX
	TZCNTQ R10, R10
	ADDQ   R10, AX
	SUBQ   R10, CX
	JMP    done

avx2_done6:
	ADDQ   $0x000000c0, AX
	SUBQ   $0x000000c0, CX
	TZCNTQ R11, R11
	ADDQ   R11, AX
	SUBQ   R11, CX
	JMP    done

avx2_done7:
	ADDQ   $0x000000e0, AX
	SUBQ   $0x000000e0, CX
	TZCNTQ R12, R12
	ADDQ   R12, AX
	SUBQ   R12, CX
	JMP    done

// func indexPair32(b []byte) int
// Requires: AVX, AVX2, BMI, CMOV, SSE2, SSE4.1
TEXT ·indexPair32(SB), NOSPLIT, $0-32
	MOVQ b_base+0(FP), AX
	MOVQ b_len+8(FP), CX
	MOVQ AX, DX
	CMPQ CX, $0x00
	JLE  fail
	SUBQ $0x20, CX
	BTL  $0x08, github·com∕segmentio∕asm∕cpu·X86+0(SB)
	JCS  avx2

tail:
	CMPQ CX, $0x00
	JE   fail

generic:
	MOVOU    (AX), X0
	MOVOU    16(AX), X1
	MOVOU    32(AX), X2
	MOVOU    48(AX), X3
	PCMPEQQ  X0, X2
	PCMPEQQ  X1, X3
	PMOVMSKB X2, BX
	PMOVMSKB X3, SI
	ANDL     SI, BX
	CMPL     BX, $0x0000ffff
	JE       done
	ADDQ     $0x20, AX
	SUBQ     $0x20, CX
	CMPQ     CX, $0x00
	JA       generic

fail:
	MOVQ $0xffffffffffffffff, AX
	MOVQ AX, ret+24(FP)
	RET

done:
	SUBQ DX, AX
	MOVQ AX, ret+24(FP)
	RET

avx2:
	CMPQ CX, $0x00000120
	JB   avx2_tail128
	XORQ BX, BX
	XORQ SI, SI
	XORQ DI, DI
	XORQ R8, R8
	XORQ R9, R9
	XORQ R10, R10
	XORQ R11, R11
	XORQ R12, R12

avx2_loop256:
	XORQ      R13, R13
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y5
	VMOVDQU   128(AX), Y7
	VMOVDQU   160(AX), Y8
	VMOVDQU   192(AX), Y9
	VMOVDQU   224(AX), Y10
	VMOVDQU   256(AX), Y11
	VPCMPEQQ  Y0, Y2, Y1
	VPCMPEQQ  Y2, Y4, Y3
	VPCMPEQQ  Y4, Y5, Y0
	VPCMPEQQ  Y5, Y7, Y6
	VPCMPEQQ  Y7, Y8, Y2
	VPCMPEQQ  Y8, Y9, Y4
	VPCMPEQQ  Y9, Y10, Y5
	VPCMPEQQ  Y10, Y11, Y11
	VPMOVMSKB Y1, BX
	CMPL      BX, $0xffffffff
	CMOVLNE   R13, BX
	VPMOVMSKB Y3, SI
	CMPL      SI, $0xffffffff
	CMOVLNE   R13, SI
	VPMOVMSKB Y0, DI
	CMPL      DI, $0xffffffff
	CMOVLNE   R13, DI
	VPMOVMSKB Y6, R8
	CMPL      R8, $0xffffffff
	CMOVLNE   R13, R8
	VPMOVMSKB Y2, R9
	CMPL      R9, $0xffffffff
	CMOVLNE   R13, R9
	VPMOVMSKB Y4, R10
	CMPL      R10, $0xffffffff
	CMOVLNE   R13, R10
	VPMOVMSKB Y5, R11
	CMPL      R11, $0xffffffff
	CMOVLNE   R13, R11
	VPMOVMSKB Y11, R12
	CMPL      R12, $0xffffffff
	CMOVLNE   R13, R12
	XORQ      R13, R13
	ORQ       BX, R13
	ORQ       SI, R13
	ORQ       DI, R13
	ORQ       R8, R13
	ORQ       R9, R13
	ORQ       R10, R13
	ORQ       R11, R13
	ORQ       R12, R13
	CMPQ      R13, $0x00
	JNE       avx2_done
	ADDQ      $0x00000100, AX
	SUBQ      $0x00000100, CX
	CMPQ      CX, $0x00000120
	JAE       avx2_loop256

avx2_tail128:
	CMPQ      CX, $0xa0
	JB        avx2_tail64
	XORQ      R13, R13
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y5
	VMOVDQU   128(AX), Y6
	VPCMPEQQ  Y0, Y2, Y1
	VPCMPEQQ  Y2, Y4, Y3
	VPCMPEQQ  Y4, Y5, Y0
	VPCMPEQQ  Y5, Y6, Y6
	VPMOVMSKB Y1, BX
	CMPL      BX, $0xffffffff
	CMOVLNE   R13, BX
	VPMOVMSKB Y3, SI
	CMPL      SI, $0xffffffff
	CMOVLNE   R13, SI
	VPMOVMSKB Y0, DI
	CMPL      DI, $0xffffffff
	CMOVLNE   R13, DI
	VPMOVMSKB Y6, R8
	CMPL      R8, $0xffffffff
	CMOVLNE   R13, R8
	XORQ      R13, R13
	ORQ       BX, R13
	ORQ       SI, R13
	ORQ       DI, R13
	ORQ       R8, R13
	CMPQ      R13, $0x00
	JNE       avx2_done
	ADDQ      $0x00000080, AX
	SUBQ      $0x00000080, CX

avx2_tail64:
	CMPQ      CX, $0x60
	JB        avx2_tail32
	XORQ      R13, R13
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y3
	VPCMPEQQ  Y0, Y2, Y1
	VPCMPEQQ  Y2, Y3, Y3
	VPMOVMSKB Y1, BX
	CMPL      BX, $0xffffffff
	CMOVLNE   R13, BX
	VPMOVMSKB Y3, SI
	CMPL      SI, $0xffffffff
	CMOVLNE   R13, SI
	XORQ      R13, R13
	ORQ       BX, R13
	ORQ       SI, R13
	CMPQ      R13, $0x00
	JNE       avx2_done
	ADDQ      $0x00000040, AX
	SUBQ      $0x00000040, CX

avx2_tail32:
	CMPQ      CX, $0x40
	JB        avx2_tail16
	XORQ      R13, R13
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y1
	VPCMPEQQ  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	CMPL      BX, $0xffffffff
	CMOVLNE   R13, BX
	CMPQ      BX, $0x00
	JNE       avx2_done
	ADDQ      $0x00000020, AX
	SUBQ      $0x00000020, CX

avx2_tail16:
	VZEROUPPER
	JMP tail

avx2_done:
	VZEROUPPER
	CMPQ BX, $0x00
	JNE  avx2_done0
	CMPQ SI, $0x00
	JNE  avx2_done1
	CMPQ DI, $0x00
	JNE  avx2_done2
	CMPQ R8, $0x00
	JNE  avx2_done3
	CMPQ R9, $0x00
	JNE  avx2_done4
	CMPQ R10, $0x00
	JNE  avx2_done5
	CMPQ R11, $0x00
	JNE  avx2_done6
	CMPQ R12, $0x00
	JNE  avx2_done7

avx2_done0:
	TZCNTQ BX, BX
	ADDQ   BX, AX
	SUBQ   BX, CX
	JMP    done

avx2_done1:
	ADDQ   $0x00000020, AX
	SUBQ   $0x00000020, CX
	TZCNTQ SI, SI
	ADDQ   SI, AX
	SUBQ   SI, CX
	JMP    done

avx2_done2:
	ADDQ   $0x00000040, AX
	SUBQ   $0x00000040, CX
	TZCNTQ DI, DI
	ADDQ   DI, AX
	SUBQ   DI, CX
	JMP    done

avx2_done3:
	ADDQ   $0x00000060, AX
	SUBQ   $0x00000060, CX
	TZCNTQ R8, R8
	ADDQ   R8, AX
	SUBQ   R8, CX
	JMP    done

avx2_done4:
	ADDQ   $0x00000080, AX
	SUBQ   $0x00000080, CX
	TZCNTQ R9, R9
	ADDQ   R9, AX
	SUBQ   R9, CX
	JMP    done

avx2_done5:
	ADDQ   $0x000000a0, AX
	SUBQ   $0x000000a0, CX
	TZCNTQ R10, R10
	ADDQ   R10, AX
	SUBQ   R10, CX
	JMP    done

avx2_done6:
	ADDQ   $0x000000c0, AX
	SUBQ   $0x000000c0, CX
	TZCNTQ R11, R11
	ADDQ   R11, AX
	SUBQ   R11, CX
	JMP    done

avx2_done7:
	ADDQ   $0x000000e0, AX
	SUBQ   $0x000000e0, CX
	TZCNTQ R12, R12
	ADDQ   R12, AX
	SUBQ   R12, CX
	JMP    done
