/*
 *  Copyright 2024 The LibYuv Project Authors. All rights reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS. All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "libyuv/row_sve.h"
#include "libyuv/row.h"

#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif

#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)

#define RGBTOARGB8_SVE_2X                                 \
  /* Inputs: B: z16.h,  G: z17.h,  R: z18.h,  A: z19.b */ \
  "uqshrnb     z16.b, z16.h, #6     \n" /* B0 */          \
  "uqshrnb     z17.b, z17.h, #6     \n" /* G0 */          \
  "uqshrnb     z18.b, z18.h, #6     \n" /* R0 */          \
  "uqshrnt     z16.b, z20.h, #6     \n" /* B1 */          \
  "uqshrnt     z17.b, z21.h, #6     \n" /* G1 */          \
  "uqshrnt     z18.b, z22.h, #6     \n" /* R1 */

#define RGBTOARGB8_SVE_TOP_2X                        \
  /* Inputs: B: z16.h,  G: z17.h,  R: z18.h */       \
  "uqshl     z16.h, p0/m, z16.h, #2     \n" /* B0 */ \
  "uqshl     z17.h, p0/m, z17.h, #2     \n" /* G0 */ \
  "uqshl     z18.h, p0/m, z18.h, #2     \n" /* R0 */ \
  "uqshl     z20.h, p0/m, z20.h, #2     \n" /* B1 */ \
  "uqshl     z21.h, p0/m, z21.h, #2     \n" /* G1 */ \
  "uqshl     z22.h, p0/m, z22.h, #2     \n" /* R1 */

void I444ToARGBRow_SVE2(const uint8_t* src_y,
                        const uint8_t* src_u,
                        const uint8_t* src_v,
                        uint8_t* dst_argb,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  uint64_t vl;
  asm volatile(
      "cnth     %[vl]                                   \n"
      "ptrue    p0.b                                    \n"  //
      YUVTORGB_SVE_SETUP
      "dup      z19.b, #255                             \n"  // Alpha
      "subs     %w[width], %w[width], %w[vl]            \n"
      "b.lt     2f                                      \n"

      // Run bulk of computation with an all-true predicate to avoid predicate
      // generation overhead.
      "ptrue    p1.h                                    \n"
      "1:                                               \n"  //
      READYUV444_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
      "subs     %w[width], %w[width], %w[vl]            \n"
      "st2h     {z16.h, z17.h}, p1, [%[dst_argb]]       \n"
      "add      %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
      "b.ge     1b                                      \n"

      "2:                                               \n"
      "adds     %w[width], %w[width], %w[vl]            \n"
      "b.eq     99f                                     \n"

      // Calculate a predicate for the final iteration to deal with the tail.
      "whilelt  p1.h, wzr, %w[width]                    \n"  //
      READYUV444_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
      "st2h     {z16.h, z17.h}, p1, [%[dst_argb]]       \n"

      "99:                                              \n"
      : [src_y] "+r"(src_y),                               // %[src_y]
        [src_u] "+r"(src_u),                               // %[src_u]
        [src_v] "+r"(src_v),                               // %[src_v]
        [dst_argb] "+r"(dst_argb),                         // %[dst_argb]
        [width] "+r"(width),                               // %[width]
        [vl] "=&r"(vl)                                     // %[vl]
      : [kUVCoeff] "r"(&yuvconstants->kUVCoeff),           // %[kUVCoeff]
        [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias)  // %[kRGBCoeffBias]
      : "cc", "memory", YUVTORGB_SVE_REGS);
}

void I444ToRGB24Row_SVE2(const uint8_t* src_y,
                         const uint8_t* src_u,
                         const uint8_t* src_v,
                         uint8_t* dst_rgb24,
                         const struct YuvConstants* yuvconstants,
                         int width) {
  I444ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_rgb24, yuvconstants, width);
}

void I400ToARGBRow_SVE2(const uint8_t* src_y,
                        uint8_t* dst_argb,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  I400ToARGBRow_SVE_SC(src_y, dst_argb, yuvconstants, width);
}

void I422ToARGBRow_SVE2(const uint8_t* src_y,
                        const uint8_t* src_u,
                        const uint8_t* src_v,
                        uint8_t* dst_argb,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  I422ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
}

void I422ToRGB24Row_SVE2(const uint8_t* src_y,
                         const uint8_t* src_u,
                         const uint8_t* src_v,
                         uint8_t* dst_argb,
                         const struct YuvConstants* yuvconstants,
                         int width) {
  I422ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
}

void I422ToRGB565Row_SVE2(const uint8_t* src_y,
                          const uint8_t* src_u,
                          const uint8_t* src_v,
                          uint8_t* dst_rgb565,
                          const struct YuvConstants* yuvconstants,
                          int width) {
  I422ToRGB565Row_SVE_SC(src_y, src_u, src_v, dst_rgb565, yuvconstants, width);
}

void I422ToARGB1555Row_SVE2(const uint8_t* src_y,
                            const uint8_t* src_u,
                            const uint8_t* src_v,
                            uint8_t* dst_argb1555,
                            const struct YuvConstants* yuvconstants,
                            int width) {
  I422ToARGB1555Row_SVE_SC(src_y, src_u, src_v, dst_argb1555, yuvconstants,
                           width);
}

void I422ToARGB4444Row_SVE2(const uint8_t* src_y,
                            const uint8_t* src_u,
                            const uint8_t* src_v,
                            uint8_t* dst_argb4444,
                            const struct YuvConstants* yuvconstants,
                            int width) {
  I422ToARGB4444Row_SVE_SC(src_y, src_u, src_v, dst_argb4444, yuvconstants,
                           width);
}

void I422ToRGBARow_SVE2(const uint8_t* src_y,
                        const uint8_t* src_u,
                        const uint8_t* src_v,
                        uint8_t* dst_rgba,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  I422ToRGBARow_SVE_SC(src_y, src_u, src_v, dst_rgba, yuvconstants, width);
}

void I422ToAR30Row_SVE2(const uint8_t* src_y,
                        const uint8_t* src_u,
                        const uint8_t* src_v,
                        uint8_t* dst_argb,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  I422ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
}

void I422AlphaToARGBRow_SVE2(const uint8_t* src_y,
                             const uint8_t* src_u,
                             const uint8_t* src_v,
                             const uint8_t* src_a,
                             uint8_t* dst_argb,
                             const struct YuvConstants* yuvconstants,
                             int width) {
  I422AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
                            width);
}

void I444AlphaToARGBRow_SVE2(const uint8_t* src_y,
                             const uint8_t* src_u,
                             const uint8_t* src_v,
                             const uint8_t* src_a,
                             uint8_t* dst_argb,
                             const struct YuvConstants* yuvconstants,
                             int width) {
  I444AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
                            width);
}

void NV12ToARGBRow_SVE2(const uint8_t* src_y,
                        const uint8_t* src_uv,
                        uint8_t* dst_argb,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  NV12ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
}

void NV21ToARGBRow_SVE2(const uint8_t* src_y,
                        const uint8_t* src_vu,
                        uint8_t* dst_argb,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  NV21ToARGBRow_SVE_SC(src_y, src_vu, dst_argb, yuvconstants, width);
}

void NV12ToRGB24Row_SVE2(const uint8_t* src_y,
                         const uint8_t* src_uv,
                         uint8_t* dst_rgb24,
                         const struct YuvConstants* yuvconstants,
                         int width) {
  NV12ToRGB24Row_SVE_SC(src_y, src_uv, dst_rgb24, yuvconstants, width);
}

void NV21ToRGB24Row_SVE2(const uint8_t* src_y,
                         const uint8_t* src_vu,
                         uint8_t* dst_rgb24,
                         const struct YuvConstants* yuvconstants,
                         int width) {
  NV21ToRGB24Row_SVE_SC(src_y, src_vu, dst_rgb24, yuvconstants, width);
}

void ARGBToUVMatrixRow_SVE2(const uint8_t* src_argb,
                            int src_stride_argb,
                            uint8_t* dst_u,
                            uint8_t* dst_v,
                            int width,
                            const struct ArgbConstants* c) {
  int8_t uvconstants[8] = {(int8_t)c->kRGBToU[0], (int8_t)c->kRGBToU[1],
                           (int8_t)c->kRGBToU[2], (int8_t)c->kRGBToU[3],
                           (int8_t)c->kRGBToV[0], (int8_t)c->kRGBToV[1],
                           (int8_t)c->kRGBToV[2], (int8_t)c->kRGBToV[3]};
  ARGBToUVMatrixRow_SVE_SC(src_argb, src_stride_argb, dst_u, dst_v, width,
                           uvconstants);
}

void ARGBToYMatrixRow_SVE2(const uint8_t* src_argb,
                           uint8_t* dst_y,
                           int width,
                           const struct ArgbConstants* c) {
  ARGBToYMatrixRow_SVE_SC(src_argb, dst_y, width, c);
}

#define ARGBTORGB565_SVE                    \
  /* Inputs:                                \
   * z0: rrrrrxxxbbbbbxxx                   \
   * z1: xxxxxxxxggggggxx                   \
   * z3: 0000000000000011 (3, 0, 3, 0, ...) \
   * z4: 0000011111100000                   \
   */                                       \
  "lsr     z0.b, p0/m, z0.b, z3.b       \n" \
  "lsl     z1.h, z1.h, #3               \n" \
  "bsl     z1.d, z1.d, z0.d, z4.d       \n"

void ARGBToRGB565Row_SVE2(const uint8_t* src_argb,
                          uint8_t* dst_rgb,
                          int width) {
  unsigned bsl_mask = 0x7e0;
  uint64_t vl;
  width *= 2;
  asm volatile(
      "mov     z3.h, #3                     \n"
      "dup     z4.h, %w[bsl_mask]           \n"

      "cntb    %[vl]                        \n"
      "subs    %w[width], %w[width], %w[vl] \n"
      "b.lt    2f                           \n"

      "ptrue   p0.b                         \n"
      "1:                                   \n"
      "ld2b    {z0.b, z1.b}, p0/z, [%[src]] \n"  // BR, GA
      "incb    %[src], all, mul #2          \n"
      "subs    %w[width], %w[width], %w[vl] \n"  //
      ARGBTORGB565_SVE
      "st1b    {z1.b}, p0, [%[dst]]         \n"
      "incb    %[dst]                       \n"
      "b.ge    1b                           \n"

      "2:                                   \n"
      "adds    %w[width], %w[width], %w[vl] \n"
      "b.eq    99f                          \n"

      "whilelt p0.b, wzr, %w[width]         \n"
      "ld2b    {z0.b, z1.b}, p0/z, [%[src]] \n"  // BR, GA
      ARGBTORGB565_SVE
      "st1b    {z1.b}, p0, [%[dst]]         \n"

      "99:                                  \n"
      : [src] "+r"(src_argb),     // %[src]
        [dst] "+r"(dst_rgb),      // %[dst]
        [width] "+r"(width),      // %[width]
        [vl] "=&r"(vl)            // %[vl]
      : [bsl_mask] "r"(bsl_mask)  // %[bsl_mask]
      : "cc", "memory", "z0", "z1", "z3", "z4", "p0");
}

void ARGBToRGB565DitherRow_SVE2(const uint8_t* src_argb,
                                uint8_t* dst_rgb,
                                uint32_t dither4,
                                int width) {
  unsigned bsl_mask = 0x7e0;
  uint64_t vl;
  width *= 2;
  asm volatile(
      "mov     z3.h, #3                     \n"
      "dup     z4.h, %w[bsl_mask]           \n"
      "dup     z2.s, %w[dither4]            \n"
      "zip1    z2.b, z2.b, z2.b             \n"

      "cntb    %[vl]                        \n"
      "subs    %w[width], %w[width], %w[vl] \n"
      "b.lt    2f                           \n"

      "ptrue   p0.b                         \n"
      "1:                                   \n"
      "ld2b    {z0.b, z1.b}, p0/z, [%[src]] \n"  // BR, GA
      "incb    %[src], all, mul #2          \n"
      "uqadd   z0.b, z0.b, z2.b             \n"
      "uqadd   z1.b, z1.b, z2.b             \n"
      "subs    %w[width], %w[width], %w[vl] \n"  //
      ARGBTORGB565_SVE
      "st1b    {z1.b}, p0, [%[dst]]         \n"
      "incb    %[dst]                       \n"
      "b.ge    1b                           \n"

      "2:                                   \n"
      "adds    %w[width], %w[width], %w[vl] \n"
      "b.eq    99f                          \n"

      "whilelt p0.b, wzr, %w[width]         \n"
      "ld2b    {z0.b, z1.b}, p0/z, [%[src]] \n"  // BR, GA
      "uqadd   z0.b, z0.b, z2.b             \n"
      "uqadd   z1.b, z1.b, z2.b             \n"  //
      ARGBTORGB565_SVE
      "st1b    {z1.b}, p0, [%[dst]]         \n"

      "99:                                  \n"
      : [src] "+r"(src_argb),      // %[src]
        [dst] "+r"(dst_rgb),       // %[dst]
        [width] "+r"(width),       // %[width]
        [vl] "=&r"(vl)             // %[vl]
      : [bsl_mask] "r"(bsl_mask),  // %[bsl_mask]
        [dither4] "r"(dither4)     // %[dither4]
      : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0");
}

#define ARGB1555TOARGB                                        \
  /* Input: z1/z3.h = arrrrrgggggbbbbb */                     \
  "lsl     z0.h, z1.h, #3          \n" /* rrrgggggbbbbb000 */ \
  "lsl     z2.h, z3.h, #3          \n" /* rrrgggggbbbbb000 */ \
  "asr     z1.h, z1.h, #7          \n" /* aaaaaaaarrrrrggg */ \
  "asr     z3.h, z3.h, #7          \n" /* aaaaaaaarrrrrggg */ \
  "lsl     z0.b, p0/m, z0.b, z4.b  \n" /* ggggg000bbbbb000 */ \
  "lsl     z2.b, p0/m, z2.b, z4.b  \n" /* ggggg000bbbbb000 */ \
  "sri     z1.b, z1.b, #5          \n" /* aaaaaaaarrrrrrrr */ \
  "sri     z3.b, z3.b, #5          \n" /* aaaaaaaarrrrrrrr */ \
  "sri     z0.b, z0.b, #5          \n" /* ggggggggbbbbbbbb */ \
  "sri     z2.b, z2.b, #5          \n" /* ggggggggbbbbbbbb */

void ARGB1555ToARGBRow_SVE2(const uint8_t* src_argb1555,
                            uint8_t* dst_argb,
                            int width) {
  uint64_t vl;
  asm volatile(
      "mov     z4.h, #0x0300                           \n"
      "ptrue   p0.b                                    \n"

      "cnth    %x[vl]                                  \n"
      "subs    %w[width], %w[width], %w[vl], lsl #1    \n"
      "b.lt    2f                                      \n"

      "1:                                              \n"
      "ld1h    {z1.h}, p0/z, [%[src]]                  \n"
      "ld1h    {z3.h}, p0/z, [%[src], #1, mul vl]      \n"
      "incb    %[src], all, mul #2                     \n"  //
      ARGB1555TOARGB
      "subs    %w[width], %w[width], %w[vl], lsl #1    \n"
      "st2h    {z0.h, z1.h}, p0, [%[dst]]              \n"
      "st2h    {z2.h, z3.h}, p0, [%[dst], #2, mul vl]  \n"
      "incb    %[dst], all, mul #4                     \n"
      "b.ge    1b                                      \n"

      "2:                                              \n"
      "adds    %w[width], %w[width], %w[vl], lsl #1    \n"
      "b.eq    99f                                     \n"

      "whilelt p1.h, wzr, %w[width]                    \n"
      "whilelt p2.h, %w[vl], %w[width]                 \n"
      "ld1h    {z1.h}, p1/z, [%[src]]                  \n"
      "ld1h    {z3.h}, p2/z, [%[src], #1, mul vl]      \n"  //
      ARGB1555TOARGB
      "st2h    {z0.h, z1.h}, p1, [%[dst]]              \n"
      "st2h    {z2.h, z3.h}, p2, [%[dst], #2, mul vl]  \n"

      "99:                                             \n"
      : [src] "+r"(src_argb1555),  // %[src]
        [dst] "+r"(dst_argb),      // %[dst]
        [width] "+r"(width),       // %[width]
        [vl] "=&r"(vl)             // %[vl]
      :
      : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0", "p1", "p2");
}

// clang-format off
#define AYUVTOUV_SVE(zU0, zV0, zU1, zV1)                   /* e.g. */          \
  "ld2h     {z0.h, z1.h}, p0/z, [%[src0]]              \n" /* VUVU.. YAYA.. */ \
  "ld2h     {z1.h, z2.h}, p1/z, [%[src0], #2, mul vl]  \n" /* VUVU.. YAYA.. */ \
  "ld2h     {z2.h, z3.h}, p0/z, [%[src1]]              \n" /* VUVU.. YAYA.. */ \
  "ld2h     {z3.h, z4.h}, p1/z, [%[src1], #2, mul vl]  \n" /* VUVU.. YAYA.. */ \
  "incb     %[src0], all, mul #4                       \n"                     \
  "incb     %[src1], all, mul #4                       \n"                     \
  "uaddlb   z4.h, z0.b, z2.b                           \n" /* V */             \
  "uaddlt   z5.h, z0.b, z2.b                           \n" /* U */             \
  "uaddlb   z6.h, z1.b, z3.b                           \n" /* V */             \
  "uaddlt   z7.h, z1.b, z3.b                           \n" /* U */             \
  "addp   " #zU0 ".h, p0/m, " #zU0 ".h, " #zV0 ".h     \n" /* UV */            \
  "addp   " #zU1 ".h, p1/m, " #zU1 ".h, " #zV1 ".h     \n" /* UV */            \
  "subs     %w[width], %w[width], %w[vl]               \n"                     \
  "urshr  " #zU0 ".h, p0/m, " #zU0 ".h, #2             \n" /* U0V0 */          \
  "urshr  " #zU1 ".h, p1/m, " #zU1 ".h, #2             \n" /* U0V0 */          \
  "st1b     {" #zU0 ".h}, p0, [%[dst]]                 \n"                     \
  "st1b     {" #zU1 ".h}, p1, [%[dst], #1, mul vl]     \n"                     \
  "incb     %[dst]                                     \n"
// clang-format on

// Filter 2 rows of AYUV UV's (444) into UV (420).
// AYUV is VUYA in memory.  UV for NV12 is UV order in memory.
void AYUVToUVRow_SVE2(const uint8_t* src_ayuv,
                      int src_stride_ayuv,
                      uint8_t* dst_uv,
                      int width) {
  // Output a row of UV values, filtering 2x2 rows of AYUV.
  const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv;
  int vl;
  asm volatile (
      "cntb    %x[vl]                            \n"
      "subs    %w[width], %w[width], %w[vl]      \n"
      "b.lt    2f                                \n"

      "ptrue   p0.h                              \n"
      "ptrue   p1.h                              \n"
      "1:                                        \n"
      AYUVTOUV_SVE(z5, z4, z7, z6)
      "b.ge    1b                                \n"

      "2:                                        \n"
      "adds    %w[width], %w[width], %w[vl]      \n"
      "b.eq    99f                               \n"

      "cnth    %x[vl]                            \n"
      "whilelt p0.h, wzr, %w[width]              \n" // first row
      "whilelt p1.h, %w[vl], %w[width]           \n" // second row
      AYUVTOUV_SVE(z5, z4, z7, z6)

      "99:                                       \n"
      : [src0]"+r"(src_ayuv),   // %[src0]
        [src1]"+r"(src_ayuv1),  // %[src1]
        [dst]"+r"(dst_uv),      // %[dst]
        [width]"+r"(width),     // %[width]
        [vl]"=&r"(vl)           // %[vl]
      :
      : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
        "p1");
}

// Filter 2 rows of AYUV UV's (444) into VU (420).
void AYUVToVURow_SVE2(const uint8_t* src_ayuv,
                      int src_stride_ayuv,
                      uint8_t* dst_vu,
                      int width) {
  // Output a row of VU values, filtering 2x2 rows of AYUV.
  const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv;
  int vl;
  asm volatile (
      "cntb    %x[vl]                            \n"
      "cmp     %w[width], %w[vl]                 \n"
      "subs    %w[width], %w[width], %w[vl]      \n"
      "b.lt    2f                                \n"

      "ptrue   p0.h                              \n"
      "ptrue   p1.h                              \n"
      "1:                                        \n"
      AYUVTOUV_SVE(z4, z5, z6, z7)
      "b.ge    1b                                \n"

      "2:                                        \n"
      "adds    %w[width], %w[width], %w[vl]      \n"
      "b.eq    99f                               \n"

      "cnth    %x[vl]                            \n"
      "whilelt p0.h, wzr, %w[width]              \n" // first row
      "whilelt p1.h, %w[vl], %w[width]           \n" // second row
      AYUVTOUV_SVE(z4, z5, z6, z7)

      "99:                                       \n"
      : [src0]"+r"(src_ayuv),   // %[src0]
        [src1]"+r"(src_ayuv1),  // %[src1]
        [dst]"+r"(dst_vu),      // %[dst]
        [width]"+r"(width),     // %[width]
        [vl]"=&r"(vl)           // %[vl]
      :
      : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
        "p1");
}

void YUY2ToARGBRow_SVE2(const uint8_t* src_yuy2,
                        uint8_t* dst_argb,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  YUY2ToARGBRow_SVE_SC(src_yuy2, dst_argb, yuvconstants, width);
}

void UYVYToARGBRow_SVE2(const uint8_t* src_uyvy,
                        uint8_t* dst_argb,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  UYVYToARGBRow_SVE_SC(src_uyvy, dst_argb, yuvconstants, width);
}

static inline void RAWToWXYZRow_SVE2(const uint8_t* src_raw,
                                     uint8_t* dst_wxyz,
                                     int width,
                                     uint32_t idx_start,
                                     uint32_t idx_step,
                                     uint32_t alpha) {
  uint32_t vl;
  asm("cntw %x0" : "=r"(vl));
  uint32_t vl_mul3 = vl * 3;
  uint32_t rem_mul3;
  asm volatile(
      "index   z31.s, %w[idx_start], %w[idx_step]        \n"
      "dup     z30.s, %w[alpha]                          \n"
      "subs    %w[width], %w[width], %w[vl], lsl #1      \n"
      "b.lt    2f                                        \n"

      // Run bulk of computation with the same predicates to avoid predicate
      // generation overhead. We set up p1 to only load 3/4 of a vector.
      "ptrue   p0.s                                      \n"
      "whilelt p1.b, wzr, %w[vl_mul3]                    \n"
      "1:                                                \n"
      "ld1b    {z0.b}, p1/z, [%[src]]                    \n"
      "add     %[src], %[src], %x[vl_mul3]               \n"
      "ld1b    {z1.b}, p1/z, [%[src]]                    \n"
      "add     %[src], %[src], %x[vl_mul3]               \n"
      "tbl     z0.b, {z0.b}, z31.b                       \n"
      "tbl     z1.b, {z1.b}, z31.b                       \n"
      "subs    %w[width], %w[width], %w[vl], lsl #1      \n"
      "orr     z0.d, z0.d, z30.d                         \n"
      "orr     z1.d, z1.d, z30.d                         \n"
      "st1w    {z0.s}, p0, [%[dst]]                      \n"
      "st1w    {z1.s}, p0, [%[dst], #1, mul vl]          \n"
      "incb    %[dst], all, mul #2                       \n"
      "b.ge    1b                                        \n"

      "2:                                                \n"
      "adds     %w[width], %w[width], %w[vl], lsl #1     \n"
      "b.eq     99f                                      \n"

      // Calculate a pair of predicates for the final iteration to deal with
      // the tail.
      "3:                                                \n"
      "add     %w[rem_mul3], %w[width], %w[width], lsl #1 \n"
      "whilelt p0.s, wzr, %w[width]                      \n"
      "whilelt p1.b, wzr, %w[rem_mul3]                    \n"
      "ld1b    {z0.b}, p1/z, [%[src]]                    \n"
      "add     %[src], %[src], %x[vl_mul3]               \n"
      "tbl     z0.b, {z0.b}, z31.b                       \n"
      "subs    %w[width], %w[width], %w[vl]              \n"
      "orr     z0.d, z0.d, z30.d                         \n"
      "st1w    {z0.s}, p0, [%[dst]]                      \n"
      "incb    %[dst]                                    \n"
      "b.gt    3b                                        \n"

      "99:                                               \n"
      : [src] "+r"(src_raw),         // %[src]
        [dst] "+r"(dst_wxyz),        // %[dst]
        [width] "+r"(width),         // %[width]
        [vl_mul3] "+r"(vl_mul3),     // %[vl_mul3]
        [rem_mul3] "=&r"(rem_mul3)   // %[rem_mul3]
      : [idx_start] "r"(idx_start),  // %[idx_start]
        [idx_step] "r"(idx_step),    // %[idx_step]
        [alpha] "r"(alpha),          // %[alpha]
        [vl] "r"(vl)                 // %[vl]
      : "cc", "memory", "z0", "z1", "z30", "z31", "p0", "p1");
}

void RAWToARGBRow_SVE2(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
  RAWToWXYZRow_SVE2(src_raw, dst_argb, width, 0xff000102U, 0x00030303U,
                    0xff000000U);
}

void RAWToRGBARow_SVE2(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
  RAWToWXYZRow_SVE2(src_raw, dst_rgba, width, 0x000102ffU, 0x03030300U,
                    0x000000ffU);
}

void RGB24ToARGBRow_SVE2(const uint8_t* src_rgb24,
                         uint8_t* dst_argb,
                         int width) {
  RAWToWXYZRow_SVE2(src_rgb24, dst_argb, width, 0xff020100U, 0x00030303U,
                    0xff000000U);
}

static const uint8_t kRAWToRGB24Indices[] = {
    2,   1,   0,   5,   4,   3,   8,   7,   6,   11,  10,  9,   14,  13,  12,
    17,  16,  15,  20,  19,  18,  23,  22,  21,  26,  25,  24,  29,  28,  27,
    32,  31,  30,  35,  34,  33,  38,  37,  36,  41,  40,  39,  44,  43,  42,
    47,  46,  45,  50,  49,  48,  53,  52,  51,  56,  55,  54,  59,  58,  57,
    62,  61,  60,  65,  64,  63,  68,  67,  66,  71,  70,  69,  74,  73,  72,
    77,  76,  75,  80,  79,  78,  83,  82,  81,  86,  85,  84,  89,  88,  87,
    92,  91,  90,  95,  94,  93,  98,  97,  96,  101, 100, 99,  104, 103, 102,
    107, 106, 105, 110, 109, 108, 113, 112, 111, 116, 115, 114, 119, 118, 117,
    122, 121, 120, 125, 124, 123, 128, 127, 126, 131, 130, 129, 134, 133, 132,
    137, 136, 135, 140, 139, 138, 143, 142, 141, 146, 145, 144, 149, 148, 147,
    152, 151, 150, 155, 154, 153, 158, 157, 156, 161, 160, 159, 164, 163, 162,
    167, 166, 165, 170, 169, 168, 173, 172, 171, 176, 175, 174, 179, 178, 177,
    182, 181, 180, 185, 184, 183, 188, 187, 186, 191, 190, 189, 194, 193, 192,
    197, 196, 195, 200, 199, 198, 203, 202, 201, 206, 205, 204, 209, 208, 207,
    212, 211, 210, 215, 214, 213, 218, 217, 216, 221, 220, 219, 224, 223, 222,
    227, 226, 225, 230, 229, 228, 233, 232, 231, 236, 235, 234, 239, 238, 237,
    242, 241, 240, 245, 244, 243, 248, 247, 246, 251, 250, 249, 254, 253, 252};

void RAWToRGB24Row_SVE2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
  // width is in elements, convert to bytes.
  width *= 3;
  // we use the mul3 predicate pattern throughout to use the largest multiple
  // of three number of lanes, for instance with a vector length of 16 bytes
  // only the first 15 bytes will be used for load/store instructions.
  uint32_t vl;
  asm volatile(
      "cntb    %x[vl], mul3                              \n"
      "ptrue   p0.b, mul3                                \n"
      "ld1b    {z31.b}, p0/z, [%[kIndices]]              \n"
      "subs    %w[width], %w[width], %w[vl]              \n"
      "b.lt    2f                                        \n"

      // Run bulk of computation with the same predicate to avoid predicate
      // generation overhead.
      "1:                                                \n"
      "ld1b    {z0.b}, p0/z, [%[src]]                    \n"
      "add     %[src], %[src], %x[vl]                    \n"
      "tbl     z0.b, {z0.b}, z31.b                       \n"
      "subs    %w[width], %w[width], %w[vl]              \n"
      "st1b    {z0.b}, p0, [%[dst]]                      \n"
      "add     %[dst], %[dst], %x[vl]                    \n"
      "b.ge    1b                                        \n"

      "2:                                                \n"
      "adds    %w[width], %w[width], %w[vl]              \n"
      "b.eq    99f                                       \n"

      // Calculate a predicate for the final iteration to deal with the tail.
      "whilelt p0.b, wzr, %w[width]                      \n"
      "ld1b    {z0.b}, p0/z, [%[src]]                    \n"
      "tbl     z0.b, {z0.b}, z31.b                       \n"
      "st1b    {z0.b}, p0, [%[dst]]                      \n"

      "99:                                               \n"
      : [src] "+r"(src_raw),                // %[src]
        [dst] "+r"(dst_rgb24),              // %[dst]
        [width] "+r"(width),                // %[width]
        [vl] "=&r"(vl)                      // %[vl]
      : [kIndices] "r"(kRAWToRGB24Indices)  // %[kIndices]
      : "cc", "memory", "z0", "z31", "p0");
}

static inline void ARGBToXYZRow_SVE2(const uint8_t* src_argb,
                                     uint8_t* dst_xyz,
                                     int width,
                                     const uint8_t* indices) {
  uint32_t vl;
  asm("cntw %x0" : "=r"(vl));
  uint32_t vl_mul3 = vl * 3;
  uint32_t rem_mul3;
  asm volatile(
      "whilelt p1.b, wzr, %w[vl_mul3]                     \n"
      "ld1b    {z31.b}, p1/z, [%[indices]]                \n"
      "subs    %w[width], %w[width], %w[vl], lsl #1       \n"
      "b.lt    2f                                         \n"

      // Run bulk of computation with the same predicates to avoid predicate
      // generation overhead. We set up p1 to only store 3/4 of a vector.
      "ptrue   p0.s                                       \n"
      "1:                                                 \n"
      "ld1w    {z0.s}, p0/z, [%[src]]                     \n"
      "ld1w    {z1.s}, p0/z, [%[src], #1, mul vl]         \n"
      "incb    %[src], all, mul #2                        \n"
      "tbl     z0.b, {z0.b}, z31.b                        \n"
      "tbl     z1.b, {z1.b}, z31.b                        \n"
      "subs    %w[width], %w[width], %w[vl], lsl #1       \n"
      "st1b    {z0.b}, p1, [%[dst]]                       \n"
      "add     %[dst], %[dst], %x[vl_mul3]                \n"
      "st1b    {z1.b}, p1, [%[dst]]                       \n"
      "add     %[dst], %[dst], %x[vl_mul3]                \n"
      "b.ge    1b                                         \n"

      "2:                                                 \n"
      "adds    %w[width], %w[width], %w[vl], lsl #1       \n"
      "b.eq    99f                                        \n"

      // Calculate predicates for the final iteration to deal with the tail.
      "add     %w[rem_mul3], %w[width], %w[width], lsl #1 \n"
      "whilelt p0.s, wzr, %w[width]                       \n"
      "whilelt p1.b, wzr, %w[rem_mul3]                    \n"
      "whilelt p2.s, %w[vl], %w[width]                    \n"
      "whilelt p3.b, %w[vl_mul3], %w[rem_mul3]            \n"
      "ld1w    {z0.s}, p0/z, [%[src]]                     \n"
      "ld1w    {z1.s}, p2/z, [%[src], #1, mul vl]         \n"
      "tbl     z0.b, {z0.b}, z31.b                        \n"
      "tbl     z1.b, {z1.b}, z31.b                        \n"
      "st1b    {z0.b}, p1, [%[dst]]                       \n"
      "add     %[dst], %[dst], %x[vl_mul3]                \n"
      "st1b    {z1.b}, p3, [%[dst]]                       \n"

      "99:                                                \n"
      : [src] "+r"(src_argb),       // %[src]
        [dst] "+r"(dst_xyz),        // %[dst]
        [width] "+r"(width),        // %[width]
        [rem_mul3] "=&r"(rem_mul3)  // %[rem_mul3]
      : [indices] "r"(indices),     // %[indices]
        [vl_mul3] "r"(vl_mul3),     // %[vl_mul3]
        [vl] "r"(vl)                // %[vl]
      : "cc", "memory", "z0", "z1", "z31", "p0", "p1", "p2", "p3");
}

static const uint8_t kARGBToRGB24RowIndices[] = {
    0,   1,   2,   4,   5,   6,   8,   9,   10,  12,  13,  14,  16,  17,  18,
    20,  21,  22,  24,  25,  26,  28,  29,  30,  32,  33,  34,  36,  37,  38,
    40,  41,  42,  44,  45,  46,  48,  49,  50,  52,  53,  54,  56,  57,  58,
    60,  61,  62,  64,  65,  66,  68,  69,  70,  72,  73,  74,  76,  77,  78,
    80,  81,  82,  84,  85,  86,  88,  89,  90,  92,  93,  94,  96,  97,  98,
    100, 101, 102, 104, 105, 106, 108, 109, 110, 112, 113, 114, 116, 117, 118,
    120, 121, 122, 124, 125, 126, 128, 129, 130, 132, 133, 134, 136, 137, 138,
    140, 141, 142, 144, 145, 146, 148, 149, 150, 152, 153, 154, 156, 157, 158,
    160, 161, 162, 164, 165, 166, 168, 169, 170, 172, 173, 174, 176, 177, 178,
    180, 181, 182, 184, 185, 186, 188, 189, 190, 192, 193, 194, 196, 197, 198,
    200, 201, 202, 204, 205, 206, 208, 209, 210, 212, 213, 214, 216, 217, 218,
    220, 221, 222, 224, 225, 226, 228, 229, 230, 232, 233, 234, 236, 237, 238,
    240, 241, 242, 244, 245, 246, 248, 249, 250, 252, 253, 254,
};

static const uint8_t kARGBToRAWRowIndices[] = {
    2,   1,   0,   6,   5,   4,   10,  9,   8,   14,  13,  12,  18,  17,  16,
    22,  21,  20,  26,  25,  24,  30,  29,  28,  34,  33,  32,  38,  37,  36,
    42,  41,  40,  46,  45,  44,  50,  49,  48,  54,  53,  52,  58,  57,  56,
    62,  61,  60,  66,  65,  64,  70,  69,  68,  74,  73,  72,  78,  77,  76,
    82,  81,  80,  86,  85,  84,  90,  89,  88,  94,  93,  92,  98,  97,  96,
    102, 101, 100, 106, 105, 104, 110, 109, 108, 114, 113, 112, 118, 117, 116,
    122, 121, 120, 126, 125, 124, 130, 129, 128, 134, 133, 132, 138, 137, 136,
    142, 141, 140, 146, 145, 144, 150, 149, 148, 154, 153, 152, 158, 157, 156,
    162, 161, 160, 166, 165, 164, 170, 169, 168, 174, 173, 172, 178, 177, 176,
    182, 181, 180, 186, 185, 184, 190, 189, 188, 194, 193, 192, 198, 197, 196,
    202, 201, 200, 206, 205, 204, 210, 209, 208, 214, 213, 212, 218, 217, 216,
    222, 221, 220, 226, 225, 224, 230, 229, 228, 234, 233, 232, 238, 237, 236,
    242, 241, 240, 246, 245, 244, 250, 249, 248, 254, 253, 252,
};

void ARGBToRGB24Row_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
  ARGBToXYZRow_SVE2(src_argb, dst_rgb, width, kARGBToRGB24RowIndices);
}

void ARGBToRAWRow_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
  ARGBToXYZRow_SVE2(src_argb, dst_rgb, width, kARGBToRAWRowIndices);
}

void DivideRow_16_SVE2(const uint16_t* src_y,
                       uint16_t* dst_y,
                       int scale,
                       int width) {
  uint64_t vl;
  asm volatile(
      "cnth   %x[vl]                                     \n"
      "dup    z0.h, %w[scale]                            \n"
      "subs   %w[width], %w[width], %w[vl], lsl #1       \n"
      "b.le    2f                                        \n"

      // Run bulk of computation with the same predicates to avoid predicate
      // generation overhead.
      "ptrue   p0.h                                      \n"
      "1:                                                \n"
      "ld1h   {z1.h}, p0/z, [%[src]]                     \n"
      "ld1h   {z2.h}, p0/z, [%[src], #1, mul vl]         \n"
      "incb   %[src], all, mul #2                        \n"
      "umulh  z1.h, z1.h, z0.h                           \n"
      "umulh  z2.h, z2.h, z0.h                           \n"
      "subs   %w[width], %w[width], %w[vl], lsl #1       \n"
      "st1h   {z1.h}, p0, [%[dst]]                       \n"
      "st1h   {z2.h}, p0, [%[dst], #1, mul vl]           \n"
      "incb   %[dst], all, mul #2                        \n"
      "b.gt    1b                                        \n"

      "2:                                                \n"
      "adds     %w[width], %w[width], %w[vl], lsl #1     \n"
      "b.eq     99f                                      \n"

      // Calculate a pair of predicates for the final iteration to deal with
      // the tail.
      "whilelt p0.h, wzr, %w[width]                      \n"
      "whilelt p1.h, %w[vl], %w[width]                   \n"
      "ld1h   {z1.h}, p0/z, [%[src]]                     \n"
      "ld1h   {z2.h}, p1/z, [%[src], #1, mul vl]         \n"
      "umulh  z1.h, z1.h, z0.h                           \n"
      "umulh  z2.h, z2.h, z0.h                           \n"
      "st1h   {z1.h}, p0, [%[dst]]                       \n"
      "st1h   {z2.h}, p1, [%[dst], #1, mul vl]           \n"

      "99:                                               \n"
      : [src] "+r"(src_y),    // %[src]
        [dst] "+r"(dst_y),    // %[dst]
        [width] "+r"(width),  // %[width]
        [vl] "=&r"(vl)        // %[vl]
      : [scale] "r"(scale)    // %[scale]
      : "cc", "memory", "z0", "z1", "z2", "p0", "p1");
}

#define HALFFLOAT_SVE                                    \
  "scvtf       z0.s, p0/m, z0.s                      \n" \
  "scvtf       z1.s, p0/m, z1.s                      \n" \
  "scvtf       z2.s, p0/m, z2.s                      \n" \
  "scvtf       z3.s, p0/m, z3.s                      \n" \
  "fmul        z0.s, z0.s, z4.s                      \n" \
  "fmul        z1.s, z1.s, z4.s                      \n" \
  "fmul        z2.s, z2.s, z4.s                      \n" \
  "fmul        z3.s, z3.s, z4.s                      \n" \
  "uqshrnb     z0.h, z0.s, #13                       \n" \
  "uqshrnb     z1.h, z1.s, #13                       \n" \
  "uqshrnb     z2.h, z2.s, #13                       \n" \
  "uqshrnb     z3.h, z3.s, #13                       \n"

void HalfFloatRow_SVE2(const uint16_t* src,
                       uint16_t* dst,
                       float scale,
                       int width) {
  uint64_t vl;
  asm("cntw %x0" : "=r"(vl));
  asm volatile(
      "mov         z4.s, %s[scale]                       \n"
      "subs        %w[width], %w[width], %w[vl], lsl #2  \n"
      "b.lt        2f                                    \n"

      // Run bulk of computation with all-true predicates to avoid predicate
      // generation overhead.
      "ptrue       p0.s                                  \n"
      "1:                                                \n"
      "ld1h        {z0.s}, p0/z, [%[src]]                \n"
      "ld1h        {z1.s}, p0/z, [%[src], #1, mul vl]    \n"
      "ld1h        {z2.s}, p0/z, [%[src], #2, mul vl]    \n"
      "ld1h        {z3.s}, p0/z, [%[src], #3, mul vl]    \n"
      "incb        %[src], all, mul #2                   \n"  //
      HALFFLOAT_SVE
      "subs        %w[width], %w[width], %w[vl], lsl #2  \n"
      "st1h        {z0.s}, p0, [%[dst]]                  \n"
      "st1h        {z1.s}, p0, [%[dst], #1, mul vl]      \n"
      "st1h        {z2.s}, p0, [%[dst], #2, mul vl]      \n"
      "st1h        {z3.s}, p0, [%[dst], #3, mul vl]      \n"
      "incb        %[dst], all, mul #2                   \n"
      "b.ge        1b                                    \n"

      "2:                                                \n"
      "adds     %w[width], %w[width], %w[vl], lsl #2     \n"
      "b.eq     99f                                      \n"

      // Calculate predicates for the final iteration to deal with the tail.
      "whilelt     p0.s, wzr, %w[width]                  \n"
      "whilelt     p1.s, %w[vl], %w[width]               \n"
      "whilelt     p2.s, %w[vl2], %w[width]              \n"
      "whilelt     p3.s, %w[vl3], %w[width]              \n"
      "ld1h        {z0.s}, p0/z, [%[src]]                \n"
      "ld1h        {z1.s}, p1/z, [%[src], #1, mul vl]    \n"
      "ld1h        {z2.s}, p2/z, [%[src], #2, mul vl]    \n"
      "ld1h        {z3.s}, p3/z, [%[src], #3, mul vl]    \n"  //
      HALFFLOAT_SVE
      "st1h        {z0.s}, p0, [%[dst]]                  \n"
      "st1h        {z1.s}, p1, [%[dst], #1, mul vl]      \n"
      "st1h        {z2.s}, p2, [%[dst], #2, mul vl]      \n"
      "st1h        {z3.s}, p3, [%[dst], #3, mul vl]      \n"

      "99:                                               \n"
      : [src] "+r"(src),                        // %[src]
        [dst] "+r"(dst),                        // %[dst]
        [width] "+r"(width)                     // %[width]
      : [vl] "r"(vl),                           // %[vl]
        [vl2] "r"(vl * 2),                      // %[vl2]
        [vl3] "r"(vl * 3),                      // %[vl3]
        [scale] "w"(scale * 1.9259299444e-34f)  // %[scale]
      : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0", "p1", "p2", "p3");
}

void HalfFloat1Row_SVE2(const uint16_t* src,
                        uint16_t* dst,
                        float scale,
                        int width) {
  uint64_t vl;
  asm volatile(
      "cnth        %x[vl]                                \n"
      "subs        %w[width], %w[width], %w[vl], lsl #1  \n"
      "b.lt        2f                                    \n"

      // Run bulk of computation with all-true predicates to avoid predicate
      // generation overhead.
      "ptrue       p0.h                                  \n"
      "1:                                                \n"
      "ld1h        {z0.h}, p0/z, [%[src]]                \n"
      "ld1h        {z1.h}, p0/z, [%[src], #1, mul vl]    \n"
      "incb        %[src], all, mul #2                   \n"
      "ucvtf       z0.h, p0/m, z0.h                      \n"
      "ucvtf       z1.h, p0/m, z1.h                      \n"
      "subs        %w[width], %w[width], %w[vl], lsl #1  \n"
      "st1h        {z0.h}, p0, [%[dst]]                  \n"
      "st1h        {z1.h}, p0, [%[dst], #1, mul vl]      \n"
      "incb        %[dst], all, mul #2                   \n"
      "b.ge        1b                                    \n"

      "2:                                                \n"
      "adds     %w[width], %w[width], %w[vl], lsl #1     \n"
      "b.eq     99f                                      \n"

      // Calculate predicates for the final iteration to deal with the tail.
      "whilelt     p0.h, wzr, %w[width]                  \n"
      "whilelt     p1.h, %w[vl], %w[width]               \n"
      "ld1h        {z0.h}, p0/z, [%[src]]                \n"
      "ld1h        {z1.h}, p1/z, [%[src], #1, mul vl]    \n"
      "ucvtf       z0.h, p0/m, z0.h                      \n"
      "ucvtf       z1.h, p0/m, z1.h                      \n"
      "st1h        {z0.h}, p0, [%[dst]]                  \n"
      "st1h        {z1.h}, p1, [%[dst], #1, mul vl]      \n"

      "99:                                               \n"
      : [src] "+r"(src),      // %[src]
        [dst] "+r"(dst),      // %[dst]
        [width] "+r"(width),  // %[width]
        [vl] "=&r"(vl)        // %[vl]
      :
      : "cc", "memory", "z0", "z1", "p0", "p1");
}

void I210ToARGBRow_SVE2(const uint16_t* src_y,
                        const uint16_t* src_u,
                        const uint16_t* src_v,
                        uint8_t* dst_argb,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  I210ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
}

void I210AlphaToARGBRow_SVE2(const uint16_t* src_y,
                             const uint16_t* src_u,
                             const uint16_t* src_v,
                             const uint16_t* src_a,
                             uint8_t* dst_argb,
                             const struct YuvConstants* yuvconstants,
                             int width) {
  I210AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
                            width);
}

void I210ToAR30Row_SVE2(const uint16_t* src_y,
                        const uint16_t* src_u,
                        const uint16_t* src_v,
                        uint8_t* dst_ar30,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  I210ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
}

void P210ToARGBRow_SVE2(const uint16_t* src_y,
                        const uint16_t* src_uv,
                        uint8_t* dst_argb,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  P210ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
}

void P210ToAR30Row_SVE2(const uint16_t* src_y,
                        const uint16_t* src_uv,
                        uint8_t* dst_ar30,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  P210ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width);
}

void I410ToARGBRow_SVE2(const uint16_t* src_y,
                        const uint16_t* src_u,
                        const uint16_t* src_v,
                        uint8_t* dst_argb,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  I410ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
}

void I410AlphaToARGBRow_SVE2(const uint16_t* src_y,
                             const uint16_t* src_u,
                             const uint16_t* src_v,
                             const uint16_t* src_a,
                             uint8_t* dst_argb,
                             const struct YuvConstants* yuvconstants,
                             int width) {
  I410AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
                            width);
}

void I410ToAR30Row_SVE2(const uint16_t* src_y,
                        const uint16_t* src_u,
                        const uint16_t* src_v,
                        uint8_t* dst_ar30,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  I410ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
}

void P410ToARGBRow_SVE2(const uint16_t* src_y,
                        const uint16_t* src_uv,
                        uint8_t* dst_argb,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  P410ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
}

void P410ToAR30Row_SVE2(const uint16_t* src_y,
                        const uint16_t* src_uv,
                        uint8_t* dst_ar30,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  P410ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width);
}

void I212ToAR30Row_SVE2(const uint16_t* src_y,
                        const uint16_t* src_u,
                        const uint16_t* src_v,
                        uint8_t* dst_ar30,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  I212ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
}

void I212ToARGBRow_SVE2(const uint16_t* src_y,
                        const uint16_t* src_u,
                        const uint16_t* src_v,
                        uint8_t* dst_argb,
                        const struct YuvConstants* yuvconstants,
                        int width) {
  I212ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
}

void Convert8To8Row_SVE2(const uint8_t* src_y,
                         uint8_t* dst_y,
                         int scale,
                         int bias,
                         int width) {
  Convert8To8Row_SVE_SC(src_y, dst_y, scale, bias, width);
}

void MergeUVRow_SVE2(const uint8_t* src_u,
                     const uint8_t* src_v,
                     uint8_t* dst_uv,
                     int width) {
  int vl;
  asm volatile(
      "cntb    %x[vl]                                   \n"
      "subs    %w[width], %w[width], %w[vl]             \n"
      "b.lt    2f                                       \n"

      "ptrue   p0.b                                     \n"
      "1:                                               \n"
      "ld1b    {z1.b}, p0/z, [%[src_u]]                 \n"
      "ld1b    {z2.b}, p0/z, [%[src_v]]                 \n"
      "incb    %[src_u]                                 \n"
      "incb    %[src_v]                                 \n"
      "subs    %w[width], %w[width], %w[vl]             \n"
      "st2b    {z1.b, z2.b}, p0, [%[dst_uv]]            \n"
      "incb    %[dst_uv], all, mul #2                   \n"
      "b.ge    1b                                       \n"

      "2:                                               \n"
      "adds    %w[width], %w[width], %w[vl]             \n"
      "b.eq    99f                                      \n"

      "whilelt p0.b, wzr, %w[width]                     \n"
      "ld1b    {z1.b}, p0/z, [%[src_u]]                 \n"
      "ld1b    {z2.b}, p0/z, [%[src_v]]                 \n"
      "st2b    {z1.b, z2.b}, p0, [%[dst_uv]]            \n"

      "99:                                              \n"
      : [src_u] "+r"(src_u),    // %[src_u]
        [src_v] "+r"(src_v),    // %[src_v]
        [dst_uv] "+r"(dst_uv),  // %[dst_uv]
        [width] "+r"(width),    // %[width]
        [vl] "=&r"(vl)          // %[vl]
      :
      : "memory", "cc", "z1", "z2", "p0");
}

void SplitUVRow_SVE2(const uint8_t* src_uv,
                     uint8_t* dst_u,
                     uint8_t* dst_v,
                     int width) {
  int vl;
  asm volatile(
      "cntb    %x[vl]                                   \n"
      "subs    %w[width], %w[width], %w[vl]             \n"
      "b.lt    2f                                       \n"

      "ptrue   p0.b                                     \n"
      "1:                                               \n"
      "ld2b    {z1.b, z2.b}, p0/z, [%[src_uv]]          \n"
      "incb    %[src_uv], all, mul #2                   \n"
      "subs    %w[width], %w[width], %w[vl]             \n"
      "st1b    {z1.b}, p0, [%[dst_u]]                   \n"
      "st1b    {z2.b}, p0, [%[dst_v]]                   \n"
      "incb    %[dst_u]                                 \n"
      "incb    %[dst_v]                                 \n"
      "b.ge    1b                                       \n"

      "2:                                               \n"
      "adds    %w[width], %w[width], %w[vl]             \n"
      "b.eq    99f                                      \n"

      "whilelt p0.b, wzr, %w[width]                     \n"
      "ld2b    {z1.b, z2.b}, p0/z, [%[src_uv]]          \n"
      "st1b    {z1.b}, p0, [%[dst_u]]                   \n"
      "st1b    {z2.b}, p0, [%[dst_v]]                   \n"

      "99:                                              \n"
      : [src_uv] "+r"(src_uv),  // %[src_uv]
        [dst_u] "+r"(dst_u),    // %[dst_u]
        [dst_v] "+r"(dst_v),    // %[dst_v]
        [width] "+r"(width),    // %[width]
        [vl] "=&r"(vl)          // %[vl]
      :
      : "memory", "cc", "z1", "z2", "p0");
}

void HalfMergeUVRow_SVE2(const uint8_t* src_u,
                         int src_stride_u,
                         const uint8_t* src_v,
                         int src_stride_v,
                         uint8_t* dst_uv,
                         int width) {
  const uint8_t* src_u1 = src_u + src_stride_u;
  const uint8_t* src_v1 = src_v + src_stride_v;
  int vl;
  asm volatile(
      "cntb     %x[vl]                                  \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "b.lt     2f                                      \n"

      "ptrue    p0.b                                    \n"
      "1:                                               \n"
      "ld1b     {z1.b}, p0/z, [%[src_u]]                \n"
      "ld1b     {z2.b}, p0/z, [%[src_u1]]               \n"
      "ld1b     {z3.b}, p0/z, [%[src_v]]                \n"
      "ld1b     {z4.b}, p0/z, [%[src_v1]]               \n"
      "incb     %[src_u]                                \n"
      "incb     %[src_u1]                               \n"
      "incb     %[src_v]                                \n"
      "incb     %[src_v1]                               \n"
      "urhadd   z1.b, p0/m, z1.b, z2.b                  \n"
      "urhadd   z3.b, p0/m, z3.b, z4.b                  \n"
      "mov      z2.b, p0/m, z3.b                        \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "st2b     {z1.b, z2.b}, p0, [%[dst_uv]]           \n"
      "incb     %[dst_uv], all, mul #2                  \n"
      "b.ge     1b                                      \n"

      "2:                                               \n"
      "adds     %w[width], %w[width], %w[vl]            \n"
      "b.eq     99f                                     \n"

      "whilelt  p0.b, wzr, %w[width]                    \n"
      "ld1b     {z1.b}, p0/z, [%[src_u]]                \n"
      "ld1b     {z2.b}, p0/z, [%[src_u1]]               \n"
      "ld1b     {z3.b}, p0/z, [%[src_v]]                \n"
      "ld1b     {z4.b}, p0/z, [%[src_v1]]               \n"
      "urhadd   z1.b, p0/m, z1.b, z2.b                  \n"
      "urhadd   z3.b, p0/m, z3.b, z4.b                  \n"
      "mov      z2.b, p0/m, z3.b                        \n"
      "st2b     {z1.b, z2.b}, p0, [%[dst_uv]]           \n"

      "99:                                              \n"
      : [src_u] "+r"(src_u),      // %[src_u]
        [src_u1] "+r"(src_u1),    // %[src_u1]
        [src_v] "+r"(src_v),      // %[src_v]
        [src_v1] "+r"(src_v1),    // %[src_v1]
        [dst_uv] "+r"(dst_uv),    // %[dst_uv]
        [width] "+r"(width),      // %[width]
        [vl] "=&r"(vl)            // %[vl]
      :
      : "cc", "memory", "z1", "z2", "z3", "z4", "p0");
}

void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) {
  int vl;
  int vl2;
  asm volatile(
      "cntb     %x[vl]                                  \n"
      "lsl      %w[vl2], %w[vl], #1                     \n"
      "subs     %w[width], %w[width], %w[vl2]           \n"
      "b.lt     2f                                      \n"

      "ptrue    p0.b                                    \n"
      "1:                                               \n"
      "ld1b     {z0.b}, p0/z, [%[src], #0, mul vl]      \n"
      "ld1b     {z1.b}, p0/z, [%[src], #1, mul vl]      \n"
      "incb     %[src], all, mul #2                     \n"
      "subs     %w[width], %w[width], %w[vl2]           \n"
      "st1b     {z0.b}, p0, [%[dst], #0, mul vl]        \n"
      "st1b     {z1.b}, p0, [%[dst], #1, mul vl]        \n"
      "incb     %[dst], all, mul #2                     \n"
      "b.gt     1b                                      \n"

      "2:                                               \n"
      "adds     %w[width], %w[width], %w[vl2]           \n"
      "b.eq     99f                                     \n"

      "3:                                               \n"
      "whilelt  p0.b, wzr, %w[width]                    \n"
      "ld1b     {z0.b}, p0/z, [%[src]]                  \n"
      "incb     %[src]                                  \n"
      "st1b     {z0.b}, p0, [%[dst]]                    \n"
      "incb     %[dst]                                  \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "b.gt     3b                                      \n"

      "99:                                              \n"
      : [src] "+r"(src),
        [dst] "+r"(dst),
        [width] "+r"(width),
        [vl] "=&r"(vl),
        [vl2] "=&r"(vl2)
      :
      : "cc", "memory", "z0", "z1", "p0");
}

void InterpolateRow_SVE2(uint8_t* dst_ptr,
                         const uint8_t* src_ptr,
                         ptrdiff_t src_stride,
                         int width,
                         int source_y_fraction) {
  int y1_fraction = source_y_fraction;
  int y0_fraction = 256 - y1_fraction;
  const uint8_t* src_ptr1 = src_ptr + src_stride;

  if (y0_fraction == 0) {
    CopyRow_SVE2(src_ptr1, dst_ptr, width);
    return;
  }
  if (y0_fraction == 256) {
    CopyRow_SVE2(src_ptr, dst_ptr, width);
    return;
  }
  if (y0_fraction == 128) {
    int vl;
    asm volatile(
        "cntb     %x[vl]                                  \n"
        "subs     %w[width], %w[width], %w[vl]            \n"
        "b.lt     2f                                      \n"

        "ptrue    p0.b                                    \n"
        "1:                                               \n"
        "ld1b     {z2.b}, p0/z, [%[src_ptr]]              \n"
        "ld1b     {z3.b}, p0/z, [%[src_ptr1]]             \n"
        "incb     %[src_ptr]                              \n"
        "incb     %[src_ptr1]                             \n"
        "subs     %w[width], %w[width], %w[vl]            \n"
        "urhadd   z2.b, p0/m, z2.b, z3.b                  \n"
        "st1b     {z2.b}, p0, [%[dst_ptr]]                \n"
        "incb     %[dst_ptr]                              \n"
        "b.ge     1b                                      \n"

        "2:                                               \n"
        "adds     %w[width], %w[width], %w[vl]            \n"
        "b.eq     99f                                     \n"

        "whilelt  p0.b, wzr, %w[width]                    \n"
        "ld1b     {z2.b}, p0/z, [%[src_ptr]]              \n"
        "ld1b     {z3.b}, p0/z, [%[src_ptr1]]             \n"
        "urhadd   z2.b, p0/m, z2.b, z3.b                  \n"
        "st1b     {z2.b}, p0, [%[dst_ptr]]                \n"

        "99:                                              \n"
        : [src_ptr] "+r"(src_ptr),
          [src_ptr1] "+r"(src_ptr1),
          [dst_ptr] "+r"(dst_ptr),
          [width] "+r"(width),
          [vl] "=&r"(vl)
        :
        : "cc", "memory", "z2", "z3", "p0");
    return;
  }

  int vl;
  asm volatile(
      "cntb     %x[vl]                                  \n"
      "dup      z0.b, %w[y0_fraction]                   \n"
      "dup      z1.b, %w[y1_fraction]                   \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "b.lt     2f                                      \n"

      "ptrue    p0.b                                    \n"
      "1:                                               \n"
      "ld1b     {z2.b}, p0/z, [%[src_ptr]]              \n"
      "ld1b     {z3.b}, p0/z, [%[src_ptr1]]             \n"
      "incb     %[src_ptr]                              \n"
      "incb     %[src_ptr1]                             \n"
      "umullb   z4.h, z2.b, z0.b                        \n"
      "umullt   z2.h, z2.b, z0.b                        \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "umlalb   z4.h, z3.b, z1.b                        \n"
      "umlalt   z2.h, z3.b, z1.b                        \n"
      "rshrnb   z3.b, z4.h, #8                          \n"
      "rshrnt   z3.b, z2.h, #8                          \n"
      "st1b     {z3.b}, p0, [%[dst_ptr]]                \n"
      "incb     %[dst_ptr]                              \n"
      "b.ge     1b                                      \n"

      "2:                                               \n"
      "adds     %w[width], %w[width], %w[vl]            \n"
      "b.eq     99f                                     \n"

      "whilelt  p0.b, wzr, %w[width]                    \n"
      "ld1b     {z2.b}, p0/z, [%[src_ptr]]              \n"
      "ld1b     {z3.b}, p0/z, [%[src_ptr1]]             \n"
      "umullb   z4.h, z2.b, z0.b                        \n"
      "umullt   z2.h, z2.b, z0.b                        \n"
      "umlalb   z4.h, z3.b, z1.b                        \n"
      "umlalt   z2.h, z3.b, z1.b                        \n"
      "rshrnb   z3.b, z4.h, #8                          \n"
      "rshrnt   z3.b, z2.h, #8                          \n"
      "st1b     {z3.b}, p0, [%[dst_ptr]]                \n"

      "99:                                              \n"
      : [src_ptr] "+r"(src_ptr),         // %[src_ptr]
        [src_ptr1] "+r"(src_ptr1),       // %[src_ptr1]
        [dst_ptr] "+r"(dst_ptr),         // %[dst_ptr]
        [width] "+r"(width),             // %[width]
        [vl] "=&r"(vl)                   // %[vl]
      : [y0_fraction] "r"(y0_fraction),  // %[y0_fraction]
        [y1_fraction] "r"(y1_fraction)   // %[y1_fraction]
      : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0");
}

void YUY2ToYRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
  int vl;
  asm volatile(
      "cntb     %x[vl]                                  \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "b.lt     2f                                      \n"

      "ptrue    p0.b                                    \n"
      "1:                                               \n"
      "ld2b     {z0.b, z1.b}, p0/z, [%[src_yuy2]]       \n"
      "incb     %[src_yuy2], all, mul #2                \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "st1b     {z0.b}, p0, [%[dst_y]]                  \n"
      "incb     %[dst_y]                                \n"
      "b.ge     1b                                      \n"

      "2:                                               \n"
      "adds     %w[width], %w[width], %w[vl]            \n"
      "b.eq     99f                                     \n"

      "whilelt  p0.b, wzr, %w[width]                    \n"
      "ld2b     {z0.b, z1.b}, p0/z, [%[src_yuy2]]       \n"
      "st1b     {z0.b}, p0, [%[dst_y]]                  \n"

      "99:                                              \n"
      : [src_yuy2] "+r"(src_yuy2),  // %[src_yuy2]
        [dst_y] "+r"(dst_y),        // %[dst_y]
        [width] "+r"(width),        // %[width]
        [vl] "=&r"(vl)              // %[vl]
      :
      : "memory", "cc", "z0", "z1", "p0");
}

void UYVYToYRow_SVE2(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
  int vl;
  asm volatile(
      "cntb     %x[vl]                                  \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "b.lt     2f                                      \n"

      "ptrue    p0.b                                    \n"
      "1:                                               \n"
      "ld2b     {z0.b, z1.b}, p0/z, [%[src_uyvy]]       \n"
      "incb     %[src_uyvy], all, mul #2                \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "st1b     {z1.b}, p0, [%[dst_y]]                  \n"
      "incb     %[dst_y]                                \n"
      "b.ge     1b                                      \n"

      "2:                                               \n"
      "adds     %w[width], %w[width], %w[vl]            \n"
      "b.eq     99f                                     \n"

      "whilelt  p0.b, wzr, %w[width]                    \n"
      "ld2b     {z0.b, z1.b}, p0/z, [%[src_uyvy]]       \n"
      "st1b     {z1.b}, p0, [%[dst_y]]                  \n"

      "99:                                              \n"
      : [src_uyvy] "+r"(src_uyvy),  // %[src_uyvy]
        [dst_y] "+r"(dst_y),        // %[dst_y]
        [width] "+r"(width),        // %[width]
        [vl] "=&r"(vl)              // %[vl]
      :
      : "memory", "cc", "z0", "z1", "p0");
}

void YUY2ToUVRow_SVE2(const uint8_t* src_yuy2,
                      int src_stride_yuy2,
                      uint8_t* dst_u,
                      uint8_t* dst_v,
                      int width) {
  const uint8_t* src_yuy2_1 = src_yuy2 + src_stride_yuy2;
  int vl;
  int vl_half;
  int width_half;
  if (src_stride_yuy2 == 0) {
    src_yuy2_1 = src_yuy2;
  }
  asm volatile(
      "cntb     %x[vl]                                  \n"
      "lsr      %w[vl_half], %w[vl], #1                 \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "b.lt     2f                                      \n"

      "ptrue    p0.b                                    \n"
      "whilelt  p1.b, wzr, %w[vl_half]                  \n"
      "1:                                               \n"
      "ld2b     {z0.b, z1.b}, p0/z, [%[src_yuy2]]       \n"
      "ld2b     {z4.b, z5.b}, p0/z, [%[src_yuy2_1]]     \n"
      "incb     %[src_yuy2], all, mul #2                \n"
      "incb     %[src_yuy2_1], all, mul #2              \n"
      "urhadd   z1.b, p0/m, z1.b, z5.b                  \n"
      "uzp1     z0.b, z1.b, z1.b                        \n"
      "uzp2     z2.b, z1.b, z1.b                        \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "st1b     {z0.b}, p1, [%[dst_u]]                  \n"
      "st1b     {z2.b}, p1, [%[dst_v]]                  \n"
      "add      %[dst_u], %[dst_u], %x[vl_half]         \n"
      "add      %[dst_v], %[dst_v], %x[vl_half]         \n"
      "b.ge     1b                                      \n"

      "2:                                               \n"
      "adds     %w[width], %w[width], %w[vl]            \n"
      "b.eq     99f                                     \n"

      "add      %w[width_half], %w[width], #1           \n"
      "lsr      %w[width_half], %w[width_half], #1      \n"
      "whilelt  p1.b, wzr, %w[width_half]               \n"
      "zip1     p0.b, p1.b, p1.b                        \n"
      "ld2b     {z0.b, z1.b}, p0/z, [%[src_yuy2]]       \n"
      "ld2b     {z4.b, z5.b}, p0/z, [%[src_yuy2_1]]     \n"
      "urhadd   z1.b, p0/m, z1.b, z5.b                  \n"
      "uzp1     z0.b, z1.b, z1.b                        \n"
      "uzp2     z2.b, z1.b, z1.b                        \n"
      "st1b     {z0.b}, p1, [%[dst_u]]                  \n"
      "st1b     {z2.b}, p1, [%[dst_v]]                  \n"

      "99:                                              \n"
      : [src_yuy2] "+r"(src_yuy2),      // %[src_yuy2]
        [src_yuy2_1] "+r"(src_yuy2_1),  // %[src_yuy2_1]
        [dst_u] "+r"(dst_u),            // %[dst_u]
        [dst_v] "+r"(dst_v),            // %[dst_v]
        [width] "+r"(width),            // %[width]
        [vl] "=&r"(vl),                 // %[vl]
        [vl_half] "=&r"(vl_half),       // %[vl_half]
        [width_half] "=&r"(width_half)  // %[width_half]
      :
      : "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1");
}

void UYVYToUVRow_SVE2(const uint8_t* src_uyvy,
                      int src_stride_uyvy,
                      uint8_t* dst_u,
                      uint8_t* dst_v,
                      int width) {
  const uint8_t* src_uyvy_1 = src_uyvy + src_stride_uyvy;
  int vl;
  int vl_half;
  int width_half;
  if (src_stride_uyvy == 0) {
    src_uyvy_1 = src_uyvy;
  }
  asm volatile(
      "cntb     %x[vl]                                  \n"
      "lsr      %w[vl_half], %w[vl], #1                 \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "b.lt     2f                                      \n"

      "ptrue    p0.b                                    \n"
      "whilelt  p1.b, wzr, %w[vl_half]                  \n"
      "1:                                               \n"
      "ld2b     {z0.b, z1.b}, p0/z, [%[src_uyvy]]       \n"
      "ld2b     {z4.b, z5.b}, p0/z, [%[src_uyvy_1]]     \n"
      "incb     %[src_uyvy], all, mul #2                \n"
      "incb     %[src_uyvy_1], all, mul #2              \n"
      "urhadd   z0.b, p0/m, z0.b, z4.b                  \n"
      "uzp1     z1.b, z0.b, z0.b                        \n"
      "uzp2     z2.b, z0.b, z0.b                        \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "st1b     {z1.b}, p1, [%[dst_u]]                  \n"
      "st1b     {z2.b}, p1, [%[dst_v]]                  \n"
      "add      %[dst_u], %[dst_u], %x[vl_half]         \n"
      "add      %[dst_v], %[dst_v], %x[vl_half]         \n"
      "b.ge     1b                                      \n"

      "2:                                               \n"
      "adds     %w[width], %w[width], %w[vl]            \n"
      "b.eq     99f                                     \n"

      "add      %w[width_half], %w[width], #1           \n"
      "lsr      %w[width_half], %w[width_half], #1      \n"
      "whilelt  p1.b, wzr, %w[width_half]               \n"
      "zip1     p0.b, p1.b, p1.b                        \n"
      "ld2b     {z0.b, z1.b}, p0/z, [%[src_uyvy]]       \n"
      "ld2b     {z4.b, z5.b}, p0/z, [%[src_uyvy_1]]     \n"
      "urhadd   z0.b, p0/m, z0.b, z4.b                  \n"
      "uzp1     z1.b, z0.b, z0.b                        \n"
      "uzp2     z2.b, z0.b, z0.b                        \n"
      "st1b     {z1.b}, p1, [%[dst_u]]                  \n"
      "st1b     {z2.b}, p1, [%[dst_v]]                  \n"

      "99:                                              \n"
      : [src_uyvy] "+r"(src_uyvy),      // %[src_uyvy]
        [src_uyvy_1] "+r"(src_uyvy_1),  // %[src_uyvy_1]
        [dst_u] "+r"(dst_u),            // %[dst_u]
        [dst_v] "+r"(dst_v),            // %[dst_v]
        [width] "+r"(width),            // %[width]
        [vl] "=&r"(vl),                 // %[vl]
        [vl_half] "=&r"(vl_half),       // %[vl_half]
        [width_half] "=&r"(width_half)  // %[width_half]
      :
      : "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1");
}

#ifdef HAS_CONVERT16TO8ROW_SVE2
// Disabled: NEON version is faster for 128 bit vectors.
void Convert16To8Row_SVE2(const uint16_t* src_y,
                          uint8_t* dst_y,
                          int scale,
                          int width) {
  const int shift = 23 - __builtin_clz((int32_t)scale);
  int vl;
  asm volatile(
      "cnth     %x[vl]                                  \n"
      "dup      z31.h, %w[shift]                        \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "b.lt     2f                                      \n"

      "ptrue    p0.h                                    \n"
      "1:                                               \n"
      "ld1h     {z2.h}, p0/z, [%[src_ptr]]              \n"
      "incb     %[src_ptr]                              \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "uqshl    z2.h, p0/m, z2.h, z31.h                 \n"
      "shrnb    z2.b, z2.h, #8                          \n"
      "st1b     {z2.h}, p0, [%[dst_ptr]]                \n"
      "inch     %[dst_ptr]                              \n"
      "b.ge     1b                                      \n"

      "2:                                               \n"
      "adds     %w[width], %w[width], %w[vl]            \n"
      "b.eq     99f                                     \n"

      "whilelt  p0.h, wzr, %w[width]                    \n"
      "ld1h     {z2.h}, p0/z, [%[src_ptr]]              \n"
      "uqshl    z2.h, p0/m, z2.h, z31.h                 \n"
      "shrnb    z2.b, z2.h, #8                          \n"
      "st1b     {z2.h}, p0, [%[dst_ptr]]                \n"

      "99:                                              \n"
      : [src_ptr] "+r"(src_y),  // %[src_ptr]
        [dst_ptr] "+r"(dst_y),  // %[dst_ptr]
        [width] "+r"(width),    // %[width]
        [vl] "=&r"(vl)          // %[vl]
      : [shift] "r"(shift)      // %[shift]
      : "cc", "memory", "z2", "z31", "p0");
}
#endif  // HAS_CONVERT16TO8ROW_SVE2

#ifdef HAS_HALFROW_16TO8_SVE2
// Disabled: NEON version is faster for 128 bit vectors.
void HalfRow_16To8_SVE2(const uint16_t* src_uv,
                        ptrdiff_t src_uv_stride,
                        uint8_t* dst_uv,
                        int scale,
                        int width) {
  const uint16_t* src_uv1 = src_uv + src_uv_stride;
  const int shift = 23 - __builtin_clz((int32_t)scale);
  int vl;
  asm volatile(
      "cnth     %x[vl]                                  \n"
      "dup      z31.h, %w[shift]                        \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "b.lt     2f                                      \n"

      "ptrue    p0.h                                    \n"
      "1:                                               \n"
      "ld1h     {z2.h}, p0/z, [%[src_ptr]]              \n"
      "ld1h     {z3.h}, p0/z, [%[src_ptr1]]             \n"
      "incb     %[src_ptr]                              \n"
      "incb     %[src_ptr1]                             \n"
      "urhadd   z2.h, p0/m, z2.h, z3.h                  \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "uqshl    z2.h, p0/m, z2.h, z31.h                 \n"
      "shrnb    z2.b, z2.h, #8                          \n"
      "st1b     {z2.h}, p0, [%[dst_ptr]]                \n"
      "inch     %[dst_ptr]                              \n"
      "b.ge     1b                                      \n"

      "2:                                               \n"
      "adds     %w[width], %w[width], %w[vl]            \n"
      "b.eq     99f                                     \n"

      "whilelt  p0.h, wzr, %w[width]                    \n"
      "ld1h     {z2.h}, p0/z, [%[src_ptr]]              \n"
      "ld1h     {z3.h}, p0/z, [%[src_ptr1]]             \n"
      "urhadd   z2.h, p0/m, z2.h, z3.h                  \n"
      "uqshl    z2.h, p0/m, z2.h, z31.h                 \n"
      "shrnb    z2.b, z2.h, #8                          \n"
      "st1b     {z2.h}, p0, [%[dst_ptr]]                \n"

      "99:                                              \n"
      : [src_ptr] "+r"(src_uv),    // %[src_ptr]
        [src_ptr1] "+r"(src_uv1),  // %[src_ptr1]
        [dst_ptr] "+r"(dst_uv),    // %[dst_ptr]
        [width] "+r"(width),       // %[width]
        [vl] "=&r"(vl)             // %[vl]
      : [shift] "r"(shift)         // %[shift]
      : "cc", "memory", "z2", "z3", "z31", "p0");
}
#endif  // HAS_HALFROW_16TO8_SVE2

#ifdef HAS_HALFWIDTHROW_16TO8_SVE2
// Disabled: NEON version is faster for 128 bit vectors.
void HalfWidthRow_16To8_SVE2(const uint16_t* src_uv,
                             ptrdiff_t src_uv_stride,
                             uint8_t* dst_uv,
                             int scale,
                             int width) {
  const uint16_t* src_uv1 = src_uv + src_uv_stride;
  const int shift = 23 - __builtin_clz((int32_t)scale);
  int vl;
  int vl_input;
  asm volatile(
      "cnth     %x[vl]                                  \n"
      "dup      z31.h, %w[shift]                        \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "b.lt     2f                                      \n"

      "ptrue    p0.h                                    \n"
      "1:                                               \n"
      "ld1h     {z0.h}, p0/z, [%[src_ptr]]              \n"
      "ld1h     {z1.h}, p0/z, [%[src_ptr], #1, mul vl]  \n"
      "ld1h     {z2.h}, p0/z, [%[src_ptr1]]             \n"
      "ld1h     {z3.h}, p0/z, [%[src_ptr1], #1, mul vl] \n"
      "incb     %[src_ptr], all, mul #4                 \n"
      "incb     %[src_ptr1], all, mul #4                \n"
      "uzp1     z4.h, z0.h, z1.h                        \n"
      "uzp2     z5.h, z0.h, z1.h                        \n"
      "uzp1     z6.h, z2.h, z3.h                        \n"
      "uzp2     z7.h, z2.h, z3.h                        \n"
      "uaddlb   z0.s, z4.h, z5.h                        \n"
      "uaddlt   z1.s, z4.h, z5.h                        \n"
      "uaddlb   z2.s, z6.h, z7.h                        \n"
      "uaddlt   z3.s, z6.h, z7.h                        \n"
      "add      z0.s, z0.s, z2.s                        \n"
      "add      z1.s, z1.s, z3.s                        \n"
      "rshrnb   z0.h, z0.s, #2                          \n"
      "rshrnt   z0.h, z1.s, #2                          \n"
      "subs     %w[width], %w[width], %w[vl]            \n"
      "uqshl    z0.h, p0/m, z0.h, z31.h                 \n"
      "shrnb    z0.b, z0.h, #8                          \n"
      "st1b     {z0.h}, p0, [%[dst_ptr]]                \n"
      "inch     %[dst_ptr]                              \n"
      "b.ge     1b                                      \n"

      "2:                                               \n"
      "adds     %w[width], %w[width], %w[vl]            \n"
      "b.eq     99f                                     \n"

      "lsl      %w[vl_input], %w[width], #1             \n"
      "whilelt  p0.h, wzr, %w[vl_input]                 \n"
      "whilelt  p1.h, %w[vl], %w[vl_input]              \n"
      "whilelt  p2.h, wzr, %w[width]                    \n"
      "ld1h     {z0.h}, p0/z, [%[src_ptr]]              \n"
      "ld1h     {z1.h}, p1/z, [%[src_ptr], #1, mul vl]  \n"
      "ld1h     {z2.h}, p0/z, [%[src_ptr1]]             \n"
      "ld1h     {z3.h}, p1/z, [%[src_ptr1], #1, mul vl] \n"
      "uzp1     z4.h, z0.h, z1.h                        \n"
      "uzp2     z5.h, z0.h, z1.h                        \n"
      "uzp1     z6.h, z2.h, z3.h                        \n"
      "uzp2     z7.h, z2.h, z3.h                        \n"
      "uaddlb   z0.s, z4.h, z5.h                        \n"
      "uaddlt   z1.s, z4.h, z5.h                        \n"
      "uaddlb   z2.s, z6.h, z7.h                        \n"
      "uaddlt   z3.s, z6.h, z7.h                        \n"
      "add      z0.s, z0.s, z2.s                        \n"
      "add      z1.s, z1.s, z3.s                        \n"
      "rshrnb   z0.h, z0.s, #2                          \n"
      "rshrnt   z0.h, z1.s, #2                          \n"
      "uqshl    z0.h, p2/m, z0.h, z31.h                 \n"
      "shrnb    z0.b, z0.h, #8                          \n"
      "st1b     {z0.h}, p2, [%[dst_ptr]]                \n"

      "99:                                              \n"
      : [src_ptr] "+r"(src_uv),        // %[src_ptr]
        [src_ptr1] "+r"(src_uv1),      // %[src_ptr1]
        [dst_ptr] "+r"(dst_uv),        // %[dst_ptr]
        [width] "+r"(width),           // %[width]
        [vl] "=&r"(vl),                // %[vl]
        [vl_input] "=&r"(vl_input)     // %[vl_input]
      : [shift] "r"(shift)             // %[shift]
      : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z31",
        "p0", "p1", "p2");
}
#endif  // HAS_HALFWIDTHROW_16TO8_SVE2

#endif  // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)

#ifdef __cplusplus
}  // extern "C"
}  // namespace libyuv
#endif
