1 /*
2 * Copyright (c) 2017-2021 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #pragma once
25
26 #ifdef __arm__
27
28 #include "transpose_interleave_common.hpp"
29
30 // Generic unblocked transposed 8x32-bit sized specialisation
31 template <>
32 template <typename T>
Transform(T * out,const T * const in,const int stride,const int x0,const int xmax,const int k0,const int kmax)33 void TransformImpl<8, 1, true, 4, 4, VLType::None>::Transform(
34 T* out, const T* const in, const int stride,
35 const int x0, const int xmax, const int k0, const int kmax
36 ) {
37 // Redirect to a 16x uint16_t specialisation
38 TransformImpl<16, 1, true, 2, 2, VLType::None>::Transform(
39 reinterpret_cast<uint16_t *>(out),
40 reinterpret_cast<const uint16_t *>(in),
41 stride*2, x0*2, xmax*2, k0, kmax
42 );
43 }
44
45 // Generic 16x16-bit sized specialisation
46 template <>
47 template <typename T>
Transform(T * out,const T * const in,const int stride,const int x0,const int xmax,const int k0,const int kmax)48 void TransformImpl<16, 1, true, 2, 2, VLType::None>::Transform(
49 T* out, const T* const in, const int stride,
50 const int x0, const int xmax, const int k0, const int kmax
51 ) {
52 // Redirect to a uint16_t specialisation
53 Transform(
54 reinterpret_cast<uint16_t *>(out),
55 reinterpret_cast<const uint16_t *>(in),
56 stride, x0, xmax, k0, kmax
57 );
58 }
59
60 // Specialised 16 x uint16_t version
61 template <>
moveblock_1x1(const uint16_t * & in0,uint16_t * out)62 void TransposeInterleaveCommon<16, uint16_t, uint16_t>::moveblock_1x1(const uint16_t *&in0, uint16_t *out) {
63 __asm volatile (
64 "VLD1.32 {d0-d3}, [%[in0]]!\n"
65 "VST1.32 {d0-d3}, [%[out]]\n"
66 ASM_PREFETCH("[%[in0], #192]")
67 : [in0] "+r" (in0),
68 [out] "+r" (out)
69 :
70 : "q0", "q1", "memory"
71 );
72 }
73
74 template <>
moveblock_1x2(const uint16_t * & in0,const uint16_t * & in1,uint16_t * out)75 void TransposeInterleaveCommon<16, uint16_t, uint16_t>::moveblock_1x2(const uint16_t *&in0, const uint16_t *&in1, uint16_t *out) {
76 __asm volatile (
77 "VLD1.32 {d0-d3}, [%[in0]]!\n"
78 "VST1.32 {d0-d3}, [%[out]]!\n"
79 ASM_PREFETCH("[%[in0], #192]")
80 "VLD1.32 {d0-d3}, [%[in1]]!\n"
81 "VST1.32 {d0-d3}, [%[out]]\n"
82 ASM_PREFETCH("[%[in1], #192]")
83 "SUB %[out], %[out], #32\n"
84 : [in0] "+r" (in0),
85 [in1] "+r" (in1),
86 [out] "+r" (out)
87 :
88 : "q0", "q1", "memory"
89 );
90 }
91
92 template <>
moveblock_1x4(const uint16_t * & in0,const uint16_t * & in1,const uint16_t * & in2,const uint16_t * & in3,uint16_t * out)93 void TransposeInterleaveCommon<16, uint16_t, uint16_t>::moveblock_1x4(const uint16_t *&in0, const uint16_t *&in1, const uint16_t *&in2, const uint16_t *&in3, uint16_t *out) {
94 __asm __volatile (
95 "VLD1.32 {d0-d3}, [%[in0]]!\n"
96 "VST1.32 {d0-d3}, [%[out]]!\n"
97 ASM_PREFETCH("[%[in0], #192]")
98 "VLD1.32 {d0-d3}, [%[in1]]!\n"
99 "VST1.32 {d0-d3}, [%[out]]!\n"
100 ASM_PREFETCH("[%[in1], #192]")
101 "VLD1.32 {d0-d3}, [%[in2]]!\n"
102 "VST1.32 {d0-d3}, [%[out]]!\n"
103 ASM_PREFETCH("[%[in2], #192]")
104 "VLD1.32 {d0-d3}, [%[in3]]!\n"
105 "VST1.32 {d0-d3}, [%[out]]\n"
106 ASM_PREFETCH("[%[in3], #192]")
107 "SUB %[out], %[out], #96\n"
108 : [in0] "+r" (in0),
109 [in1] "+r" (in1),
110 [in2] "+r" (in2),
111 [in3] "+r" (in3),
112 [out] "+r" (out)
113 :
114 : "q0", "q1", "memory"
115 );
116 }
117
118 template <>
119 template <>
Transform(uint16_t * out,const uint16_t * const in,const int stride,const int x0,const int xmax,const int k0,const int kmax)120 void TransformImpl<16, 1, true, 2, 2, VLType::None>::Transform(
121 uint16_t* out, const uint16_t* const in, const int stride,
122 const int x0, const int xmax, const int k0, const int kmax
123 ) {
124 TransposeInterleaveCommon<16, uint16_t, uint16_t>::Transform(out, in, stride, x0, xmax, k0, kmax);
125 }
126
127 #endif // __arm__
128