1/*
2 *  Copyright 2011 The LibYuv Project Authors. All rights reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS. All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "libyuv/cpu_id.h"
12
13#if defined(_MSC_VER)
14#include <intrin.h>  // For __cpuidex()
15#endif
16#if !defined(__pnacl__) && !defined(__CLR_VER) &&                           \
17    !defined(__native_client__) && (defined(_M_IX86) || defined(_M_X64)) && \
18    defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 160040219)
19#include <immintrin.h>  // For _xgetbv()
20#endif
21
22#if !defined(__native_client__)
23#include <stdlib.h>  // For getenv()
24#endif
25
26// For ArmCpuCaps() but unittested on all platforms
27#include <stdio.h>
28#include <string.h>
29
30#include "libyuv/basic_types.h"  // For CPU_X86
31
32#ifdef __cplusplus
33namespace libyuv {
34extern "C" {
35#endif
36
37// For functions that use the stack and have runtime checks for overflow,
38// use SAFEBUFFERS to avoid additional check.
39#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 160040219) && \
40    !defined(__clang__)
41#define SAFEBUFFERS __declspec(safebuffers)
42#else
43#define SAFEBUFFERS
44#endif
45
46// Low level cpuid for X86.
47#if (defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \
48     defined(__x86_64__)) &&                                     \
49    !defined(__pnacl__) && !defined(__CLR_VER)
50LIBYUV_API
51void CpuId(uint32 info_eax, uint32 info_ecx, uint32* cpu_info) {
52#if defined(_MSC_VER)
53// Visual C version uses intrinsic or inline x86 assembly.
54#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 160040219)
55  __cpuidex((int*)(cpu_info), info_eax, info_ecx);
56#elif defined(_M_IX86)
57  __asm {
58    mov        eax, info_eax
59    mov        ecx, info_ecx
60    mov        edi, cpu_info
61    cpuid
62    mov        [edi], eax
63    mov        [edi + 4], ebx
64    mov        [edi + 8], ecx
65    mov        [edi + 12], edx
66  }
67#else  // Visual C but not x86
68  if (info_ecx == 0) {
69    __cpuid((int*)(cpu_info), info_eax);
70  } else {
71    cpu_info[3] = cpu_info[2] = cpu_info[1] = cpu_info[0] = 0u;
72  }
73#endif
74// GCC version uses inline x86 assembly.
75#else  // defined(_MSC_VER)
76  uint32 info_ebx, info_edx;
77  asm volatile(
78#if defined(__i386__) && defined(__PIC__)
79      // Preserve ebx for fpic 32 bit.
80      "mov %%ebx, %%edi                          \n"
81      "cpuid                                     \n"
82      "xchg %%edi, %%ebx                         \n"
83      : "=D"(info_ebx),
84#else
85      "cpuid                                     \n"
86      : "=b"(info_ebx),
87#endif  //  defined( __i386__) && defined(__PIC__)
88        "+a"(info_eax), "+c"(info_ecx), "=d"(info_edx));
89  cpu_info[0] = info_eax;
90  cpu_info[1] = info_ebx;
91  cpu_info[2] = info_ecx;
92  cpu_info[3] = info_edx;
93#endif  // defined(_MSC_VER)
94}
95#else  // (defined(_M_IX86) || defined(_M_X64) ...
96LIBYUV_API
97void CpuId(uint32 eax, uint32 ecx, uint32* cpu_info) {
98  (void)eax;
99  (void)ecx;
100  cpu_info[0] = cpu_info[1] = cpu_info[2] = cpu_info[3] = 0;
101}
102#endif
103
104// For VS2010 and earlier emit can be used:
105//   _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0  // For VS2010 and earlier.
106//  __asm {
107//    xor        ecx, ecx    // xcr 0
108//    xgetbv
109//    mov        xcr0, eax
110//  }
111// For VS2013 and earlier 32 bit, the _xgetbv(0) optimizer produces bad code.
112// https://code.google.com/p/libyuv/issues/detail?id=529
113#if defined(_M_IX86) && (_MSC_VER < 1900)
114#pragma optimize("g", off)
115#endif
116#if (defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \
117     defined(__x86_64__)) &&                                     \
118    !defined(__pnacl__) && !defined(__CLR_VER) && !defined(__native_client__)
119// X86 CPUs have xgetbv to detect OS saves high parts of ymm registers.
120int GetXCR0() {
121  uint32 xcr0 = 0u;
122#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 160040219)
123  xcr0 = (uint32)(_xgetbv(0));  // VS2010 SP1 required.
124#elif defined(__i386__) || defined(__x86_64__)
125  asm(".byte 0x0f, 0x01, 0xd0" : "=a"(xcr0) : "c"(0) : "%edx");
126#endif  // defined(__i386__) || defined(__x86_64__)
127  return xcr0;
128}
129#else
130// xgetbv unavailable to query for OSSave support.  Return 0.
131#define GetXCR0() 0
132#endif  // defined(_M_IX86) || defined(_M_X64) ..
133// Return optimization to previous setting.
134#if defined(_M_IX86) && (_MSC_VER < 1900)
135#pragma optimize("g", on)
136#endif
137
138// based on libvpx arm_cpudetect.c
139// For Arm, but public to allow testing on any CPU
140LIBYUV_API SAFEBUFFERS int ArmCpuCaps(const char* cpuinfo_name) {
141  char cpuinfo_line[512];
142  FILE* f = fopen(cpuinfo_name, "r");
143  if (!f) {
144    // Assume Neon if /proc/cpuinfo is unavailable.
145    // This will occur for Chrome sandbox for Pepper or Render process.
146    return kCpuHasNEON;
147  }
148  while (fgets(cpuinfo_line, sizeof(cpuinfo_line) - 1, f)) {
149    if (memcmp(cpuinfo_line, "Features", 8) == 0) {
150      char* p = strstr(cpuinfo_line, " neon");
151      if (p && (p[5] == ' ' || p[5] == '\n')) {
152        fclose(f);
153        return kCpuHasNEON;
154      }
155      // aarch64 uses asimd for Neon.
156      p = strstr(cpuinfo_line, " asimd");
157      if (p && (p[6] == ' ' || p[6] == '\n')) {
158        fclose(f);
159        return kCpuHasNEON;
160      }
161    }
162  }
163  fclose(f);
164  return 0;
165}
166
167LIBYUV_API SAFEBUFFERS int MipsCpuCaps(const char* cpuinfo_name,
168                                       const char ase[]) {
169  char cpuinfo_line[512];
170  int len = (int)strlen(ase);
171  FILE* f = fopen(cpuinfo_name, "r");
172  if (!f) {
173    // ase enabled if /proc/cpuinfo is unavailable.
174    if (strcmp(ase, " msa") == 0) {
175      return kCpuHasMSA;
176    }
177    return kCpuHasDSPR2;
178  }
179  while (fgets(cpuinfo_line, sizeof(cpuinfo_line) - 1, f)) {
180    if (memcmp(cpuinfo_line, "ASEs implemented", 16) == 0) {
181      char* p = strstr(cpuinfo_line, ase);
182      if (p && (p[len] == ' ' || p[len] == '\n')) {
183        fclose(f);
184        if (strcmp(ase, " msa") == 0) {
185          return kCpuHasMSA;
186        }
187        return kCpuHasDSPR2;
188      }
189    }
190  }
191  fclose(f);
192  return 0;
193}
194
195// CPU detect function for SIMD instruction sets.
196LIBYUV_API
197int cpu_info_ = 0;  // cpu_info is not initialized yet.
198
199// Test environment variable for disabling CPU features. Any non-zero value
200// to disable. Zero ignored to make it easy to set the variable on/off.
201#if !defined(__native_client__) && !defined(_M_ARM)
202
203static LIBYUV_BOOL TestEnv(const char* name) {
204  const char* var = getenv(name);
205  if (var) {
206    if (var[0] != '0') {
207      return LIBYUV_TRUE;
208    }
209  }
210  return LIBYUV_FALSE;
211}
212#else  // nacl does not support getenv().
213static LIBYUV_BOOL TestEnv(const char*) {
214  return LIBYUV_FALSE;
215}
216#endif
217
218LIBYUV_API SAFEBUFFERS int InitCpuFlags(void) {
219  int cpu_info = 0;
220#if !defined(__pnacl__) && !defined(__CLR_VER) && defined(CPU_X86)
221  uint32 cpu_info0[4] = {0, 0, 0, 0};
222  uint32 cpu_info1[4] = {0, 0, 0, 0};
223  uint32 cpu_info7[4] = {0, 0, 0, 0};
224  CpuId(0, 0, cpu_info0);
225  CpuId(1, 0, cpu_info1);
226  if (cpu_info0[0] >= 7) {
227    CpuId(7, 0, cpu_info7);
228  }
229  cpu_info = kCpuHasX86 | ((cpu_info1[3] & 0x04000000) ? kCpuHasSSE2 : 0) |
230             ((cpu_info1[2] & 0x00000200) ? kCpuHasSSSE3 : 0) |
231             ((cpu_info1[2] & 0x00080000) ? kCpuHasSSE41 : 0) |
232             ((cpu_info1[2] & 0x00100000) ? kCpuHasSSE42 : 0) |
233             ((cpu_info7[1] & 0x00000200) ? kCpuHasERMS : 0);
234
235  // AVX requires OS saves YMM registers.
236  if (((cpu_info1[2] & 0x1c000000) == 0x1c000000) &&  // AVX and OSXSave
237      ((GetXCR0() & 6) == 6)) {  // Test OS saves YMM registers
238    cpu_info |= kCpuHasAVX | ((cpu_info7[1] & 0x00000020) ? kCpuHasAVX2 : 0) |
239                ((cpu_info1[2] & 0x00001000) ? kCpuHasFMA3 : 0) |
240                ((cpu_info1[2] & 0x20000000) ? kCpuHasF16C : 0);
241
242    // Detect AVX512bw
243    if ((GetXCR0() & 0xe0) == 0xe0) {
244      cpu_info |= (cpu_info7[1] & 0x40000000) ? kCpuHasAVX3 : 0;
245    }
246  }
247
248  // Environment variable overrides for testing.
249  if (TestEnv("LIBYUV_DISABLE_X86")) {
250    cpu_info &= ~kCpuHasX86;
251  }
252  if (TestEnv("LIBYUV_DISABLE_SSE2")) {
253    cpu_info &= ~kCpuHasSSE2;
254  }
255  if (TestEnv("LIBYUV_DISABLE_SSSE3")) {
256    cpu_info &= ~kCpuHasSSSE3;
257  }
258  if (TestEnv("LIBYUV_DISABLE_SSE41")) {
259    cpu_info &= ~kCpuHasSSE41;
260  }
261  if (TestEnv("LIBYUV_DISABLE_SSE42")) {
262    cpu_info &= ~kCpuHasSSE42;
263  }
264  if (TestEnv("LIBYUV_DISABLE_AVX")) {
265    cpu_info &= ~kCpuHasAVX;
266  }
267  if (TestEnv("LIBYUV_DISABLE_AVX2")) {
268    cpu_info &= ~kCpuHasAVX2;
269  }
270  if (TestEnv("LIBYUV_DISABLE_ERMS")) {
271    cpu_info &= ~kCpuHasERMS;
272  }
273  if (TestEnv("LIBYUV_DISABLE_FMA3")) {
274    cpu_info &= ~kCpuHasFMA3;
275  }
276  if (TestEnv("LIBYUV_DISABLE_AVX3")) {
277    cpu_info &= ~kCpuHasAVX3;
278  }
279  if (TestEnv("LIBYUV_DISABLE_F16C")) {
280    cpu_info &= ~kCpuHasF16C;
281  }
282
283#endif
284#if defined(__mips__) && defined(__linux__)
285#if defined(__mips_dspr2)
286  cpu_info |= kCpuHasDSPR2;
287#endif
288#if defined(__mips_msa)
289  cpu_info = MipsCpuCaps("/proc/cpuinfo", " msa");
290#endif
291  cpu_info |= kCpuHasMIPS;
292  if (getenv("LIBYUV_DISABLE_DSPR2")) {
293    cpu_info &= ~kCpuHasDSPR2;
294  }
295  if (getenv("LIBYUV_DISABLE_MSA")) {
296    cpu_info &= ~kCpuHasMSA;
297  }
298#endif
299#if defined(__arm__) || defined(__aarch64__)
300// gcc -mfpu=neon defines __ARM_NEON__
301// __ARM_NEON__ generates code that requires Neon.  NaCL also requires Neon.
302// For Linux, /proc/cpuinfo can be tested but without that assume Neon.
303#if defined(__ARM_NEON__) || defined(__native_client__) || !defined(__linux__)
304  cpu_info = kCpuHasNEON;
305// For aarch64(arm64), /proc/cpuinfo's feature is not complete, e.g. no neon
306// flag in it.
307// So for aarch64, neon enabling is hard coded here.
308#endif
309#if defined(__aarch64__)
310  cpu_info = kCpuHasNEON;
311#else
312  // Linux arm parse text file for neon detect.
313  cpu_info = ArmCpuCaps("/proc/cpuinfo");
314#endif
315  cpu_info |= kCpuHasARM;
316  if (TestEnv("LIBYUV_DISABLE_NEON")) {
317    cpu_info &= ~kCpuHasNEON;
318  }
319#endif  // __arm__
320  if (TestEnv("LIBYUV_DISABLE_ASM")) {
321    cpu_info = 0;
322  }
323  cpu_info |= kCpuInitialized;
324  cpu_info_ = cpu_info;
325  return cpu_info;
326}
327
328// Note that use of this function is not thread safe.
329LIBYUV_API
330void MaskCpuFlags(int enable_flags) {
331  cpu_info_ = InitCpuFlags() & enable_flags;
332}
333
334#ifdef __cplusplus
335}  // extern "C"
336}  // namespace libyuv
337#endif
338