18dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com/*
28dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com * Copyright 2011 Google Inc.
38dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com *
48dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com * Use of this source code is governed by a BSD-style license that can be
58dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com * found in the LICENSE file.
68dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com */
78dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
88c4953c6f176469ad287c3270ab146e292b23badcommit-bot@chromium.org#include <emmintrin.h>
98dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com#include "SkBlitRect_opts_SSE2.h"
108dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com#include "SkBlitRow.h"
118dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com#include "SkColorPriv.h"
128dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
138c4953c6f176469ad287c3270ab146e292b23badcommit-bot@chromium.org/* Simple blitting of opaque rectangles less than 31 pixels wide:
148c4953c6f176469ad287c3270ab146e292b23badcommit-bot@chromium.org * inlines and merges sections of Color32_SSE2 and sk_memset32_SSE2.
158c4953c6f176469ad287c3270ab146e292b23badcommit-bot@chromium.org */
1683ecdc3ac69c9208493c4c3fc8ea9f84b1350535caryclark@google.comstatic void BlitRect32_OpaqueNarrow_SSE2(SkPMColor* SK_RESTRICT destination,
178dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com                                  int width, int height,
188dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com                                  size_t rowBytes, uint32_t color) {
198dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com    SkASSERT(255 == SkGetPackedA32(color));
208dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com    SkASSERT(width > 0);
218dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com    SkASSERT(width < 31);
228dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
238dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com    while (--height >= 0) {
248dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        SkPMColor* dst = destination;
258dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        int count = width;
268dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
278dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        while (count > 4) {
288dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            *dst++ = color;
298dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            *dst++ = color;
308dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            *dst++ = color;
318dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            *dst++ = color;
328dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            count -= 4;
338dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        }
348dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
358dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        while (count > 0) {
368dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            *dst++ = color;
378dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            --count;
388dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        }
398dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
408dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        destination = (uint32_t*)((char*)destination + rowBytes);
418dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com    }
428dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com}
438dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
448c4953c6f176469ad287c3270ab146e292b23badcommit-bot@chromium.org/*
458c4953c6f176469ad287c3270ab146e292b23badcommit-bot@chromium.org * Fast blitting of opaque rectangles at least 31 pixels wide:
468c4953c6f176469ad287c3270ab146e292b23badcommit-bot@chromium.org * inlines and merges sections of Color32_SSE2 and sk_memset32_SSE2.
478c4953c6f176469ad287c3270ab146e292b23badcommit-bot@chromium.org * A 31 pixel rectangle is guaranteed to have at least one
488c4953c6f176469ad287c3270ab146e292b23badcommit-bot@chromium.org * 16-pixel aligned span that can take advantage of mm_store.
498c4953c6f176469ad287c3270ab146e292b23badcommit-bot@chromium.org */
5083ecdc3ac69c9208493c4c3fc8ea9f84b1350535caryclark@google.comstatic void BlitRect32_OpaqueWide_SSE2(SkPMColor* SK_RESTRICT destination,
518dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com                                int width, int height,
528dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com                                size_t rowBytes, uint32_t color) {
538dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com    SkASSERT(255 == SkGetPackedA32(color));
548dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com    SkASSERT(width >= 31);
558dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
568dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com    __m128i color_wide = _mm_set1_epi32(color);
578dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com    while (--height >= 0) {
588dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        // Prefetching one row ahead to L1 cache can equal hardware
598dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        // performance for large/tall rects, but never *beats*
608dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        // hardware performance.
618dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        SkPMColor* dst = destination;
628dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        int count = width;
638dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
648dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        while (((size_t)dst) & 0x0F) {
658dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            *dst++ = color;
668dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            --count;
678dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        }
688dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        __m128i *d = reinterpret_cast<__m128i*>(dst);
698dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
708dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        // Googling suggests _mm_stream is only going to beat _mm_store
718dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        // for things that wouldn't fit in L2 cache anyway, typically
728dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        // >500kB, and precisely fill cache lines.  For us, with
738dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        // arrays > 100k elements _mm_stream is still 100%+ slower than
748dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        // mm_store.
758dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
768dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        // Unrolling to count >= 64 is a break-even for most
778dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        // input patterns; we seem to be saturating the bus and having
788dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        // low enough overhead at 32.
798dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
808dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        while (count >= 32) {
818dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            _mm_store_si128(d++, color_wide);
828dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            _mm_store_si128(d++, color_wide);
838dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            _mm_store_si128(d++, color_wide);
848dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            _mm_store_si128(d++, color_wide);
858dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            _mm_store_si128(d++, color_wide);
868dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            _mm_store_si128(d++, color_wide);
878dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            _mm_store_si128(d++, color_wide);
888dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            _mm_store_si128(d++, color_wide);
898dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            count -= 32;
908dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        }
918dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        if (count >= 16) {
928dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            _mm_store_si128(d++, color_wide);
938dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            _mm_store_si128(d++, color_wide);
948dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            _mm_store_si128(d++, color_wide);
958dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            _mm_store_si128(d++, color_wide);
968dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            count -= 16;
978dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        }
988dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        dst = reinterpret_cast<uint32_t*>(d);
998dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
1008dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        // Unrolling the loop in the Narrow code is a significant performance
1018dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        // gain, but unrolling this loop appears to make no difference in
1028dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        // benchmarks with either mm_store_si128 or individual sets.
1038dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
1048dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        while (count > 0) {
1058dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            *dst++ = color;
1068dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com            --count;
1078dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        }
1088dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
1098dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        destination = (uint32_t*)((char*)destination + rowBytes);
1108dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com    }
1118dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com}
1128dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com
1138dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.comvoid ColorRect32_SSE2(SkPMColor* destination,
1148dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com                      int width, int height,
1158dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com                      size_t rowBytes, uint32_t color) {
1168dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com    if (0 == height || 0 == width || 0 == color) {
1178dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        return;
1188dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com    }
1198dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com    unsigned colorA = SkGetPackedA32(color);
120b58a639b2fbe919489654bb506efdb024a308a8ebsalomon@google.com    colorA = 0; // skip below if () for now...(has been disabled since this was added in r3423).
121b58a639b2fbe919489654bb506efdb024a308a8ebsalomon@google.com    if (255 == colorA) {
12283ecdc3ac69c9208493c4c3fc8ea9f84b1350535caryclark@google.com        if (width < 31) {
12383ecdc3ac69c9208493c4c3fc8ea9f84b1350535caryclark@google.com            BlitRect32_OpaqueNarrow_SSE2(destination, width, height,
12483ecdc3ac69c9208493c4c3fc8ea9f84b1350535caryclark@google.com                                         rowBytes, color);
12583ecdc3ac69c9208493c4c3fc8ea9f84b1350535caryclark@google.com        } else {
12683ecdc3ac69c9208493c4c3fc8ea9f84b1350535caryclark@google.com            BlitRect32_OpaqueWide_SSE2(destination, width, height,
12783ecdc3ac69c9208493c4c3fc8ea9f84b1350535caryclark@google.com                                       rowBytes, color);
12883ecdc3ac69c9208493c4c3fc8ea9f84b1350535caryclark@google.com        }
12983ecdc3ac69c9208493c4c3fc8ea9f84b1350535caryclark@google.com    } else {
1308dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com        SkBlitRow::ColorRect32(destination, width, height, rowBytes, color);
13183ecdc3ac69c9208493c4c3fc8ea9f84b1350535caryclark@google.com    }
1328dd90a926a8660da2bacc7af149f4ac5b2e7c64ctomhudson@google.com}
133