1/*
2 * Copyright (c) 2008, Google Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are
6 * met:
7 *
8 *     * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 *     * Redistributions in binary form must reproduce the above
11 * copyright notice, this list of conditions and the following disclaimer
12 * in the documentation and/or other materials provided with the
13 * distribution.
14 *     * Neither the name of Google Inc. nor the names of its
15 * contributors may be used to endorse or promote products derived from
16 * this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31#include "config.h"
32#include "platform/graphics/skia/NativeImageSkia.h"
33
34#include "platform/PlatformInstrumentation.h"
35#include "platform/TraceEvent.h"
36#include "platform/geometry/FloatPoint.h"
37#include "platform/geometry/FloatRect.h"
38#include "platform/geometry/FloatSize.h"
39#include "platform/graphics/DeferredImageDecoder.h"
40#include "platform/graphics/GraphicsContext.h"
41#include "platform/graphics/Image.h"
42#include "platform/graphics/skia/SkiaUtils.h"
43#include "skia/ext/image_operations.h"
44#include "third_party/skia/include/core/SkMatrix.h"
45#include "third_party/skia/include/core/SkPaint.h"
46#include "third_party/skia/include/core/SkScalar.h"
47#include "third_party/skia/include/core/SkShader.h"
48
49#include <math.h>
50
51namespace blink {
52
53// This function is used to scale an image and extract a scaled fragment.
54//
55// ALGORITHM
56//
57// Because the scaled image size has to be integers, we approximate the real
58// scale with the following formula (only X direction is shown):
59//
60// scaledImageWidth = round(scaleX * imageRect.width())
61// approximateScaleX = scaledImageWidth / imageRect.width()
62//
63// With this method we maintain a constant scale factor among fragments in
64// the scaled image. This allows fragments to stitch together to form the
65// full scaled image. The downside is there will be a small difference
66// between |scaleX| and |approximateScaleX|.
67//
68// A scaled image fragment is identified by:
69//
70// - Scaled image size
71// - Scaled image fragment rectangle (IntRect)
72//
73// Scaled image size has been determined and the next step is to compute the
74// rectangle for the scaled image fragment which needs to be an IntRect.
75//
76// scaledSrcRect = srcRect * (approximateScaleX, approximateScaleY)
77// enclosingScaledSrcRect = enclosingIntRect(scaledSrcRect)
78//
79// Finally we extract the scaled image fragment using
80// (scaledImageSize, enclosingScaledSrcRect).
81//
82SkBitmap NativeImageSkia::extractScaledImageFragment(const SkRect& srcRect, float scaleX, float scaleY, SkRect* scaledSrcRect) const
83{
84    SkISize imageSize = SkISize::Make(bitmap().width(), bitmap().height());
85    SkISize scaledImageSize = SkISize::Make(clampToInteger(roundf(imageSize.width() * scaleX)),
86        clampToInteger(roundf(imageSize.height() * scaleY)));
87
88    SkRect imageRect = SkRect::MakeWH(imageSize.width(), imageSize.height());
89    SkRect scaledImageRect = SkRect::MakeWH(scaledImageSize.width(), scaledImageSize.height());
90
91    SkMatrix scaleTransform;
92    scaleTransform.setRectToRect(imageRect, scaledImageRect, SkMatrix::kFill_ScaleToFit);
93    scaleTransform.mapRect(scaledSrcRect, srcRect);
94
95    scaledSrcRect->intersect(scaledImageRect);
96    SkIRect enclosingScaledSrcRect = enclosingIntRect(*scaledSrcRect);
97
98    // |enclosingScaledSrcRect| can be larger than |scaledImageSize| because
99    // of float inaccuracy so clip to get inside.
100    enclosingScaledSrcRect.intersect(SkIRect::MakeSize(scaledImageSize));
101
102    // scaledSrcRect is relative to the pixel snapped fragment we're extracting.
103    scaledSrcRect->offset(-enclosingScaledSrcRect.x(), -enclosingScaledSrcRect.y());
104
105    return resizedBitmap(scaledImageSize, enclosingScaledSrcRect);
106}
107
108NativeImageSkia::NativeImageSkia()
109    : m_resizeRequests(0)
110{
111}
112
113NativeImageSkia::NativeImageSkia(const SkBitmap& other)
114    : m_bitmap(other)
115    , m_resizeRequests(0)
116{
117}
118
119NativeImageSkia::~NativeImageSkia()
120{
121}
122
123bool NativeImageSkia::hasResizedBitmap(const SkISize& scaledImageSize, const SkIRect& scaledImageSubset) const
124{
125    bool imageScaleEqual = m_cachedImageInfo.scaledImageSize == scaledImageSize;
126    bool scaledImageSubsetAvailable = m_cachedImageInfo.scaledImageSubset.contains(scaledImageSubset);
127    return imageScaleEqual && scaledImageSubsetAvailable && !m_resizedImage.empty();
128}
129
130SkBitmap NativeImageSkia::resizedBitmap(const SkISize& scaledImageSize, const SkIRect& scaledImageSubset) const
131{
132    ASSERT(!DeferredImageDecoder::isLazyDecoded(bitmap()));
133
134    if (!hasResizedBitmap(scaledImageSize, scaledImageSubset)) {
135        bool shouldCache = isDataComplete()
136            && shouldCacheResampling(scaledImageSize, scaledImageSubset);
137
138        TRACE_EVENT1(TRACE_DISABLED_BY_DEFAULT("devtools.timeline"), "ResizeImage", "cached", shouldCache);
139        // FIXME(361045): remove InspectorInstrumentation calls once DevTools Timeline migrates to tracing.
140        PlatformInstrumentation::willResizeImage(shouldCache);
141        SkBitmap resizedImage = skia::ImageOperations::Resize(bitmap(), skia::ImageOperations::RESIZE_LANCZOS3, scaledImageSize.width(), scaledImageSize.height(), scaledImageSubset);
142        resizedImage.setImmutable();
143        PlatformInstrumentation::didResizeImage();
144
145        if (!shouldCache)
146            return resizedImage;
147
148        m_resizedImage = resizedImage;
149    }
150
151    SkBitmap resizedSubset;
152    SkIRect resizedSubsetRect = m_cachedImageInfo.rectInSubset(scaledImageSubset);
153    m_resizedImage.extractSubset(&resizedSubset, resizedSubsetRect);
154    return resizedSubset;
155}
156
157void NativeImageSkia::draw(
158    GraphicsContext* context,
159    const SkRect& srcRect,
160    const SkRect& destRect,
161    CompositeOperator compositeOp,
162    WebBlendMode blendMode) const
163{
164    TRACE_EVENT0("skia", "NativeImageSkia::draw");
165
166    bool isLazyDecoded = DeferredImageDecoder::isLazyDecoded(bitmap());
167
168    SkPaint paint;
169    context->preparePaintForDrawRectToRect(&paint, srcRect, destRect, compositeOp, blendMode, isLazyDecoded, isDataComplete());
170    // We want to filter it if we decided to do interpolation above, or if
171    // there is something interesting going on with the matrix (like a rotation).
172    // Note: for serialization, we will want to subset the bitmap first so we
173    // don't send extra pixels.
174    context->drawBitmapRect(bitmap(), &srcRect, destRect, &paint);
175
176    if (isLazyDecoded)
177        PlatformInstrumentation::didDrawLazyPixelRef(bitmap().getGenerationID());
178    context->didDrawRect(destRect, paint, &bitmap());
179}
180
181static SkBitmap createBitmapWithSpace(const SkBitmap& bitmap, int spaceWidth, int spaceHeight)
182{
183    SkImageInfo info = bitmap.info();
184    info.fWidth += spaceWidth;
185    info.fHeight += spaceHeight;
186    info.fAlphaType = kPremul_SkAlphaType;
187
188    SkBitmap result;
189    result.allocPixels(info);
190    result.eraseColor(SK_ColorTRANSPARENT);
191    bitmap.copyPixelsTo(reinterpret_cast<uint8_t*>(result.getPixels()), result.rowBytes() * result.height(), result.rowBytes());
192
193    return result;
194}
195
196void NativeImageSkia::drawPattern(
197    GraphicsContext* context,
198    const FloatRect& floatSrcRect,
199    const FloatSize& scale,
200    const FloatPoint& phase,
201    CompositeOperator compositeOp,
202    const FloatRect& destRect,
203    WebBlendMode blendMode,
204    const IntSize& repeatSpacing) const
205{
206    FloatRect normSrcRect = floatSrcRect;
207    normSrcRect.intersect(FloatRect(0, 0, bitmap().width(), bitmap().height()));
208    if (destRect.isEmpty() || normSrcRect.isEmpty())
209        return; // nothing to draw
210
211    SkMatrix totalMatrix = context->getTotalMatrix();
212    AffineTransform ctm = context->getCTM();
213    SkScalar ctmScaleX = ctm.xScale();
214    SkScalar ctmScaleY = ctm.yScale();
215    totalMatrix.preScale(scale.width(), scale.height());
216
217    // Figure out what size the bitmap will be in the destination. The
218    // destination rect is the bounds of the pattern, we need to use the
219    // matrix to see how big it will be.
220    SkRect destRectTarget;
221    totalMatrix.mapRect(&destRectTarget, normSrcRect);
222
223    float destBitmapWidth = SkScalarToFloat(destRectTarget.width());
224    float destBitmapHeight = SkScalarToFloat(destRectTarget.height());
225
226    bool isLazyDecoded = DeferredImageDecoder::isLazyDecoded(bitmap());
227
228    // Compute the resampling mode.
229    InterpolationQuality resampling;
230    if (context->isAccelerated() || context->printing())
231        resampling = InterpolationLow;
232    else if (isLazyDecoded)
233        resampling = InterpolationHigh;
234    else
235        resampling = computeInterpolationQuality(totalMatrix, normSrcRect.width(), normSrcRect.height(), destBitmapWidth, destBitmapHeight, isDataComplete());
236    resampling = limitInterpolationQuality(context, resampling);
237
238    SkMatrix localMatrix;
239    // We also need to translate it such that the origin of the pattern is the
240    // origin of the destination rect, which is what WebKit expects. Skia uses
241    // the coordinate system origin as the base for the pattern. If WebKit wants
242    // a shifted image, it will shift it from there using the localMatrix.
243    const float adjustedX = phase.x() + normSrcRect.x() * scale.width();
244    const float adjustedY = phase.y() + normSrcRect.y() * scale.height();
245    localMatrix.setTranslate(SkFloatToScalar(adjustedX), SkFloatToScalar(adjustedY));
246
247    RefPtr<SkShader> shader;
248    SkPaint::FilterLevel filterLevel = static_cast<SkPaint::FilterLevel>(resampling);
249
250    // Bicubic filter is only applied to defer-decoded images, see
251    // NativeImageSkia::draw for details.
252    if (resampling == InterpolationHigh && !isLazyDecoded) {
253        // Do nice resampling.
254        filterLevel = SkPaint::kNone_FilterLevel;
255        float scaleX = destBitmapWidth / normSrcRect.width();
256        float scaleY = destBitmapHeight / normSrcRect.height();
257        SkRect scaledSrcRect;
258
259        // Since we are resizing the bitmap, we need to remove the scale
260        // applied to the pixels in the bitmap shader. This means we need
261        // CTM * localMatrix to have identity scale. Since we
262        // can't modify CTM (or the rectangle will be drawn in the wrong
263        // place), we must set localMatrix's scale to the inverse of
264        // CTM scale.
265        localMatrix.preScale(ctmScaleX ? 1 / ctmScaleX : 1, ctmScaleY ? 1 / ctmScaleY : 1);
266
267        // The image fragment generated here is not exactly what is
268        // requested. The scale factor used is approximated and image
269        // fragment is slightly larger to align to integer
270        // boundaries.
271        SkBitmap resampled = extractScaledImageFragment(normSrcRect, scaleX, scaleY, &scaledSrcRect);
272        if (repeatSpacing.isZero()) {
273            shader = adoptRef(SkShader::CreateBitmapShader(resampled, SkShader::kRepeat_TileMode, SkShader::kRepeat_TileMode, &localMatrix));
274        } else {
275            shader = adoptRef(SkShader::CreateBitmapShader(
276                createBitmapWithSpace(resampled, repeatSpacing.width() * ctmScaleX, repeatSpacing.height() * ctmScaleY),
277                SkShader::kRepeat_TileMode, SkShader::kRepeat_TileMode, &localMatrix));
278        }
279    } else {
280        // Because no resizing occurred, the shader transform should be
281        // set to the pattern's transform, which just includes scale.
282        localMatrix.preScale(scale.width(), scale.height());
283
284        // No need to resample before drawing.
285        SkBitmap srcSubset;
286        bitmap().extractSubset(&srcSubset, enclosingIntRect(normSrcRect));
287        if (repeatSpacing.isZero()) {
288            shader = adoptRef(SkShader::CreateBitmapShader(srcSubset, SkShader::kRepeat_TileMode, SkShader::kRepeat_TileMode, &localMatrix));
289        } else {
290            shader = adoptRef(SkShader::CreateBitmapShader(
291                createBitmapWithSpace(srcSubset, repeatSpacing.width() * ctmScaleX, repeatSpacing.height() * ctmScaleY),
292                SkShader::kRepeat_TileMode, SkShader::kRepeat_TileMode, &localMatrix));
293        }
294    }
295
296    SkPaint paint;
297    paint.setShader(shader.get());
298    paint.setXfermodeMode(WebCoreCompositeToSkiaComposite(compositeOp, blendMode));
299    paint.setColorFilter(context->colorFilter());
300    paint.setFilterLevel(filterLevel);
301
302    if (isLazyDecoded)
303        PlatformInstrumentation::didDrawLazyPixelRef(bitmap().getGenerationID());
304
305    context->drawRect(destRect, paint);
306}
307
308bool NativeImageSkia::shouldCacheResampling(const SkISize& scaledImageSize, const SkIRect& scaledImageSubset) const
309{
310    // Check whether the requested dimensions match previous request.
311    bool matchesPreviousRequest = m_cachedImageInfo.isEqual(scaledImageSize, scaledImageSubset);
312    if (matchesPreviousRequest)
313        ++m_resizeRequests;
314    else {
315        m_cachedImageInfo.set(scaledImageSize, scaledImageSubset);
316        m_resizeRequests = 0;
317        // Reset m_resizedImage now, because we don't distinguish
318        // between the last requested resize info and m_resizedImage's
319        // resize info.
320        m_resizedImage.reset();
321    }
322
323    // We can not cache incomplete frames. This might be a good optimization in
324    // the future, were we know how much of the frame has been decoded, so when
325    // we incrementally draw more of the image, we only have to resample the
326    // parts that are changed.
327    if (!isDataComplete())
328        return false;
329
330    // If the destination bitmap is excessively large, we'll never allow caching.
331    static const unsigned long long kLargeBitmapSize = 4096ULL * 4096ULL;
332    unsigned long long fullSize = static_cast<unsigned long long>(scaledImageSize.width()) * static_cast<unsigned long long>(scaledImageSize.height());
333    unsigned long long fragmentSize = static_cast<unsigned long long>(scaledImageSubset.width()) * static_cast<unsigned long long>(scaledImageSubset.height());
334
335    if (fragmentSize > kLargeBitmapSize)
336        return false;
337
338    // If the destination bitmap is small, we'll always allow caching, since
339    // there is not very much penalty for computing it and it may come in handy.
340    static const unsigned kSmallBitmapSize = 4096;
341    if (fragmentSize <= kSmallBitmapSize)
342        return true;
343
344    // If "too many" requests have been made for this bitmap, we assume that
345    // many more will be made as well, and we'll go ahead and cache it.
346    static const int kManyRequestThreshold = 4;
347    if (m_resizeRequests >= kManyRequestThreshold)
348        return true;
349
350    // If more than 1/4 of the resized image is requested, it's worth caching.
351    return fragmentSize > fullSize / 4;
352}
353
354NativeImageSkia::ImageResourceInfo::ImageResourceInfo()
355{
356    scaledImageSize.setEmpty();
357    scaledImageSubset.setEmpty();
358}
359
360bool NativeImageSkia::ImageResourceInfo::isEqual(const SkISize& otherScaledImageSize, const SkIRect& otherScaledImageSubset) const
361{
362    return scaledImageSize == otherScaledImageSize && scaledImageSubset == otherScaledImageSubset;
363}
364
365void NativeImageSkia::ImageResourceInfo::set(const SkISize& otherScaledImageSize, const SkIRect& otherScaledImageSubset)
366{
367    scaledImageSize = otherScaledImageSize;
368    scaledImageSubset = otherScaledImageSubset;
369}
370
371SkIRect NativeImageSkia::ImageResourceInfo::rectInSubset(const SkIRect& otherScaledImageSubset)
372{
373    if (!scaledImageSubset.contains(otherScaledImageSubset))
374        return SkIRect::MakeEmpty();
375    SkIRect subsetRect = otherScaledImageSubset;
376    subsetRect.offset(-scaledImageSubset.x(), -scaledImageSubset.y());
377    return subsetRect;
378}
379
380} // namespace blink
381