1d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* 2d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * GPL HEADER START 3d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 4d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 6d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * This program is free software; you can redistribute it and/or modify 7d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * it under the terms of the GNU General Public License version 2 only, 8d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * as published by the Free Software Foundation. 9d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 10d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * This program is distributed in the hope that it will be useful, but 11d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * WITHOUT ANY WARRANTY; without even the implied warranty of 12d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * General Public License version 2 for more details (a copy is included 14d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * in the LICENSE file that accompanied this code). 15d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 16d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * You should have received a copy of the GNU General Public License 17d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * version 2 along with this program; If not, see 18d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf 19d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 20d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, 21d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * CA 95054 USA or visit www.sun.com if you need additional information or 22d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * have any questions. 23d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 24d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * GPL HEADER END 25d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 26d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* 27d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 28d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Use is subject to license terms. 29d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 30d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Copyright (c) 2011, 2012, Intel Corporation. 31d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 32d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* 33d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * This file is part of Lustre, http://www.lustre.org/ 34d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Lustre is a trademark of Sun Microsystems, Inc. 35d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 36d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Implementation of cl_page for OSC layer. 37d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 38d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Author: Nikita Danilov <nikita.danilov@sun.com> 39d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 40d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 41d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao#define DEBUG_SUBSYSTEM S_OSC 42d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 43d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao#include "osc_cl_internal.h" 44d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 45d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_lru_del(struct client_obd *cli, struct osc_page *opg, bool del); 46d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_lru_add(struct client_obd *cli, struct osc_page *opg); 47d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_lru_reserve(const struct lu_env *env, struct osc_object *obj, 48d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg); 49d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 50d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/** \addtogroup osc 51d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * @{ 52d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 53d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 54d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* 55d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Comment out osc_page_protected because it may sleep inside the 56d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * the client_obd_list_lock. 57d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * client_obd_list_lock -> osc_ap_completion -> osc_completion -> 58d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * -> osc_page_protected -> osc_page_is_dlocked -> osc_match_base 59d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * -> ldlm_lock_match -> sptlrpc_import_check_ctx -> sleep. 60d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 61d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao#if 0 62d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_is_dlocked(const struct lu_env *env, 63d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct osc_page *opg, 64d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao enum cl_lock_mode mode, int pending, int unref) 65d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 66d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page *page; 67d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *obj; 68d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_thread_info *info; 69d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct ldlm_res_id *resname; 70d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct lustre_handle *lockh; 71d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao ldlm_policy_data_t *policy; 72d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao ldlm_mode_t dlmmode; 73875332d47f8c3a0c23ebe469bd8b31efb77d997aDmitry Eremin __u64 flags; 74d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 75d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao might_sleep(); 76d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 77d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao info = osc_env_info(env); 78d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao resname = &info->oti_resname; 79d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao policy = &info->oti_policy; 80d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao lockh = &info->oti_handle; 81d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao page = opg->ops_cl.cpl_page; 82d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao obj = cl2osc(opg->ops_cl.cpl_obj); 83d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 84d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao flags = LDLM_FL_TEST_LOCK | LDLM_FL_BLOCK_GRANTED; 85d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (pending) 86d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao flags |= LDLM_FL_CBPENDING; 87d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 88d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao dlmmode = osc_cl_lock2ldlm(mode) | LCK_PW; 89d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_lock_build_res(env, obj, resname); 90d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_index2policy(policy, page->cp_obj, page->cp_index, page->cp_index); 91d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return osc_match_base(osc_export(obj), resname, LDLM_EXTENT, policy, 92d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao dlmmode, &flags, NULL, lockh, unref); 93d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 94d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 95d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/** 96d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Checks an invariant that a page in the cache is covered by a lock, as 97d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * needed. 98d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 99d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_protected(const struct lu_env *env, 100d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct osc_page *opg, 101d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao enum cl_lock_mode mode, int unref) 102d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 103d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_object_header *hdr; 104d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_lock *scan; 105d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page *page; 106d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_lock_descr *descr; 107d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int result; 108d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 109d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LINVRNT(!opg->ops_temp); 110d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 111d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao page = opg->ops_cl.cpl_page; 112d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (page->cp_owner != NULL && 113d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_io_top(page->cp_owner)->ci_lockreq == CILR_NEVER) 114d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 115d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * If IO is done without locks (liblustre, or lloop), lock is 116d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * not required. 117d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 118d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao result = 1; 119d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao else 120d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* otherwise check for a DLM lock */ 121d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao result = osc_page_is_dlocked(env, opg, mode, 1, unref); 122d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (result == 0) { 123d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* maybe this page is a part of a lockless io? */ 124d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao hdr = cl_object_header(opg->ops_cl.cpl_obj); 125d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao descr = &osc_env_info(env)->oti_descr; 126d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao descr->cld_mode = mode; 127d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao descr->cld_start = page->cp_index; 128d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao descr->cld_end = page->cp_index; 129d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_lock(&hdr->coh_lock_guard); 130d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_for_each_entry(scan, &hdr->coh_locks, cll_linkage) { 131d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 132d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Lock-less sub-lock has to be either in HELD state 133d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * (when io is actively going on), or in CACHED state, 134d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * when top-lock is being unlocked: 135d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * cl_io_unlock()->cl_unuse()->...->lov_lock_unuse(). 136d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 137d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if ((scan->cll_state == CLS_HELD || 138d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao scan->cll_state == CLS_CACHED) && 139d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_lock_ext_match(&scan->cll_descr, descr)) { 140d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_lock *olck; 141d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 142d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao olck = osc_lock_at(scan); 143d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao result = osc_lock_is_lockless(olck); 144d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao break; 145d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 146d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 147d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_unlock(&hdr->coh_lock_guard); 148d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 149d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return result; 150d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 151d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao#else 152d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_protected(const struct lu_env *env, 153d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct osc_page *opg, 154d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao enum cl_lock_mode mode, int unref) 155d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 156d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return 1; 157d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 158d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao#endif 159d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 160d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/***************************************************************************** 161d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 162d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Page operations. 163d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 164d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 165d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_fini(const struct lu_env *env, 166d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page_slice *slice) 167d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 168d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 169d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao CDEBUG(D_TRACE, "%p\n", opg); 170d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(opg->ops_lock == NULL); 171d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 172d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 173d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_transfer_get(struct osc_page *opg, const char *label) 174d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 175d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page *page = cl_page_top(opg->ops_cl.cpl_page); 176d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 177d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(!opg->ops_transfer_pinned); 178d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_get(page); 179d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao lu_ref_add_atomic(&page->cp_reference, label, page); 180d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_transfer_pinned = 1; 181d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 182d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 183d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_transfer_put(const struct lu_env *env, 184d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg) 185d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 186d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page *page = cl_page_top(opg->ops_cl.cpl_page); 187d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 188d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (opg->ops_transfer_pinned) { 189d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao lu_ref_del(&page->cp_reference, "transfer", page); 190d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_transfer_pinned = 0; 191d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_put(env, page); 192d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 193d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 194d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 195d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/** 196d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * This is called once for every page when it is submitted for a transfer 197d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * either opportunistic (osc_page_cache_add()), or immediate 198d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * (osc_page_submit()). 199d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 200d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_transfer_add(const struct lu_env *env, 201d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg, enum cl_req_type crt) 202d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 203d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *obj = cl2osc(opg->ops_cl.cpl_obj); 204d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 205d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* ops_lru and ops_inflight share the same field, so take it from LRU 206d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * first and then use it as inflight. */ 207d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_lru_del(osc_cli(obj), opg, false); 208d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 209d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_lock(&obj->oo_seatbelt); 210d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_add(&opg->ops_inflight, &obj->oo_inflight[crt]); 211d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_submitter = current; 212d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_unlock(&obj->oo_seatbelt); 213d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 214d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 215d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_cache_add(const struct lu_env *env, 216d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 217d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_io *io) 218d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 219d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_io *oio = osc_env_io(env); 220d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 221d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int result; 222d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 223d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LINVRNT(osc_page_protected(env, opg, CLM_WRITE, 0)); 224d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 225d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_transfer_get(opg, "transfer\0cache"); 226d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao result = osc_queue_async_io(env, io, opg); 227d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (result != 0) 228d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_transfer_put(env, opg); 229d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao else 230d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_transfer_add(env, opg, CRT_WRITE); 231d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 232d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* for sync write, kernel will wait for this page to be flushed before 233d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * osc_io_end() is called, so release it earlier. 234d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * for mkwrite(), it's known there is no further pages. */ 235d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (cl_io_is_sync_write(io) || cl_io_is_mkwrite(io)) { 236d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (oio->oi_active != NULL) { 237d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_extent_release(env, oio->oi_active); 238d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oio->oi_active = NULL; 239d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 240d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 241d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 2420a3bdb00710bf253ba8ba8f645645f22297c7a04Greg Kroah-Hartman return result; 243d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 244d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 245d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taovoid osc_index2policy(ldlm_policy_data_t *policy, const struct cl_object *obj, 246d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao pgoff_t start, pgoff_t end) 247d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 248ec83e611c21592c38744a5b86e1c8286ed1c4f90Joe Perches memset(policy, 0, sizeof(*policy)); 249d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao policy->l_extent.start = cl_offset(obj, start); 250d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao policy->l_extent.end = cl_offset(obj, end + 1) - 1; 251d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 252d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 253d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_addref_lock(const struct lu_env *env, 254d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg, 255d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_lock *lock) 256d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 257d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_lock *olock; 258d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int rc; 259d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 260d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(opg->ops_lock == NULL); 261d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 262d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao olock = osc_lock_at(lock); 263d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (atomic_inc_return(&olock->ols_pageref) <= 0) { 264d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_dec(&olock->ols_pageref); 265d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = -ENODATA; 266d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } else { 267d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_lock_get(lock); 268d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_lock = lock; 269d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = 0; 270d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 271d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return rc; 272d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 273d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 274d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_putref_lock(const struct lu_env *env, 275d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg) 276d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 277d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_lock *lock = opg->ops_lock; 278d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_lock *olock; 279d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 280d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(lock != NULL); 281d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao olock = osc_lock_at(lock); 282d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 283d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_dec(&olock->ols_pageref); 284d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_lock = NULL; 285d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 286d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_lock_put(env, lock); 287d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 288d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 289d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_is_under_lock(const struct lu_env *env, 290d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 291d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_io *unused) 292d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 293d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_lock *lock; 294d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int result = -ENODATA; 295d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 296d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao lock = cl_lock_at_page(env, slice->cpl_obj, slice->cpl_page, 297d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao NULL, 1, 0); 298d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (lock != NULL) { 299d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (osc_page_addref_lock(env, cl2osc_page(slice), lock) == 0) 300d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao result = -EBUSY; 301d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_lock_put(env, lock); 302d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 3030a3bdb00710bf253ba8ba8f645645f22297c7a04Greg Kroah-Hartman return result; 304d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 305d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 306d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_disown(const struct lu_env *env, 307d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 308d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_io *io) 309d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 310d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 311d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 312d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (unlikely(opg->ops_lock)) 313d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_putref_lock(env, opg); 314d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 315d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 316d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_completion_read(const struct lu_env *env, 317d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 318d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int ioret) 319d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 320d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 321d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *obj = cl2osc(opg->ops_cl.cpl_obj); 322d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 323d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (likely(opg->ops_lock)) 324d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_putref_lock(env, opg); 325d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_lru_add(osc_cli(obj), opg); 326d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 327d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 328d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_completion_write(const struct lu_env *env, 329d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 330d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int ioret) 331d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 332d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 333d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *obj = cl2osc(slice->cpl_obj); 334d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 335d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_lru_add(osc_cli(obj), opg); 336d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 337d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 338d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_fail(const struct lu_env *env, 339d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 340d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_io *unused) 341d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 342d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 343d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Cached read? 344d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 345d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LBUG(); 346d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return 0; 347d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 348d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 349d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 350d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic const char *osc_list(struct list_head *head) 351d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 352d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return list_empty(head) ? "-" : "+"; 353d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 354d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 355a649ad1d7b44d2b5da62eec804539bb221396570Greg Kroah-Hartmanstatic inline unsigned long osc_submit_duration(struct osc_page *opg) 356d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 357d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (opg->ops_submit_time == 0) 358d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return 0; 359d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 360d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return (cfs_time_current() - opg->ops_submit_time); 361d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 362d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 363d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_print(const struct lu_env *env, 364d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 365d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao void *cookie, lu_printer_t printer) 366d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 367d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 368d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_async_page *oap = &opg->ops_oap; 369d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *obj = cl2osc(slice->cpl_obj); 370d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct client_obd *cli = &osc_export(obj)->exp_obd->u.cli; 371d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 372d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return (*printer)(env, cookie, LUSTRE_OSC_NAME"-page@%p: " 373d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao "1< %#x %d %u %s %s > " 374b0f5aad587ea1fc3563d056609ee54a961ee1256Greg Kroah-Hartman "2< %llu %u %u %#x %#x | %p %p %p > " 375d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao "3< %s %p %d %lu %d > " 376d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao "4< %d %d %d %lu %s | %s %s %s %s > " 377d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao "5< %s %s %s %s | %d %s | %d %s %s>\n", 378d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg, 379d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 1 */ 380d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_magic, oap->oap_cmd, 381d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_interrupted, 382d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&oap->oap_pending_item), 383d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&oap->oap_rpc_item), 384d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 2 */ 385d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_obj_off, oap->oap_page_off, oap->oap_count, 386d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_async_flags, oap->oap_brw_flags, 387d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_request, oap->oap_cli, obj, 388d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 3 */ 389d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&opg->ops_inflight), 390d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_submitter, opg->ops_transfer_pinned, 391d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_submit_duration(opg), opg->ops_srvlock, 392d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 4 */ 393d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli->cl_r_in_flight, cli->cl_w_in_flight, 394d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli->cl_max_rpcs_in_flight, 395d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli->cl_avail_grant, 396d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&cli->cl_cache_waiters), 397d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&cli->cl_loi_ready_list), 398d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&cli->cl_loi_hp_ready_list), 399d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&cli->cl_loi_write_list), 400d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&cli->cl_loi_read_list), 401d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 5 */ 402d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&obj->oo_ready_item), 403d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&obj->oo_hp_ready_item), 404d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&obj->oo_write_item), 405d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&obj->oo_read_item), 406d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(&obj->oo_nr_reads), 407d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&obj->oo_reading_exts), 408d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(&obj->oo_nr_writes), 409d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&obj->oo_hp_exts), 410d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&obj->oo_urgent_exts)); 411d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 412d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 413d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_delete(const struct lu_env *env, 414d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice) 415d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 416d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 417d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *obj = cl2osc(opg->ops_cl.cpl_obj); 418d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int rc; 419d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 420d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LINVRNT(opg->ops_temp || osc_page_protected(env, opg, CLM_READ, 1)); 421d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 422d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao CDEBUG(D_TRACE, "%p\n", opg); 423d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_transfer_put(env, opg); 424d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = osc_teardown_async_page(env, obj, opg); 425d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc) { 426d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao CL_PAGE_DEBUG(D_ERROR, env, cl_page_top(slice->cpl_page), 427d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao "Trying to teardown failed: %d\n", rc); 428d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(0); 429d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 430d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 431d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_lock(&obj->oo_seatbelt); 432d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (opg->ops_submitter != NULL) { 433d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(!list_empty(&opg->ops_inflight)); 434d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_del_init(&opg->ops_inflight); 435d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_submitter = NULL; 436d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 437d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_unlock(&obj->oo_seatbelt); 438d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 439d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_lru_del(osc_cli(obj), opg, true); 440d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 441d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 442d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taovoid osc_page_clip(const struct lu_env *env, const struct cl_page_slice *slice, 443d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int from, int to) 444d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 445d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 446d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_async_page *oap = &opg->ops_oap; 447d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 448d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LINVRNT(osc_page_protected(env, opg, CLM_READ, 0)); 449d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 450d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_from = from; 451d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_to = to; 452d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_lock(&oap->oap_lock); 453d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_async_flags |= ASYNC_COUNT_STABLE; 454d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_unlock(&oap->oap_lock); 455d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 456d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 457d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_cancel(const struct lu_env *env, 458d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice) 459d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 460d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 461d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int rc = 0; 462d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 463d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LINVRNT(osc_page_protected(env, opg, CLM_READ, 0)); 464d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 465d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* Check if the transferring against this page 466d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * is completed, or not even queued. */ 467d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (opg->ops_transfer_pinned) 468d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* FIXME: may not be interrupted.. */ 469d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = osc_cancel_async_page(env, opg); 470d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(ergo(rc == 0, opg->ops_transfer_pinned == 0)); 471d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return rc; 472d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 473d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 474d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_flush(const struct lu_env *env, 475d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 476d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_io *io) 477d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 478d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 479d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int rc = 0; 48029aaf4962a3bce337d37176858ef1025b9f29cc4Greg Kroah-Hartman 481d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = osc_flush_async_page(env, io, opg); 4820a3bdb00710bf253ba8ba8f645645f22297c7a04Greg Kroah-Hartman return rc; 483d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 484d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 485d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic const struct cl_page_operations osc_page_ops = { 486d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_fini = osc_page_fini, 487d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_print = osc_page_print, 488d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_delete = osc_page_delete, 489d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_is_under_lock = osc_page_is_under_lock, 490d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_disown = osc_page_disown, 491d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .io = { 492d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao [CRT_READ] = { 493d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_cache_add = osc_page_fail, 494d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_completion = osc_page_completion_read 495d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao }, 496d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao [CRT_WRITE] = { 497d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_cache_add = osc_page_cache_add, 498d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_completion = osc_page_completion_write 499d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 500d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao }, 501d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_clip = osc_page_clip, 502d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_cancel = osc_page_cancel, 503d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_flush = osc_page_flush 504d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao}; 505d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 506d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taoint osc_page_init(const struct lu_env *env, struct cl_object *obj, 507d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page *page, struct page *vmpage) 508d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 509d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *osc = cl2osc(obj); 510d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl_object_page_slice(obj, page); 511d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int result; 512d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 513d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_from = 0; 514d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_to = PAGE_CACHE_SIZE; 515d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 516d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao result = osc_prep_async_page(osc, opg, vmpage, 517d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_offset(obj, page->cp_index)); 518d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (result == 0) { 519d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_io *oio = osc_env_io(env); 520d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_srvlock = osc_io_srvlock(oio); 521d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_slice_add(page, &opg->ops_cl, obj, 522d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao &osc_page_ops); 523d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 524d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 525d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Cannot assert osc_page_protected() here as read-ahead 526d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * creates temporary pages outside of a lock. 527d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 528d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* ops_inflight and ops_lru are the same field, but it doesn't 529d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * hurt to initialize it twice :-) */ 530d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao INIT_LIST_HEAD(&opg->ops_inflight); 531d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao INIT_LIST_HEAD(&opg->ops_lru); 532d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 533d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* reserve an LRU space for this page */ 534d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (page->cp_type == CPT_CACHEABLE && result == 0) 535d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao result = osc_lru_reserve(env, osc, opg); 536d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 537d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return result; 538d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 539d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 540d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/** 541d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Helper function called by osc_io_submit() for every page in an immediate 542d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * transfer (i.e., transferred synchronously). 543d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 544d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taovoid osc_page_submit(const struct lu_env *env, struct osc_page *opg, 545d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao enum cl_req_type crt, int brw_flags) 546d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 547d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_async_page *oap = &opg->ops_oap; 548d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *obj = oap->oap_obj; 549d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 550d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LINVRNT(osc_page_protected(env, opg, 551d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao crt == CRT_WRITE ? CLM_WRITE : CLM_READ, 1)); 552d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 553d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERTF(oap->oap_magic == OAP_MAGIC, "Bad oap magic: oap %p, " 554d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao "magic 0x%x\n", oap, oap->oap_magic); 555d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(oap->oap_async_flags & ASYNC_READY); 556d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(oap->oap_async_flags & ASYNC_COUNT_STABLE); 557d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 558d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_cmd = crt == CRT_WRITE ? OBD_BRW_WRITE : OBD_BRW_READ; 559d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_page_off = opg->ops_from; 560d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_count = opg->ops_to - opg->ops_from; 561d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_brw_flags = OBD_BRW_SYNC | brw_flags; 562d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 563d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (!client_is_remote(osc_export(obj)) && 5642eb90a757e9d953c9e2a8fce530422189992fb1bPeng Tao capable(CFS_CAP_SYS_RESOURCE)) { 565d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_brw_flags |= OBD_BRW_NOQUOTA; 566d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_cmd |= OBD_BRW_NOQUOTA; 567d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 568d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 569d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_submit_time = cfs_time_current(); 570d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_transfer_get(opg, "transfer\0imm"); 571d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_transfer_add(env, opg, crt); 572d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 573d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 574d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* --------------- LRU page management ------------------ */ 575d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 576d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* OSC is a natural place to manage LRU pages as applications are specialized 577d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * to write OSC by OSC. Ideally, if one OSC is used more frequently it should 578d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * occupy more LRU slots. On the other hand, we should avoid using up all LRU 579d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * slots (client_obd::cl_lru_left) otherwise process has to be put into sleep 580d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * for free LRU slots - this will be very bad so the algorithm requires each 581d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * OSC to free slots voluntarily to maintain a reasonable number of free slots 582d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * at any time. 583d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 584d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 5852f5723a1bf2d2f440b1cf8850bc6a9d8d4e82d3bGreg Kroah-Hartmanstatic DECLARE_WAIT_QUEUE_HEAD(osc_lru_waitq); 586d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic atomic_t osc_lru_waiters = ATOMIC_INIT(0); 587d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* LRU pages are freed in batch mode. OSC should at least free this 588d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * number of pages to avoid running out of LRU budget, and.. */ 589d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic const int lru_shrink_min = 2 << (20 - PAGE_CACHE_SHIFT); /* 2M */ 59011d66e89264024ca52f6128d38650c93dbda5c78Masanari Iida/* free this number at most otherwise it will take too long time to finish. */ 591d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic const int lru_shrink_max = 32 << (20 - PAGE_CACHE_SHIFT); /* 32M */ 592d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 593d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* Check if we can free LRU slots from this OSC. If there exists LRU waiters, 594d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * we should free slots aggressively. In this way, slots are freed in a steady 595d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * step to maintain fairness among OSCs. 596d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 597d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Return how many LRU pages should be freed. */ 598d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_cache_too_much(struct client_obd *cli) 599d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 600d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_client_cache *cache = cli->cl_cache; 601d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int pages = atomic_read(&cli->cl_lru_in_list) >> 1; 602d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 603d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (atomic_read(&osc_lru_waiters) > 0 && 604d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(cli->cl_lru_left) < lru_shrink_max) 605d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* drop lru pages aggressively */ 606d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return min(pages, lru_shrink_max); 607d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 608d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* if it's going to run out LRU slots, we should free some, but not 60911d66e89264024ca52f6128d38650c93dbda5c78Masanari Iida * too much to maintain fairness among OSCs. */ 610d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (atomic_read(cli->cl_lru_left) < cache->ccc_lru_max >> 4) { 611d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao unsigned long tmp; 612d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 613d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao tmp = cache->ccc_lru_max / atomic_read(&cache->ccc_users); 614d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (pages > tmp) 615d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return min(pages, lru_shrink_max); 616d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 617d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return pages > lru_shrink_min ? lru_shrink_min : 0; 618d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 619d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 620d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return 0; 621d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 622d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 623d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* Return how many pages are not discarded in @pvec. */ 624d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int discard_pagevec(const struct lu_env *env, struct cl_io *io, 625d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page **pvec, int max_index) 626d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 627d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int count; 628d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int i; 629d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 630d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao for (count = 0, i = 0; i < max_index; i++) { 631d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page *page = pvec[i]; 632d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (cl_page_own_try(env, io, page) == 0) { 633d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* free LRU page only if nobody is using it. 634d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * This check is necessary to avoid freeing the pages 635d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * having already been removed from LRU and pinned 636d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * for IO. */ 637d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (!cl_page_in_use(page)) { 638d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_unmap(env, io, page); 639d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_discard(env, io, page); 640d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao ++count; 641d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 642d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_disown(env, io, page); 643d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 644d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_put(env, page); 645d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao pvec[i] = NULL; 646d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 647d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return max_index - count; 648d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 649d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 650d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/** 651d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Drop @target of pages from LRU at most. 652d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 653d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taoint osc_lru_shrink(struct client_obd *cli, int target) 654d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 655d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_env_nest nest; 656d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct lu_env *env; 657d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_io *io; 658d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_object *clobj = NULL; 659d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page **pvec; 660d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg; 661d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int maxscan = 0; 662d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int count = 0; 663d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int index = 0; 664d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int rc = 0; 665d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 666d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(atomic_read(&cli->cl_lru_in_list) >= 0); 667d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (atomic_read(&cli->cl_lru_in_list) == 0 || target <= 0) 6680a3bdb00710bf253ba8ba8f645645f22297c7a04Greg Kroah-Hartman return 0; 669d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 670d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao env = cl_env_nested_get(&nest); 671d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (IS_ERR(env)) 6720a3bdb00710bf253ba8ba8f645645f22297c7a04Greg Kroah-Hartman return PTR_ERR(env); 673d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 674d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao pvec = osc_env_info(env)->oti_pvec; 675d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao io = &osc_env_info(env)->oti_io; 676d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 677d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_lock(&cli->cl_lru_list_lock); 678d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_inc(&cli->cl_lru_shrinkers); 679d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao maxscan = min(target << 1, atomic_read(&cli->cl_lru_in_list)); 680d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao while (!list_empty(&cli->cl_lru_list)) { 681d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page *page; 682d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 683d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (--maxscan < 0) 684d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao break; 685d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 686d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg = list_entry(cli->cl_lru_list.next, struct osc_page, 687d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao ops_lru); 688d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao page = cl_page_top(opg->ops_cl.cpl_page); 689d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (cl_page_in_use_noref(page)) { 690d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_move_tail(&opg->ops_lru, &cli->cl_lru_list); 691d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao continue; 692d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 693d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 694d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(page->cp_obj != NULL); 695d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (clobj != page->cp_obj) { 696d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_object *tmp = page->cp_obj; 697d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 698d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_object_get(tmp); 699d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_unlock(&cli->cl_lru_list_lock); 700d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 701d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (clobj != NULL) { 702d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao count -= discard_pagevec(env, io, pvec, index); 703d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao index = 0; 704d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 705d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_io_fini(env, io); 706d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_object_put(env, clobj); 707d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao clobj = NULL; 708d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 709d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 710d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao clobj = tmp; 711d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao io->ci_obj = clobj; 712d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao io->ci_ignore_layout = 1; 713d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = cl_io_init(env, io, CIT_MISC, clobj); 714d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 715d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_lock(&cli->cl_lru_list_lock); 716d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 717d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc != 0) 718d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao break; 719d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 720d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao ++maxscan; 721d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao continue; 722d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 723d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 724d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* move this page to the end of list as it will be discarded 725d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * soon. The page will be finally removed from LRU list in 726d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * osc_page_delete(). */ 727d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_move_tail(&opg->ops_lru, &cli->cl_lru_list); 728d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 729d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* it's okay to grab a refcount here w/o holding lock because 730d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * it has to grab cl_lru_list_lock to delete the page. */ 731d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_get(page); 732d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao pvec[index++] = page; 733d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (++count >= target) 734d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao break; 735d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 736d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (unlikely(index == OTI_PVEC_SIZE)) { 737d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_unlock(&cli->cl_lru_list_lock); 738d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao count -= discard_pagevec(env, io, pvec, index); 739d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao index = 0; 740d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 741d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_lock(&cli->cl_lru_list_lock); 742d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 743d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 744d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_unlock(&cli->cl_lru_list_lock); 745d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 746d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (clobj != NULL) { 747d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao count -= discard_pagevec(env, io, pvec, index); 748d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 749d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_io_fini(env, io); 750d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_object_put(env, clobj); 751d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 752d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_env_nested_put(&nest, env); 753d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 754d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_dec(&cli->cl_lru_shrinkers); 7550a3bdb00710bf253ba8ba8f645645f22297c7a04Greg Kroah-Hartman return count > 0 ? count : rc; 756d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 757d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 758d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_lru_add(struct client_obd *cli, struct osc_page *opg) 759d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 760d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao bool wakeup = false; 761d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 762d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (!opg->ops_in_lru) 763d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return; 764d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 765d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_dec(&cli->cl_lru_busy); 766d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_lock(&cli->cl_lru_list_lock); 767d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (list_empty(&opg->ops_lru)) { 768d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_move_tail(&opg->ops_lru, &cli->cl_lru_list); 769d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_inc_return(&cli->cl_lru_in_list); 770d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao wakeup = atomic_read(&osc_lru_waiters) > 0; 771d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 772d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_unlock(&cli->cl_lru_list_lock); 773d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 774d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (wakeup) { 775d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_lru_shrink(cli, osc_cache_too_much(cli)); 776d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao wake_up_all(&osc_lru_waitq); 777d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 778d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 779d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 780d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* delete page from LRUlist. The page can be deleted from LRUlist for two 781d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * reasons: redirtied or deleted from page cache. */ 782d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_lru_del(struct client_obd *cli, struct osc_page *opg, bool del) 783d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 784d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (opg->ops_in_lru) { 785d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_lock(&cli->cl_lru_list_lock); 786d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (!list_empty(&opg->ops_lru)) { 787d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(atomic_read(&cli->cl_lru_in_list) > 0); 788d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_del_init(&opg->ops_lru); 789d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_dec(&cli->cl_lru_in_list); 790d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (!del) 791d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_inc(&cli->cl_lru_busy); 792d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } else if (del) { 793d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(atomic_read(&cli->cl_lru_busy) > 0); 794d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_dec(&cli->cl_lru_busy); 795d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 796d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_unlock(&cli->cl_lru_list_lock); 797d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (del) { 798d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_inc(cli->cl_lru_left); 799d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* this is a great place to release more LRU pages if 800d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * this osc occupies too many LRU pages and kernel is 801d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * stealing one of them. 802d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * cl_lru_shrinkers is to avoid recursive call in case 803d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * we're already in the context of osc_lru_shrink(). */ 804cad6fafa68de08570b499396d82a62c68140bfd3Bobi Jam if (atomic_read(&cli->cl_lru_shrinkers) == 0 && 805cad6fafa68de08570b499396d82a62c68140bfd3Bobi Jam !memory_pressure_get()) 806d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_lru_shrink(cli, osc_cache_too_much(cli)); 807d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao wake_up(&osc_lru_waitq); 808d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 809d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } else { 810d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(list_empty(&opg->ops_lru)); 811d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 812d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 813d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 814d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic inline int max_to_shrink(struct client_obd *cli) 815d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 816d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return min(atomic_read(&cli->cl_lru_in_list) >> 1, lru_shrink_max); 817d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 818d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 819d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_lru_reclaim(struct client_obd *cli) 820d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 821d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_client_cache *cache = cli->cl_cache; 822d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int max_scans; 823d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int rc; 824d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 825d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(cache != NULL); 826d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(!list_empty(&cache->ccc_lru)); 827d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 828d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = osc_lru_shrink(cli, lru_shrink_min); 829d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc != 0) { 830d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao CDEBUG(D_CACHE, "%s: Free %d pages from own LRU: %p.\n", 831d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli->cl_import->imp_obd->obd_name, rc, cli); 832d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return rc; 833d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 834d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 835d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao CDEBUG(D_CACHE, "%s: cli %p no free slots, pages: %d, busy: %d.\n", 836d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli->cl_import->imp_obd->obd_name, cli, 837d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(&cli->cl_lru_in_list), 838d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(&cli->cl_lru_busy)); 839d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 840d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* Reclaim LRU slots from other client_obd as it can't free enough 841d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * from its own. This should rarely happen. */ 842d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_lock(&cache->ccc_lru_lock); 843d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cache->ccc_lru_shrinkers++; 844d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_move_tail(&cli->cl_lru_osc, &cache->ccc_lru); 845d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 846d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao max_scans = atomic_read(&cache->ccc_users); 847d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao while (--max_scans > 0 && !list_empty(&cache->ccc_lru)) { 848d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli = list_entry(cache->ccc_lru.next, struct client_obd, 849d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_lru_osc); 850d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 851d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao CDEBUG(D_CACHE, "%s: cli %p LRU pages: %d, busy: %d.\n", 852d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli->cl_import->imp_obd->obd_name, cli, 853d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(&cli->cl_lru_in_list), 854d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(&cli->cl_lru_busy)); 855d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 856d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_move_tail(&cli->cl_lru_osc, &cache->ccc_lru); 857d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (atomic_read(&cli->cl_lru_in_list) > 0) { 858d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_unlock(&cache->ccc_lru_lock); 859d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 860d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = osc_lru_shrink(cli, max_to_shrink(cli)); 861d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_lock(&cache->ccc_lru_lock); 862d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc != 0) 863d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao break; 864d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 865d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 866d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_unlock(&cache->ccc_lru_lock); 867d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 868d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao CDEBUG(D_CACHE, "%s: cli %p freed %d pages.\n", 869d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli->cl_import->imp_obd->obd_name, cli, rc); 870d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return rc; 871d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 872d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 873d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_lru_reserve(const struct lu_env *env, struct osc_object *obj, 874d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg) 875d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 876d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct l_wait_info lwi = LWI_INTR(LWI_ON_SIGNAL_NOOP, NULL); 877d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct client_obd *cli = osc_cli(obj); 878d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int rc = 0; 879d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 880d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (cli->cl_cache == NULL) /* shall not be in LRU */ 8810a3bdb00710bf253ba8ba8f645645f22297c7a04Greg Kroah-Hartman return 0; 882d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 883d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(atomic_read(cli->cl_lru_left) >= 0); 884305ec768ff192996b4dcb326c5980cd8a96fa948Greg Kroah-Hartman while (!atomic_add_unless(cli->cl_lru_left, -1, 0)) { 885d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int gen; 886d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 887d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* run out of LRU spaces, try to drop some by itself */ 888d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = osc_lru_reclaim(cli); 889d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc < 0) 890d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao break; 891d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc > 0) 892d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao continue; 893d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 894d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cond_resched(); 895d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 896d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* slowest case, all of caching pages are busy, notifying 897d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * other OSCs that we're lack of LRU slots. */ 898d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_inc(&osc_lru_waiters); 899d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 900d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao gen = atomic_read(&cli->cl_lru_in_list); 901d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = l_wait_event(osc_lru_waitq, 902d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(cli->cl_lru_left) > 0 || 903d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao (atomic_read(&cli->cl_lru_in_list) > 0 && 904d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao gen != atomic_read(&cli->cl_lru_in_list)), 905d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao &lwi); 906d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 907d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_dec(&osc_lru_waiters); 908d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc < 0) 909d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao break; 910d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 911d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 912d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc >= 0) { 913d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_inc(&cli->cl_lru_busy); 914d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_in_lru = 1; 915d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = 0; 916d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 917d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 9180a3bdb00710bf253ba8ba8f645645f22297c7a04Greg Kroah-Hartman return rc; 919d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 920d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 921d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/** @} osc */ 922