osc_page.c revision 29aaf4962a3bce337d37176858ef1025b9f29cc4
1d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* 2d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * GPL HEADER START 3d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 4d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 6d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * This program is free software; you can redistribute it and/or modify 7d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * it under the terms of the GNU General Public License version 2 only, 8d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * as published by the Free Software Foundation. 9d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 10d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * This program is distributed in the hope that it will be useful, but 11d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * WITHOUT ANY WARRANTY; without even the implied warranty of 12d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * General Public License version 2 for more details (a copy is included 14d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * in the LICENSE file that accompanied this code). 15d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 16d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * You should have received a copy of the GNU General Public License 17d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * version 2 along with this program; If not, see 18d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf 19d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 20d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, 21d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * CA 95054 USA or visit www.sun.com if you need additional information or 22d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * have any questions. 23d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 24d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * GPL HEADER END 25d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 26d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* 27d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 28d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Use is subject to license terms. 29d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 30d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Copyright (c) 2011, 2012, Intel Corporation. 31d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 32d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* 33d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * This file is part of Lustre, http://www.lustre.org/ 34d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Lustre is a trademark of Sun Microsystems, Inc. 35d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 36d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Implementation of cl_page for OSC layer. 37d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 38d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Author: Nikita Danilov <nikita.danilov@sun.com> 39d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 40d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 41d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao#define DEBUG_SUBSYSTEM S_OSC 42d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 43d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao#include "osc_cl_internal.h" 44d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 45d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_lru_del(struct client_obd *cli, struct osc_page *opg, bool del); 46d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_lru_add(struct client_obd *cli, struct osc_page *opg); 47d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_lru_reserve(const struct lu_env *env, struct osc_object *obj, 48d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg); 49d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 50d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/** \addtogroup osc 51d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * @{ 52d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 53d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 54d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* 55d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Comment out osc_page_protected because it may sleep inside the 56d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * the client_obd_list_lock. 57d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * client_obd_list_lock -> osc_ap_completion -> osc_completion -> 58d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * -> osc_page_protected -> osc_page_is_dlocked -> osc_match_base 59d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * -> ldlm_lock_match -> sptlrpc_import_check_ctx -> sleep. 60d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 61d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao#if 0 62d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_is_dlocked(const struct lu_env *env, 63d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct osc_page *opg, 64d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao enum cl_lock_mode mode, int pending, int unref) 65d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 66d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page *page; 67d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *obj; 68d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_thread_info *info; 69d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct ldlm_res_id *resname; 70d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct lustre_handle *lockh; 71d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao ldlm_policy_data_t *policy; 72d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao ldlm_mode_t dlmmode; 73d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int flags; 74d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 75d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao might_sleep(); 76d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 77d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao info = osc_env_info(env); 78d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao resname = &info->oti_resname; 79d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao policy = &info->oti_policy; 80d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao lockh = &info->oti_handle; 81d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao page = opg->ops_cl.cpl_page; 82d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao obj = cl2osc(opg->ops_cl.cpl_obj); 83d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 84d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao flags = LDLM_FL_TEST_LOCK | LDLM_FL_BLOCK_GRANTED; 85d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (pending) 86d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao flags |= LDLM_FL_CBPENDING; 87d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 88d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao dlmmode = osc_cl_lock2ldlm(mode) | LCK_PW; 89d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_lock_build_res(env, obj, resname); 90d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_index2policy(policy, page->cp_obj, page->cp_index, page->cp_index); 91d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return osc_match_base(osc_export(obj), resname, LDLM_EXTENT, policy, 92d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao dlmmode, &flags, NULL, lockh, unref); 93d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 94d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 95d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/** 96d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Checks an invariant that a page in the cache is covered by a lock, as 97d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * needed. 98d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 99d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_protected(const struct lu_env *env, 100d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct osc_page *opg, 101d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao enum cl_lock_mode mode, int unref) 102d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 103d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_object_header *hdr; 104d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_lock *scan; 105d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page *page; 106d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_lock_descr *descr; 107d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int result; 108d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 109d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LINVRNT(!opg->ops_temp); 110d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 111d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao page = opg->ops_cl.cpl_page; 112d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (page->cp_owner != NULL && 113d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_io_top(page->cp_owner)->ci_lockreq == CILR_NEVER) 114d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 115d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * If IO is done without locks (liblustre, or lloop), lock is 116d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * not required. 117d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 118d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao result = 1; 119d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao else 120d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* otherwise check for a DLM lock */ 121d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao result = osc_page_is_dlocked(env, opg, mode, 1, unref); 122d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (result == 0) { 123d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* maybe this page is a part of a lockless io? */ 124d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao hdr = cl_object_header(opg->ops_cl.cpl_obj); 125d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao descr = &osc_env_info(env)->oti_descr; 126d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao descr->cld_mode = mode; 127d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao descr->cld_start = page->cp_index; 128d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao descr->cld_end = page->cp_index; 129d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_lock(&hdr->coh_lock_guard); 130d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_for_each_entry(scan, &hdr->coh_locks, cll_linkage) { 131d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 132d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Lock-less sub-lock has to be either in HELD state 133d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * (when io is actively going on), or in CACHED state, 134d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * when top-lock is being unlocked: 135d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * cl_io_unlock()->cl_unuse()->...->lov_lock_unuse(). 136d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 137d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if ((scan->cll_state == CLS_HELD || 138d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao scan->cll_state == CLS_CACHED) && 139d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_lock_ext_match(&scan->cll_descr, descr)) { 140d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_lock *olck; 141d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 142d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao olck = osc_lock_at(scan); 143d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao result = osc_lock_is_lockless(olck); 144d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao break; 145d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 146d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 147d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_unlock(&hdr->coh_lock_guard); 148d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 149d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return result; 150d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 151d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao#else 152d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_protected(const struct lu_env *env, 153d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct osc_page *opg, 154d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao enum cl_lock_mode mode, int unref) 155d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 156d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return 1; 157d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 158d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao#endif 159d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 160d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/***************************************************************************** 161d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 162d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Page operations. 163d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 164d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 165d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_fini(const struct lu_env *env, 166d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page_slice *slice) 167d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 168d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 169d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao CDEBUG(D_TRACE, "%p\n", opg); 170d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(opg->ops_lock == NULL); 171d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 172d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 173d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_transfer_get(struct osc_page *opg, const char *label) 174d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 175d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page *page = cl_page_top(opg->ops_cl.cpl_page); 176d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 177d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(!opg->ops_transfer_pinned); 178d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_get(page); 179d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao lu_ref_add_atomic(&page->cp_reference, label, page); 180d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_transfer_pinned = 1; 181d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 182d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 183d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_transfer_put(const struct lu_env *env, 184d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg) 185d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 186d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page *page = cl_page_top(opg->ops_cl.cpl_page); 187d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 188d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (opg->ops_transfer_pinned) { 189d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao lu_ref_del(&page->cp_reference, "transfer", page); 190d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_transfer_pinned = 0; 191d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_put(env, page); 192d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 193d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 194d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 195d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/** 196d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * This is called once for every page when it is submitted for a transfer 197d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * either opportunistic (osc_page_cache_add()), or immediate 198d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * (osc_page_submit()). 199d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 200d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_transfer_add(const struct lu_env *env, 201d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg, enum cl_req_type crt) 202d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 203d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *obj = cl2osc(opg->ops_cl.cpl_obj); 204d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 205d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* ops_lru and ops_inflight share the same field, so take it from LRU 206d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * first and then use it as inflight. */ 207d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_lru_del(osc_cli(obj), opg, false); 208d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 209d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_lock(&obj->oo_seatbelt); 210d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_add(&opg->ops_inflight, &obj->oo_inflight[crt]); 211d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_submitter = current; 212d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_unlock(&obj->oo_seatbelt); 213d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 214d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 215d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_cache_add(const struct lu_env *env, 216d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 217d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_io *io) 218d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 219d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_io *oio = osc_env_io(env); 220d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 221d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int result; 222d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 223d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LINVRNT(osc_page_protected(env, opg, CLM_WRITE, 0)); 224d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 225d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_transfer_get(opg, "transfer\0cache"); 226d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao result = osc_queue_async_io(env, io, opg); 227d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (result != 0) 228d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_transfer_put(env, opg); 229d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao else 230d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_transfer_add(env, opg, CRT_WRITE); 231d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 232d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* for sync write, kernel will wait for this page to be flushed before 233d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * osc_io_end() is called, so release it earlier. 234d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * for mkwrite(), it's known there is no further pages. */ 235d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (cl_io_is_sync_write(io) || cl_io_is_mkwrite(io)) { 236d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (oio->oi_active != NULL) { 237d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_extent_release(env, oio->oi_active); 238d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oio->oi_active = NULL; 239d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 240d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 241d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 242d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao RETURN(result); 243d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 244d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 245d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taovoid osc_index2policy(ldlm_policy_data_t *policy, const struct cl_object *obj, 246d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao pgoff_t start, pgoff_t end) 247d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 248d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao memset(policy, 0, sizeof *policy); 249d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao policy->l_extent.start = cl_offset(obj, start); 250d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao policy->l_extent.end = cl_offset(obj, end + 1) - 1; 251d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 252d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 253d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_addref_lock(const struct lu_env *env, 254d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg, 255d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_lock *lock) 256d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 257d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_lock *olock; 258d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int rc; 259d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 260d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(opg->ops_lock == NULL); 261d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 262d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao olock = osc_lock_at(lock); 263d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (atomic_inc_return(&olock->ols_pageref) <= 0) { 264d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_dec(&olock->ols_pageref); 265d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = -ENODATA; 266d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } else { 267d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_lock_get(lock); 268d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_lock = lock; 269d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = 0; 270d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 271d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return rc; 272d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 273d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 274d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_putref_lock(const struct lu_env *env, 275d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg) 276d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 277d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_lock *lock = opg->ops_lock; 278d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_lock *olock; 279d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 280d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(lock != NULL); 281d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao olock = osc_lock_at(lock); 282d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 283d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_dec(&olock->ols_pageref); 284d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_lock = NULL; 285d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 286d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_lock_put(env, lock); 287d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 288d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 289d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_is_under_lock(const struct lu_env *env, 290d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 291d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_io *unused) 292d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 293d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_lock *lock; 294d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int result = -ENODATA; 295d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 296d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao lock = cl_lock_at_page(env, slice->cpl_obj, slice->cpl_page, 297d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao NULL, 1, 0); 298d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (lock != NULL) { 299d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (osc_page_addref_lock(env, cl2osc_page(slice), lock) == 0) 300d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao result = -EBUSY; 301d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_lock_put(env, lock); 302d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 303d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao RETURN(result); 304d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 305d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 306d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_disown(const struct lu_env *env, 307d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 308d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_io *io) 309d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 310d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 311d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 312d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (unlikely(opg->ops_lock)) 313d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_putref_lock(env, opg); 314d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 315d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 316d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_completion_read(const struct lu_env *env, 317d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 318d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int ioret) 319d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 320d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 321d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *obj = cl2osc(opg->ops_cl.cpl_obj); 322d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 323d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (likely(opg->ops_lock)) 324d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_putref_lock(env, opg); 325d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_lru_add(osc_cli(obj), opg); 326d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 327d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 328d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_completion_write(const struct lu_env *env, 329d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 330d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int ioret) 331d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 332d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 333d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *obj = cl2osc(slice->cpl_obj); 334d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 335d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_lru_add(osc_cli(obj), opg); 336d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 337d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 338d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_fail(const struct lu_env *env, 339d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 340d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_io *unused) 341d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 342d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 343d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Cached read? 344d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 345d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LBUG(); 346d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return 0; 347d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 348d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 349d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 350d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic const char *osc_list(struct list_head *head) 351d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 352d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return list_empty(head) ? "-" : "+"; 353d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 354d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 355d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic inline cfs_time_t osc_submit_duration(struct osc_page *opg) 356d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 357d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (opg->ops_submit_time == 0) 358d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return 0; 359d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 360d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return (cfs_time_current() - opg->ops_submit_time); 361d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 362d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 363d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_print(const struct lu_env *env, 364d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 365d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao void *cookie, lu_printer_t printer) 366d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 367d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 368d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_async_page *oap = &opg->ops_oap; 369d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *obj = cl2osc(slice->cpl_obj); 370d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct client_obd *cli = &osc_export(obj)->exp_obd->u.cli; 371d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 372d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return (*printer)(env, cookie, LUSTRE_OSC_NAME"-page@%p: " 373d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao "1< %#x %d %u %s %s > " 374d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao "2< "LPU64" %u %u %#x %#x | %p %p %p > " 375d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao "3< %s %p %d %lu %d > " 376d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao "4< %d %d %d %lu %s | %s %s %s %s > " 377d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao "5< %s %s %s %s | %d %s | %d %s %s>\n", 378d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg, 379d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 1 */ 380d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_magic, oap->oap_cmd, 381d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_interrupted, 382d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&oap->oap_pending_item), 383d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&oap->oap_rpc_item), 384d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 2 */ 385d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_obj_off, oap->oap_page_off, oap->oap_count, 386d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_async_flags, oap->oap_brw_flags, 387d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_request, oap->oap_cli, obj, 388d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 3 */ 389d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&opg->ops_inflight), 390d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_submitter, opg->ops_transfer_pinned, 391d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_submit_duration(opg), opg->ops_srvlock, 392d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 4 */ 393d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli->cl_r_in_flight, cli->cl_w_in_flight, 394d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli->cl_max_rpcs_in_flight, 395d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli->cl_avail_grant, 396d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&cli->cl_cache_waiters), 397d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&cli->cl_loi_ready_list), 398d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&cli->cl_loi_hp_ready_list), 399d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&cli->cl_loi_write_list), 400d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&cli->cl_loi_read_list), 401d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 5 */ 402d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&obj->oo_ready_item), 403d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&obj->oo_hp_ready_item), 404d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&obj->oo_write_item), 405d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&obj->oo_read_item), 406d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(&obj->oo_nr_reads), 407d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&obj->oo_reading_exts), 408d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(&obj->oo_nr_writes), 409d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&obj->oo_hp_exts), 410d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_list(&obj->oo_urgent_exts)); 411d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 412d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 413d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_page_delete(const struct lu_env *env, 414d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice) 415d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 416d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 417d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *obj = cl2osc(opg->ops_cl.cpl_obj); 418d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int rc; 419d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 420d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LINVRNT(opg->ops_temp || osc_page_protected(env, opg, CLM_READ, 1)); 421d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 422d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao CDEBUG(D_TRACE, "%p\n", opg); 423d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_transfer_put(env, opg); 424d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = osc_teardown_async_page(env, obj, opg); 425d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc) { 426d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao CL_PAGE_DEBUG(D_ERROR, env, cl_page_top(slice->cpl_page), 427d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao "Trying to teardown failed: %d\n", rc); 428d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(0); 429d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 430d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 431d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_lock(&obj->oo_seatbelt); 432d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (opg->ops_submitter != NULL) { 433d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(!list_empty(&opg->ops_inflight)); 434d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_del_init(&opg->ops_inflight); 435d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_submitter = NULL; 436d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 437d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_unlock(&obj->oo_seatbelt); 438d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 439d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_lru_del(osc_cli(obj), opg, true); 440d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao EXIT; 441d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 442d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 443d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taovoid osc_page_clip(const struct lu_env *env, const struct cl_page_slice *slice, 444d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int from, int to) 445d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 446d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 447d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_async_page *oap = &opg->ops_oap; 448d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 449d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LINVRNT(osc_page_protected(env, opg, CLM_READ, 0)); 450d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 451d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_from = from; 452d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_to = to; 453d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_lock(&oap->oap_lock); 454d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_async_flags |= ASYNC_COUNT_STABLE; 455d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_unlock(&oap->oap_lock); 456d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 457d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 458d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_cancel(const struct lu_env *env, 459d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice) 460d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 461d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 462d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int rc = 0; 463d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 464d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LINVRNT(osc_page_protected(env, opg, CLM_READ, 0)); 465d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 466d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* Check if the transferring against this page 467d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * is completed, or not even queued. */ 468d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (opg->ops_transfer_pinned) 469d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* FIXME: may not be interrupted.. */ 470d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = osc_cancel_async_page(env, opg); 471d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(ergo(rc == 0, opg->ops_transfer_pinned == 0)); 472d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return rc; 473d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 474d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 475d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_page_flush(const struct lu_env *env, 476d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao const struct cl_page_slice *slice, 477d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_io *io) 478d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 479d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl2osc_page(slice); 480d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int rc = 0; 48129aaf4962a3bce337d37176858ef1025b9f29cc4Greg Kroah-Hartman 482d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = osc_flush_async_page(env, io, opg); 483d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao RETURN(rc); 484d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 485d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 486d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic const struct cl_page_operations osc_page_ops = { 487d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_fini = osc_page_fini, 488d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_print = osc_page_print, 489d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_delete = osc_page_delete, 490d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_is_under_lock = osc_page_is_under_lock, 491d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_disown = osc_page_disown, 492d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .io = { 493d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao [CRT_READ] = { 494d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_cache_add = osc_page_fail, 495d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_completion = osc_page_completion_read 496d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao }, 497d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao [CRT_WRITE] = { 498d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_cache_add = osc_page_cache_add, 499d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_completion = osc_page_completion_write 500d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 501d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao }, 502d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_clip = osc_page_clip, 503d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_cancel = osc_page_cancel, 504d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao .cpo_flush = osc_page_flush 505d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao}; 506d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 507d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taoint osc_page_init(const struct lu_env *env, struct cl_object *obj, 508d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page *page, struct page *vmpage) 509d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 510d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *osc = cl2osc(obj); 511d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg = cl_object_page_slice(obj, page); 512d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int result; 513d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 514d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_from = 0; 515d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_to = PAGE_CACHE_SIZE; 516d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 517d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao result = osc_prep_async_page(osc, opg, vmpage, 518d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_offset(obj, page->cp_index)); 519d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (result == 0) { 520d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_io *oio = osc_env_io(env); 521d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_srvlock = osc_io_srvlock(oio); 522d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_slice_add(page, &opg->ops_cl, obj, 523d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao &osc_page_ops); 524d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 525d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* 526d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Cannot assert osc_page_protected() here as read-ahead 527d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * creates temporary pages outside of a lock. 528d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 529d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* ops_inflight and ops_lru are the same field, but it doesn't 530d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * hurt to initialize it twice :-) */ 531d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao INIT_LIST_HEAD(&opg->ops_inflight); 532d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao INIT_LIST_HEAD(&opg->ops_lru); 533d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 534d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* reserve an LRU space for this page */ 535d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (page->cp_type == CPT_CACHEABLE && result == 0) 536d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao result = osc_lru_reserve(env, osc, opg); 537d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 538d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return result; 539d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 540d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 541d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/** 542d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Helper function called by osc_io_submit() for every page in an immediate 543d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * transfer (i.e., transferred synchronously). 544d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 545d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taovoid osc_page_submit(const struct lu_env *env, struct osc_page *opg, 546d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao enum cl_req_type crt, int brw_flags) 547d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 548d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_async_page *oap = &opg->ops_oap; 549d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_object *obj = oap->oap_obj; 550d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 551d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LINVRNT(osc_page_protected(env, opg, 552d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao crt == CRT_WRITE ? CLM_WRITE : CLM_READ, 1)); 553d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 554d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERTF(oap->oap_magic == OAP_MAGIC, "Bad oap magic: oap %p, " 555d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao "magic 0x%x\n", oap, oap->oap_magic); 556d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(oap->oap_async_flags & ASYNC_READY); 557d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(oap->oap_async_flags & ASYNC_COUNT_STABLE); 558d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 559d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_cmd = crt == CRT_WRITE ? OBD_BRW_WRITE : OBD_BRW_READ; 560d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_page_off = opg->ops_from; 561d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_count = opg->ops_to - opg->ops_from; 562d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_brw_flags = OBD_BRW_SYNC | brw_flags; 563d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 564d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (!client_is_remote(osc_export(obj)) && 565d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cfs_capable(CFS_CAP_SYS_RESOURCE)) { 566d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_brw_flags |= OBD_BRW_NOQUOTA; 567d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao oap->oap_cmd |= OBD_BRW_NOQUOTA; 568d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 569d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 570d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_submit_time = cfs_time_current(); 571d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_transfer_get(opg, "transfer\0imm"); 572d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_page_transfer_add(env, opg, crt); 573d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 574d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 575d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* --------------- LRU page management ------------------ */ 576d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 577d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* OSC is a natural place to manage LRU pages as applications are specialized 578d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * to write OSC by OSC. Ideally, if one OSC is used more frequently it should 579d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * occupy more LRU slots. On the other hand, we should avoid using up all LRU 580d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * slots (client_obd::cl_lru_left) otherwise process has to be put into sleep 581d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * for free LRU slots - this will be very bad so the algorithm requires each 582d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * OSC to free slots voluntarily to maintain a reasonable number of free slots 583d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * at any time. 584d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 585d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 586d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic CFS_DECL_WAITQ(osc_lru_waitq); 587d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic atomic_t osc_lru_waiters = ATOMIC_INIT(0); 588d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* LRU pages are freed in batch mode. OSC should at least free this 589d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * number of pages to avoid running out of LRU budget, and.. */ 590d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic const int lru_shrink_min = 2 << (20 - PAGE_CACHE_SHIFT); /* 2M */ 591d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* free this number at most otherwise it will take too long time to finsih. */ 592d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic const int lru_shrink_max = 32 << (20 - PAGE_CACHE_SHIFT); /* 32M */ 593d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 594d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* Check if we can free LRU slots from this OSC. If there exists LRU waiters, 595d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * we should free slots aggressively. In this way, slots are freed in a steady 596d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * step to maintain fairness among OSCs. 597d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * 598d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Return how many LRU pages should be freed. */ 599d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_cache_too_much(struct client_obd *cli) 600d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 601d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_client_cache *cache = cli->cl_cache; 602d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int pages = atomic_read(&cli->cl_lru_in_list) >> 1; 603d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 604d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (atomic_read(&osc_lru_waiters) > 0 && 605d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(cli->cl_lru_left) < lru_shrink_max) 606d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* drop lru pages aggressively */ 607d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return min(pages, lru_shrink_max); 608d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 609d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* if it's going to run out LRU slots, we should free some, but not 610d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * too much to maintain faireness among OSCs. */ 611d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (atomic_read(cli->cl_lru_left) < cache->ccc_lru_max >> 4) { 612d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao unsigned long tmp; 613d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 614d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao tmp = cache->ccc_lru_max / atomic_read(&cache->ccc_users); 615d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (pages > tmp) 616d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return min(pages, lru_shrink_max); 617d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 618d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return pages > lru_shrink_min ? lru_shrink_min : 0; 619d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 620d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 621d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return 0; 622d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 623d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 624d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* Return how many pages are not discarded in @pvec. */ 625d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int discard_pagevec(const struct lu_env *env, struct cl_io *io, 626d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page **pvec, int max_index) 627d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 628d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int count; 629d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int i; 630d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 631d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao for (count = 0, i = 0; i < max_index; i++) { 632d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page *page = pvec[i]; 633d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (cl_page_own_try(env, io, page) == 0) { 634d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* free LRU page only if nobody is using it. 635d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * This check is necessary to avoid freeing the pages 636d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * having already been removed from LRU and pinned 637d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * for IO. */ 638d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (!cl_page_in_use(page)) { 639d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_unmap(env, io, page); 640d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_discard(env, io, page); 641d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao ++count; 642d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 643d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_disown(env, io, page); 644d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 645d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_put(env, page); 646d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao pvec[i] = NULL; 647d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 648d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return max_index - count; 649d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 650d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 651d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/** 652d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * Drop @target of pages from LRU at most. 653d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao */ 654d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taoint osc_lru_shrink(struct client_obd *cli, int target) 655d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 656d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_env_nest nest; 657d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct lu_env *env; 658d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_io *io; 659d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_object *clobj = NULL; 660d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page **pvec; 661d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg; 662d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int maxscan = 0; 663d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int count = 0; 664d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int index = 0; 665d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int rc = 0; 666d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 667d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(atomic_read(&cli->cl_lru_in_list) >= 0); 668d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (atomic_read(&cli->cl_lru_in_list) == 0 || target <= 0) 669d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao RETURN(0); 670d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 671d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao env = cl_env_nested_get(&nest); 672d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (IS_ERR(env)) 673d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao RETURN(PTR_ERR(env)); 674d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 675d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao pvec = osc_env_info(env)->oti_pvec; 676d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao io = &osc_env_info(env)->oti_io; 677d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 678d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_lock(&cli->cl_lru_list_lock); 679d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_inc(&cli->cl_lru_shrinkers); 680d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao maxscan = min(target << 1, atomic_read(&cli->cl_lru_in_list)); 681d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao while (!list_empty(&cli->cl_lru_list)) { 682d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_page *page; 683d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 684d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (--maxscan < 0) 685d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao break; 686d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 687d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg = list_entry(cli->cl_lru_list.next, struct osc_page, 688d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao ops_lru); 689d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao page = cl_page_top(opg->ops_cl.cpl_page); 690d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (cl_page_in_use_noref(page)) { 691d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_move_tail(&opg->ops_lru, &cli->cl_lru_list); 692d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao continue; 693d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 694d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 695d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(page->cp_obj != NULL); 696d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (clobj != page->cp_obj) { 697d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_object *tmp = page->cp_obj; 698d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 699d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_object_get(tmp); 700d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_unlock(&cli->cl_lru_list_lock); 701d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 702d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (clobj != NULL) { 703d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao count -= discard_pagevec(env, io, pvec, index); 704d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao index = 0; 705d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 706d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_io_fini(env, io); 707d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_object_put(env, clobj); 708d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao clobj = NULL; 709d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 710d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 711d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao clobj = tmp; 712d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao io->ci_obj = clobj; 713d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao io->ci_ignore_layout = 1; 714d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = cl_io_init(env, io, CIT_MISC, clobj); 715d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 716d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_lock(&cli->cl_lru_list_lock); 717d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 718d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc != 0) 719d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao break; 720d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 721d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao ++maxscan; 722d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao continue; 723d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 724d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 725d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* move this page to the end of list as it will be discarded 726d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * soon. The page will be finally removed from LRU list in 727d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * osc_page_delete(). */ 728d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_move_tail(&opg->ops_lru, &cli->cl_lru_list); 729d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 730d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* it's okay to grab a refcount here w/o holding lock because 731d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * it has to grab cl_lru_list_lock to delete the page. */ 732d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_page_get(page); 733d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao pvec[index++] = page; 734d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (++count >= target) 735d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao break; 736d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 737d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (unlikely(index == OTI_PVEC_SIZE)) { 738d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_unlock(&cli->cl_lru_list_lock); 739d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao count -= discard_pagevec(env, io, pvec, index); 740d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao index = 0; 741d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 742d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_lock(&cli->cl_lru_list_lock); 743d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 744d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 745d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_unlock(&cli->cl_lru_list_lock); 746d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 747d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (clobj != NULL) { 748d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao count -= discard_pagevec(env, io, pvec, index); 749d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 750d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_io_fini(env, io); 751d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_object_put(env, clobj); 752d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 753d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_env_nested_put(&nest, env); 754d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 755d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_dec(&cli->cl_lru_shrinkers); 756d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao RETURN(count > 0 ? count : rc); 757d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 758d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 759d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_lru_add(struct client_obd *cli, struct osc_page *opg) 760d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 761d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao bool wakeup = false; 762d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 763d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (!opg->ops_in_lru) 764d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return; 765d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 766d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_dec(&cli->cl_lru_busy); 767d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_lock(&cli->cl_lru_list_lock); 768d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (list_empty(&opg->ops_lru)) { 769d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_move_tail(&opg->ops_lru, &cli->cl_lru_list); 770d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_inc_return(&cli->cl_lru_in_list); 771d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao wakeup = atomic_read(&osc_lru_waiters) > 0; 772d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 773d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_unlock(&cli->cl_lru_list_lock); 774d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 775d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (wakeup) { 776d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_lru_shrink(cli, osc_cache_too_much(cli)); 777d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao wake_up_all(&osc_lru_waitq); 778d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 779d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 780d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 781d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/* delete page from LRUlist. The page can be deleted from LRUlist for two 782d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * reasons: redirtied or deleted from page cache. */ 783d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic void osc_lru_del(struct client_obd *cli, struct osc_page *opg, bool del) 784d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 785d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (opg->ops_in_lru) { 786d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_lock(&cli->cl_lru_list_lock); 787d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (!list_empty(&opg->ops_lru)) { 788d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(atomic_read(&cli->cl_lru_in_list) > 0); 789d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_del_init(&opg->ops_lru); 790d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_dec(&cli->cl_lru_in_list); 791d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (!del) 792d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_inc(&cli->cl_lru_busy); 793d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } else if (del) { 794d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(atomic_read(&cli->cl_lru_busy) > 0); 795d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_dec(&cli->cl_lru_busy); 796d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 797d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao client_obd_list_unlock(&cli->cl_lru_list_lock); 798d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (del) { 799d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_inc(cli->cl_lru_left); 800d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* this is a great place to release more LRU pages if 801d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * this osc occupies too many LRU pages and kernel is 802d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * stealing one of them. 803d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * cl_lru_shrinkers is to avoid recursive call in case 804d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * we're already in the context of osc_lru_shrink(). */ 805cad6fafa68de08570b499396d82a62c68140bfd3Bobi Jam if (atomic_read(&cli->cl_lru_shrinkers) == 0 && 806cad6fafa68de08570b499396d82a62c68140bfd3Bobi Jam !memory_pressure_get()) 807d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao osc_lru_shrink(cli, osc_cache_too_much(cli)); 808d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao wake_up(&osc_lru_waitq); 809d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 810d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } else { 811d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(list_empty(&opg->ops_lru)); 812d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 813d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 814d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 815d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic inline int max_to_shrink(struct client_obd *cli) 816d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 817d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return min(atomic_read(&cli->cl_lru_in_list) >> 1, lru_shrink_max); 818d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 819d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 820d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_lru_reclaim(struct client_obd *cli) 821d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 822d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct cl_client_cache *cache = cli->cl_cache; 823d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int max_scans; 824d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int rc; 825d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 826d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(cache != NULL); 827d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(!list_empty(&cache->ccc_lru)); 828d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 829d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = osc_lru_shrink(cli, lru_shrink_min); 830d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc != 0) { 831d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao CDEBUG(D_CACHE, "%s: Free %d pages from own LRU: %p.\n", 832d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli->cl_import->imp_obd->obd_name, rc, cli); 833d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return rc; 834d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 835d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 836d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao CDEBUG(D_CACHE, "%s: cli %p no free slots, pages: %d, busy: %d.\n", 837d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli->cl_import->imp_obd->obd_name, cli, 838d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(&cli->cl_lru_in_list), 839d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(&cli->cl_lru_busy)); 840d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 841d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* Reclaim LRU slots from other client_obd as it can't free enough 842d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * from its own. This should rarely happen. */ 843d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_lock(&cache->ccc_lru_lock); 844d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cache->ccc_lru_shrinkers++; 845d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_move_tail(&cli->cl_lru_osc, &cache->ccc_lru); 846d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 847d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao max_scans = atomic_read(&cache->ccc_users); 848d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao while (--max_scans > 0 && !list_empty(&cache->ccc_lru)) { 849d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli = list_entry(cache->ccc_lru.next, struct client_obd, 850d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cl_lru_osc); 851d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 852d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao CDEBUG(D_CACHE, "%s: cli %p LRU pages: %d, busy: %d.\n", 853d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli->cl_import->imp_obd->obd_name, cli, 854d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(&cli->cl_lru_in_list), 855d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(&cli->cl_lru_busy)); 856d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 857d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao list_move_tail(&cli->cl_lru_osc, &cache->ccc_lru); 858d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (atomic_read(&cli->cl_lru_in_list) > 0) { 859d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_unlock(&cache->ccc_lru_lock); 860d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 861d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = osc_lru_shrink(cli, max_to_shrink(cli)); 862d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_lock(&cache->ccc_lru_lock); 863d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc != 0) 864d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao break; 865d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 866d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 867d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao spin_unlock(&cache->ccc_lru_lock); 868d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 869d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao CDEBUG(D_CACHE, "%s: cli %p freed %d pages.\n", 870d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cli->cl_import->imp_obd->obd_name, cli, rc); 871d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao return rc; 872d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 873d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 874d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Taostatic int osc_lru_reserve(const struct lu_env *env, struct osc_object *obj, 875d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct osc_page *opg) 876d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao{ 877d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct l_wait_info lwi = LWI_INTR(LWI_ON_SIGNAL_NOOP, NULL); 878d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao struct client_obd *cli = osc_cli(obj); 879d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int rc = 0; 880d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 881d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (cli->cl_cache == NULL) /* shall not be in LRU */ 882d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao RETURN(0); 883d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 884d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao LASSERT(atomic_read(cli->cl_lru_left) >= 0); 885d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao while (!cfs_atomic_add_unless(cli->cl_lru_left, -1, 0)) { 886d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao int gen; 887d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 888d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* run out of LRU spaces, try to drop some by itself */ 889d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = osc_lru_reclaim(cli); 890d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc < 0) 891d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao break; 892d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc > 0) 893d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao continue; 894d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 895d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao cond_resched(); 896d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 897d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao /* slowest case, all of caching pages are busy, notifying 898d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao * other OSCs that we're lack of LRU slots. */ 899d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_inc(&osc_lru_waiters); 900d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 901d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao gen = atomic_read(&cli->cl_lru_in_list); 902d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = l_wait_event(osc_lru_waitq, 903d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_read(cli->cl_lru_left) > 0 || 904d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao (atomic_read(&cli->cl_lru_in_list) > 0 && 905d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao gen != atomic_read(&cli->cl_lru_in_list)), 906d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao &lwi); 907d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 908d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_dec(&osc_lru_waiters); 909d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc < 0) 910d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao break; 911d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 912d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 913d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao if (rc >= 0) { 914d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao atomic_inc(&cli->cl_lru_busy); 915d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao opg->ops_in_lru = 1; 916d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao rc = 0; 917d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao } 918d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 919d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao RETURN(rc); 920d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao} 921d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao 922d7e09d0397e84eefbabfd9cb353221f3c6448d83Peng Tao/** @} osc */ 923