1/* 2 * Copyright (C) 2012 Google, Inc. 3 * 4 * This software is licensed under the terms of the GNU General Public 5 * License version 2, as published by the Free Software Foundation, and 6 * may be copied, distributed, and modified under those terms. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * GNU General Public License for more details. 12 * 13 */ 14 15#include <linux/device.h> 16#include <linux/err.h> 17#include <linux/errno.h> 18#include <linux/kernel.h> 19#include <linux/init.h> 20#include <linux/io.h> 21#include <linux/list.h> 22#include <linux/memblock.h> 23#include <linux/persistent_ram.h> 24#include <linux/rslib.h> 25#include <linux/slab.h> 26#include <linux/vmalloc.h> 27 28struct persistent_ram_buffer { 29 uint32_t sig; 30 atomic_t start; 31 atomic_t size; 32 uint8_t data[0]; 33}; 34 35#define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */ 36 37static __devinitdata LIST_HEAD(persistent_ram_list); 38 39static inline size_t buffer_size(struct persistent_ram_zone *prz) 40{ 41 return atomic_read(&prz->buffer->size); 42} 43 44static inline size_t buffer_start(struct persistent_ram_zone *prz) 45{ 46 return atomic_read(&prz->buffer->start); 47} 48 49/* increase and wrap the start pointer, returning the old value */ 50static inline size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a) 51{ 52 int old; 53 int new; 54 55 do { 56 old = atomic_read(&prz->buffer->start); 57 new = old + a; 58 while (unlikely(new > prz->buffer_size)) 59 new -= prz->buffer_size; 60 } while (atomic_cmpxchg(&prz->buffer->start, old, new) != old); 61 62 return old; 63} 64 65/* increase the size counter until it hits the max size */ 66static inline void buffer_size_add(struct persistent_ram_zone *prz, size_t a) 67{ 68 size_t old; 69 size_t new; 70 71 if (atomic_read(&prz->buffer->size) == prz->buffer_size) 72 return; 73 74 do { 75 old = atomic_read(&prz->buffer->size); 76 new = old + a; 77 if (new > prz->buffer_size) 78 new = prz->buffer_size; 79 } while (atomic_cmpxchg(&prz->buffer->size, old, new) != old); 80} 81 82static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz, 83 uint8_t *data, size_t len, uint8_t *ecc) 84{ 85 int i; 86 uint16_t par[prz->ecc_size]; 87 88 /* Initialize the parity buffer */ 89 memset(par, 0, sizeof(par)); 90 encode_rs8(prz->rs_decoder, data, len, par, 0); 91 for (i = 0; i < prz->ecc_size; i++) 92 ecc[i] = par[i]; 93} 94 95static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz, 96 void *data, size_t len, uint8_t *ecc) 97{ 98 int i; 99 uint16_t par[prz->ecc_size]; 100 101 for (i = 0; i < prz->ecc_size; i++) 102 par[i] = ecc[i]; 103 return decode_rs8(prz->rs_decoder, data, par, len, 104 NULL, 0, NULL, 0, NULL); 105} 106 107static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz, 108 unsigned int start, unsigned int count) 109{ 110 struct persistent_ram_buffer *buffer = prz->buffer; 111 uint8_t *buffer_end = buffer->data + prz->buffer_size; 112 uint8_t *block; 113 uint8_t *par; 114 int ecc_block_size = prz->ecc_block_size; 115 int ecc_size = prz->ecc_size; 116 int size = prz->ecc_block_size; 117 118 if (!prz->ecc) 119 return; 120 121 block = buffer->data + (start & ~(ecc_block_size - 1)); 122 par = prz->par_buffer + (start / ecc_block_size) * prz->ecc_size; 123 124 do { 125 if (block + ecc_block_size > buffer_end) 126 size = buffer_end - block; 127 persistent_ram_encode_rs8(prz, block, size, par); 128 block += ecc_block_size; 129 par += ecc_size; 130 } while (block < buffer->data + start + count); 131} 132 133static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz) 134{ 135 struct persistent_ram_buffer *buffer = prz->buffer; 136 137 if (!prz->ecc) 138 return; 139 140 persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer), 141 prz->par_header); 142} 143 144static void persistent_ram_ecc_old(struct persistent_ram_zone *prz) 145{ 146 struct persistent_ram_buffer *buffer = prz->buffer; 147 uint8_t *block; 148 uint8_t *par; 149 150 if (!prz->ecc) 151 return; 152 153 block = buffer->data; 154 par = prz->par_buffer; 155 while (block < buffer->data + buffer_size(prz)) { 156 int numerr; 157 int size = prz->ecc_block_size; 158 if (block + size > buffer->data + prz->buffer_size) 159 size = buffer->data + prz->buffer_size - block; 160 numerr = persistent_ram_decode_rs8(prz, block, size, par); 161 if (numerr > 0) { 162 pr_devel("persistent_ram: error in block %p, %d\n", 163 block, numerr); 164 prz->corrected_bytes += numerr; 165 } else if (numerr < 0) { 166 pr_devel("persistent_ram: uncorrectable error in block %p\n", 167 block); 168 prz->bad_blocks++; 169 } 170 block += prz->ecc_block_size; 171 par += prz->ecc_size; 172 } 173} 174 175static int persistent_ram_init_ecc(struct persistent_ram_zone *prz, 176 size_t buffer_size, struct persistent_ram *ram) 177{ 178 int numerr; 179 struct persistent_ram_buffer *buffer = prz->buffer; 180 int ecc_blocks; 181 182 if (!prz->ecc) 183 return 0; 184 185 prz->ecc_block_size = ram->ecc_block_size ?: 128; 186 prz->ecc_size = ram->ecc_size ?: 16; 187 prz->ecc_symsize = ram->ecc_symsize ?: 8; 188 prz->ecc_poly = ram->ecc_poly ?: 0x11d; 189 190 ecc_blocks = DIV_ROUND_UP(prz->buffer_size - prz->ecc_size, 191 prz->ecc_block_size + prz->ecc_size); 192 prz->buffer_size -= (ecc_blocks + 1) * prz->ecc_size; 193 194 if (prz->buffer_size > buffer_size) { 195 pr_err("persistent_ram: invalid size %zu, non-ecc datasize %zu\n", 196 buffer_size, prz->buffer_size); 197 return -EINVAL; 198 } 199 200 prz->par_buffer = buffer->data + prz->buffer_size; 201 prz->par_header = prz->par_buffer + ecc_blocks * prz->ecc_size; 202 203 /* 204 * first consecutive root is 0 205 * primitive element to generate roots = 1 206 */ 207 prz->rs_decoder = init_rs(prz->ecc_symsize, prz->ecc_poly, 0, 1, 208 prz->ecc_size); 209 if (prz->rs_decoder == NULL) { 210 pr_info("persistent_ram: init_rs failed\n"); 211 return -EINVAL; 212 } 213 214 prz->corrected_bytes = 0; 215 prz->bad_blocks = 0; 216 217 numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer), 218 prz->par_header); 219 if (numerr > 0) { 220 pr_info("persistent_ram: error in header, %d\n", numerr); 221 prz->corrected_bytes += numerr; 222 } else if (numerr < 0) { 223 pr_info("persistent_ram: uncorrectable error in header\n"); 224 prz->bad_blocks++; 225 } 226 227 return 0; 228} 229 230ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz, 231 char *str, size_t len) 232{ 233 ssize_t ret; 234 235 if (prz->corrected_bytes || prz->bad_blocks) 236 ret = snprintf(str, len, "" 237 "\n%d Corrected bytes, %d unrecoverable blocks\n", 238 prz->corrected_bytes, prz->bad_blocks); 239 else 240 ret = snprintf(str, len, "\nNo errors detected\n"); 241 242 return ret; 243} 244 245static void notrace persistent_ram_update(struct persistent_ram_zone *prz, 246 const void *s, unsigned int start, unsigned int count) 247{ 248 struct persistent_ram_buffer *buffer = prz->buffer; 249 memcpy(buffer->data + start, s, count); 250 persistent_ram_update_ecc(prz, start, count); 251} 252 253static void __devinit 254persistent_ram_save_old(struct persistent_ram_zone *prz) 255{ 256 struct persistent_ram_buffer *buffer = prz->buffer; 257 size_t size = buffer_size(prz); 258 size_t start = buffer_start(prz); 259 char *dest; 260 261 persistent_ram_ecc_old(prz); 262 263 dest = kmalloc(size, GFP_KERNEL); 264 if (dest == NULL) { 265 pr_err("persistent_ram: failed to allocate buffer\n"); 266 return; 267 } 268 269 prz->old_log = dest; 270 prz->old_log_size = size; 271 memcpy(prz->old_log, &buffer->data[start], size - start); 272 memcpy(prz->old_log + size - start, &buffer->data[0], start); 273} 274 275int notrace persistent_ram_write(struct persistent_ram_zone *prz, 276 const void *s, unsigned int count) 277{ 278 int rem; 279 int c = count; 280 size_t start; 281 282 if (unlikely(c > prz->buffer_size)) { 283 s += c - prz->buffer_size; 284 c = prz->buffer_size; 285 } 286 287 buffer_size_add(prz, c); 288 289 start = buffer_start_add(prz, c); 290 291 rem = prz->buffer_size - start; 292 if (unlikely(rem < c)) { 293 persistent_ram_update(prz, s, start, rem); 294 s += rem; 295 c -= rem; 296 start = 0; 297 } 298 persistent_ram_update(prz, s, start, c); 299 300 persistent_ram_update_header_ecc(prz); 301 302 return count; 303} 304 305size_t persistent_ram_old_size(struct persistent_ram_zone *prz) 306{ 307 return prz->old_log_size; 308} 309 310void *persistent_ram_old(struct persistent_ram_zone *prz) 311{ 312 return prz->old_log; 313} 314 315void persistent_ram_free_old(struct persistent_ram_zone *prz) 316{ 317 kfree(prz->old_log); 318 prz->old_log = NULL; 319 prz->old_log_size = 0; 320} 321 322static int persistent_ram_buffer_map(phys_addr_t start, phys_addr_t size, 323 struct persistent_ram_zone *prz) 324{ 325 struct page **pages; 326 phys_addr_t page_start; 327 unsigned int page_count; 328 pgprot_t prot; 329 unsigned int i; 330 331 page_start = start - offset_in_page(start); 332 page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE); 333 334 prot = pgprot_noncached(PAGE_KERNEL); 335 336 pages = kmalloc(sizeof(struct page *) * page_count, GFP_KERNEL); 337 if (!pages) { 338 pr_err("%s: Failed to allocate array for %u pages\n", __func__, 339 page_count); 340 return -ENOMEM; 341 } 342 343 for (i = 0; i < page_count; i++) { 344 phys_addr_t addr = page_start + i * PAGE_SIZE; 345 pages[i] = pfn_to_page(addr >> PAGE_SHIFT); 346 } 347 prz->vaddr = vmap(pages, page_count, VM_MAP, prot); 348 kfree(pages); 349 if (!prz->vaddr) { 350 pr_err("%s: Failed to map %u pages\n", __func__, page_count); 351 return -ENOMEM; 352 } 353 354 prz->buffer = prz->vaddr + offset_in_page(start); 355 prz->buffer_size = size - sizeof(struct persistent_ram_buffer); 356 357 return 0; 358} 359 360static int __devinit persistent_ram_buffer_init(const char *name, 361 struct persistent_ram_zone *prz, struct persistent_ram **ramp) 362{ 363 int i; 364 struct persistent_ram *ram; 365 struct persistent_ram_descriptor *desc; 366 phys_addr_t start; 367 368 list_for_each_entry(ram, &persistent_ram_list, node) { 369 start = ram->start; 370 for (i = 0; i < ram->num_descs; i++) { 371 desc = &ram->descs[i]; 372 if (!strcmp(desc->name, name)) { 373 *ramp = ram; 374 return persistent_ram_buffer_map(start, 375 desc->size, prz); 376 } 377 start += desc->size; 378 } 379 } 380 381 return -EINVAL; 382} 383 384static __devinit 385struct persistent_ram_zone *__persistent_ram_init(struct device *dev, bool ecc) 386{ 387 struct persistent_ram *ram; 388 struct persistent_ram_zone *prz; 389 int ret = -ENOMEM; 390 391 prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL); 392 if (!prz) { 393 pr_err("persistent_ram: failed to allocate persistent ram zone\n"); 394 goto err; 395 } 396 397 INIT_LIST_HEAD(&prz->node); 398 399 ret = persistent_ram_buffer_init(dev_name(dev), prz, &ram); 400 if (ret) { 401 pr_err("persistent_ram: failed to initialize buffer\n"); 402 goto err; 403 } 404 405 prz->ecc = ecc; 406 ret = persistent_ram_init_ecc(prz, prz->buffer_size, ram); 407 if (ret) 408 goto err; 409 410 if (prz->buffer->sig == PERSISTENT_RAM_SIG) { 411 if (buffer_size(prz) > prz->buffer_size || 412 buffer_start(prz) > buffer_size(prz)) 413 pr_info("persistent_ram: found existing invalid buffer," 414 " size %zu, start %zu\n", 415 buffer_size(prz), buffer_start(prz)); 416 else { 417 pr_info("persistent_ram: found existing buffer," 418 " size %zu, start %zu\n", 419 buffer_size(prz), buffer_start(prz)); 420 persistent_ram_save_old(prz); 421 } 422 } else { 423 pr_info("persistent_ram: no valid data in buffer" 424 " (sig = 0x%08x)\n", prz->buffer->sig); 425 } 426 427 prz->buffer->sig = PERSISTENT_RAM_SIG; 428 atomic_set(&prz->buffer->start, 0); 429 atomic_set(&prz->buffer->size, 0); 430 431 return prz; 432err: 433 kfree(prz); 434 return ERR_PTR(ret); 435} 436 437struct persistent_ram_zone * __devinit 438persistent_ram_init_ringbuffer(struct device *dev, bool ecc) 439{ 440 return __persistent_ram_init(dev, ecc); 441} 442 443int __init persistent_ram_early_init(struct persistent_ram *ram) 444{ 445 int ret; 446 447 ret = memblock_reserve(ram->start, ram->size); 448 if (ret) { 449 pr_err("Failed to reserve persistent memory from %08lx-%08lx\n", 450 (long)ram->start, (long)(ram->start + ram->size - 1)); 451 return ret; 452 } 453 454 list_add_tail(&ram->node, &persistent_ram_list); 455 456 pr_info("Initialized persistent memory from %08lx-%08lx\n", 457 (long)ram->start, (long)(ram->start + ram->size - 1)); 458 459 return 0; 460} 461