160 static inline int buffer_datalen(struct pstore_zone *zone)  in buffer_datalen()  argument
162 	return atomic_read(&zone->buffer->datalen);  in buffer_datalen()
165 static inline int buffer_start(struct pstore_zone *zone)  in buffer_start()  argument
167 	return atomic_read(&zone->buffer->start);  in buffer_start()
175 static ssize_t psz_zone_read_buffer(struct pstore_zone *zone, char *buf,  in psz_zone_read_buffer()  argument
178 	if (!buf || !zone || !zone->buffer)  in psz_zone_read_buffer()
180 	if (off > zone->buffer_size)  in psz_zone_read_buffer()
182 	len = min_t(size_t, len, zone->buffer_size - off);  in psz_zone_read_buffer()
183 	memcpy(buf, zone->buffer->data + off, len);  in psz_zone_read_buffer()
187 static int psz_zone_read_oldbuf(struct pstore_zone *zone, char *buf,  in psz_zone_read_oldbuf()  argument
190 	if (!buf || !zone || !zone->oldbuf)  in psz_zone_read_oldbuf()
192 	if (off > zone->buffer_size)  in psz_zone_read_oldbuf()
194 	len = min_t(size_t, len, zone->buffer_size - off);  in psz_zone_read_oldbuf()
195 	memcpy(buf, zone->oldbuf->data + off, len);  in psz_zone_read_oldbuf()
199 static int psz_zone_write(struct pstore_zone *zone,  in psz_zone_write()  argument
208 	if (off > zone->buffer_size)  in psz_zone_write()
211 	wlen = min_t(size_t, len, zone->buffer_size - off);  in psz_zone_write()
213 		memcpy(zone->buffer->data + off, buf, wlen);  in psz_zone_write()
214 		atomic_set(&zone->buffer->datalen, wlen + off);  in psz_zone_write()
231 		wcnt = writeop((const char *)zone->buffer->data + off, wlen,  in psz_zone_write()
232 				zone->off + sizeof(*zone->buffer) + off);  in psz_zone_write()
238 		wcnt = writeop((const char *)zone->buffer, wlen, zone->off);  in psz_zone_write()
243 		wlen = zone->buffer_size + sizeof(*zone->buffer);  in psz_zone_write()
244 		wcnt = writeop((const char *)zone->buffer, wlen, zone->off);  in psz_zone_write()
255 	atomic_set(&zone->dirty, true);  in psz_zone_write()
262 static int psz_flush_dirty_zone(struct pstore_zone *zone)  in psz_flush_dirty_zone()  argument
266 	if (unlikely(!zone))  in psz_flush_dirty_zone()
272 	if (!atomic_xchg(&zone->dirty, false))  in psz_flush_dirty_zone()
275 	ret = psz_zone_write(zone, FLUSH_ALL, NULL, 0, 0);  in psz_flush_dirty_zone()
277 		atomic_set(&zone->dirty, true);  in psz_flush_dirty_zone()
284 	struct pstore_zone *zone;  in psz_flush_dirty_zones()  local
290 		zone = zones[i];  in psz_flush_dirty_zones()
291 		if (!zone)  in psz_flush_dirty_zones()
293 		ret = psz_flush_dirty_zone(zone);  in psz_flush_dirty_zones()
335 	struct pstore_zone *zone = NULL;  in psz_kmsg_recover_data()  local
344 		zone = cxt->kpszs[i];  in psz_kmsg_recover_data()
345 		if (unlikely(!zone))  in psz_kmsg_recover_data()
347 		if (atomic_read(&zone->dirty)) {  in psz_kmsg_recover_data()
352 			ret = psz_move_zone(zone, new);  in psz_kmsg_recover_data()
360 		if (!zone->should_recover)  in psz_kmsg_recover_data()
362 		buf = zone->buffer;  in psz_kmsg_recover_data()
363 		rcnt = info->read((char *)buf, zone->buffer_size + sizeof(*buf),  in psz_kmsg_recover_data()
364 				zone->off);  in psz_kmsg_recover_data()
365 		if (rcnt != zone->buffer_size + sizeof(*buf))  in psz_kmsg_recover_data()
374 	struct pstore_zone *zone;  in psz_kmsg_recover_meta()  local
392 		zone = cxt->kpszs[i];  in psz_kmsg_recover_meta()
393 		if (unlikely(!zone))  in psz_kmsg_recover_meta()
396 		rcnt = info->read((char *)buf, len, zone->off);  in psz_kmsg_recover_meta()
399 					zone->name, i);  in psz_kmsg_recover_meta()
402 			pr_err("read %s with id %lu failed\n", zone->name, i);  in psz_kmsg_recover_meta()
406 		if (buf->sig != zone->buffer->sig) {  in psz_kmsg_recover_meta()
411 		if (zone->buffer_size < atomic_read(&buf->datalen)) {  in psz_kmsg_recover_meta()
413 					zone->name, i, zone->off,  in psz_kmsg_recover_meta()
414 					zone->buffer_size);  in psz_kmsg_recover_meta()
421 					zone->name, i, zone->off,  in psz_kmsg_recover_meta()
422 					zone->buffer_size);  in psz_kmsg_recover_meta()
444 					zone->name, i, zone->off,  in psz_kmsg_recover_meta()
445 					zone->buffer_size,  in psz_kmsg_recover_meta()
451 			zone->should_recover = true;  in psz_kmsg_recover_meta()
453 				zone->name, i, zone->off,  in psz_kmsg_recover_meta()
454 				zone->buffer_size, atomic_read(&buf->datalen));  in psz_kmsg_recover_meta()
481 static int psz_recover_zone(struct psz_context *cxt, struct pstore_zone *zone)  in psz_recover_zone()  argument
489 	if (!zone || zone->oldbuf)  in psz_recover_zone()
494 		psz_flush_dirty_zone(zone);  in psz_recover_zone()
502 	rcnt = info->read((char *)&tmpbuf, len, zone->off);  in psz_recover_zone()
504 		pr_debug("read zone %s failed\n", zone->name);  in psz_recover_zone()
508 	if (tmpbuf.sig != zone->buffer->sig) {  in psz_recover_zone()
509 		pr_debug("no valid data in zone %s\n", zone->name);  in psz_recover_zone()
513 	if (zone->buffer_size < atomic_read(&tmpbuf.datalen) ||  in psz_recover_zone()
514 		zone->buffer_size < atomic_read(&tmpbuf.start)) {  in psz_recover_zone()
516 				zone->name, zone->off, zone->buffer_size);  in psz_recover_zone()
523 				zone->name, zone->off, zone->buffer_size,  in psz_recover_zone()
529 			zone->name, zone->off, zone->buffer_size,  in psz_recover_zone()
541 	off = zone->off + sizeof(*oldbuf);  in psz_recover_zone()
546 		pr_err("read zone %s failed\n", zone->name);  in psz_recover_zone()
554 		pr_err("read zone %s failed\n", zone->name);  in psz_recover_zone()
559 	zone->oldbuf = oldbuf;  in psz_recover_zone()
560 	psz_flush_dirty_zone(zone);  in psz_recover_zone()
573 	struct pstore_zone *zone;  in psz_recover_zones()  local
579 		zone = zones[i];  in psz_recover_zones()
580 		if (unlikely(!zone))  in psz_recover_zones()
582 		ret = psz_recover_zone(cxt, zone);  in psz_recover_zones()
589 	pr_debug("recover %s[%u] failed\n", zone->name, i);  in psz_recover_zones()
643 static inline bool psz_old_ok(struct pstore_zone *zone)  in psz_old_ok()  argument
645 	if (zone && zone->oldbuf && atomic_read(&zone->oldbuf->datalen))  in psz_old_ok()
650 static inline bool psz_ok(struct pstore_zone *zone)  in psz_ok()  argument
652 	if (zone && zone->buffer && buffer_datalen(zone))  in psz_ok()
658 		struct pstore_zone *zone, struct pstore_record *record)  in psz_kmsg_erase()  argument
660 	struct psz_buffer *buffer = zone->buffer;  in psz_kmsg_erase()
665 	if (unlikely(!psz_ok(zone)))  in psz_kmsg_erase()
672 	size = buffer_datalen(zone) + sizeof(*zone->buffer);  in psz_kmsg_erase()
673 	atomic_set(&zone->buffer->datalen, 0);  in psz_kmsg_erase()
675 		return cxt->pstore_zone_info->erase(size, zone->off);  in psz_kmsg_erase()
677 		return psz_zone_write(zone, FLUSH_META, NULL, 0, 0);  in psz_kmsg_erase()
681 		struct pstore_zone *zone)  in psz_record_erase()  argument
683 	if (unlikely(!psz_old_ok(zone)))  in psz_record_erase()
686 	kfree(zone->oldbuf);  in psz_record_erase()
687 	zone->oldbuf = NULL;  in psz_record_erase()
693 	if (!buffer_datalen(zone))  in psz_record_erase()
694 		return psz_zone_write(zone, FLUSH_META, NULL, 0, 0);  in psz_record_erase()
695 	psz_flush_dirty_zone(zone);  in psz_record_erase()
720 static void psz_write_kmsg_hdr(struct pstore_zone *zone,  in psz_write_kmsg_hdr()  argument
724 	struct psz_buffer *buffer = zone->buffer;  in psz_write_kmsg_hdr()
749 	struct pstore_zone *zone;  in psz_kmsg_write_record()  local
757 		zone = cxt->kpszs[zonenum];  in psz_kmsg_write_record()
758 		if (unlikely(!zone))  in psz_kmsg_write_record()
762 		len = zone->buffer_size + sizeof(*zone->buffer);  in psz_kmsg_write_record()
763 		zone->oldbuf = zone->buffer;  in psz_kmsg_write_record()
764 		zone->buffer = kzalloc(len, GFP_ATOMIC);  in psz_kmsg_write_record()
765 		if (!zone->buffer) {  in psz_kmsg_write_record()
766 			zone->buffer = zone->oldbuf;  in psz_kmsg_write_record()
769 		zone->buffer->sig = zone->oldbuf->sig;  in psz_kmsg_write_record()
771 		pr_debug("write %s to zone id %d\n", zone->name, zonenum);  in psz_kmsg_write_record()
772 		psz_write_kmsg_hdr(zone, record);  in psz_kmsg_write_record()
774 		size = min_t(size_t, record->size, zone->buffer_size - hlen);  in psz_kmsg_write_record()
775 		ret = psz_zone_write(zone, FLUSH_ALL, record->buf, size, hlen);  in psz_kmsg_write_record()
780 			kfree(zone->oldbuf);  in psz_kmsg_write_record()
781 			zone->oldbuf = NULL;  in psz_kmsg_write_record()
787 		kfree(zone->buffer);  in psz_kmsg_write_record()
788 		zone->buffer = zone->oldbuf;  in psz_kmsg_write_record()
789 		zone->oldbuf = NULL;  in psz_kmsg_write_record()
823 static int notrace psz_record_write(struct pstore_zone *zone,  in psz_record_write()  argument
831 	if (!zone || !record)  in psz_record_write()
834 	if (atomic_read(&zone->buffer->datalen) >= zone->buffer_size)  in psz_record_write()
839 	if (unlikely(cnt > zone->buffer_size)) {  in psz_record_write()
840 		buf += cnt - zone->buffer_size;  in psz_record_write()
841 		cnt = zone->buffer_size;  in psz_record_write()
844 	start = buffer_start(zone);  in psz_record_write()
845 	rem = zone->buffer_size - start;  in psz_record_write()
847 		psz_zone_write(zone, FLUSH_PART, buf, rem, start);  in psz_record_write()
854 	atomic_set(&zone->buffer->start, cnt + start);  in psz_record_write()
855 	psz_zone_write(zone, FLUSH_PART, buf, cnt, start);  in psz_record_write()
866 		atomic_set(&zone->buffer->datalen, zone->buffer_size);  in psz_record_write()
867 		psz_zone_write(zone, FLUSH_META, NULL, 0, 0);  in psz_record_write()
908 	struct pstore_zone *zone = NULL;  in psz_read_next_zone()  local
911 		zone = cxt->kpszs[cxt->kmsg_read_cnt++];  in psz_read_next_zone()
912 		if (psz_ok(zone))  in psz_read_next_zone()
913 			return zone;  in psz_read_next_zone()
926 		zone = cxt->ppsz;  in psz_read_next_zone()
927 		if (psz_old_ok(zone))  in psz_read_next_zone()
928 			return zone;  in psz_read_next_zone()
933 		zone = cxt->cpsz;  in psz_read_next_zone()
934 		if (psz_old_ok(zone))  in psz_read_next_zone()
935 			return zone;  in psz_read_next_zone()
941 static int psz_kmsg_read_hdr(struct pstore_zone *zone,  in psz_kmsg_read_hdr()  argument
944 	struct psz_buffer *buffer = zone->buffer;  in psz_kmsg_read_hdr()
958 static ssize_t psz_kmsg_read(struct pstore_zone *zone,  in psz_kmsg_read()  argument
963 	size = buffer_datalen(zone);  in psz_kmsg_read()
965 	if (psz_kmsg_read_hdr(zone, record)) {  in psz_kmsg_read()
966 		atomic_set(&zone->buffer->datalen, 0);  in psz_kmsg_read()
967 		atomic_set(&zone->dirty, 0);  in psz_kmsg_read()
990 	size = psz_zone_read_buffer(zone, record->buf + hlen, size,  in psz_kmsg_read()
1001 static ssize_t psz_ftrace_read(struct pstore_zone *zone,  in psz_ftrace_read()  argument
1008 	if (!zone || !record)  in psz_ftrace_read()
1011 	if (!psz_old_ok(zone))  in psz_ftrace_read()
1014 	buf = (struct psz_buffer *)zone->oldbuf;  in psz_ftrace_read()
1032 static ssize_t psz_record_read(struct pstore_zone *zone,  in psz_record_read()  argument
1038 	if (!zone || !record)  in psz_record_read()
1041 	buf = (struct psz_buffer *)zone->oldbuf;  in psz_record_read()
1050 	if (unlikely(psz_zone_read_oldbuf(zone, record->buf, len, 0))) {  in psz_record_read()
1061 	ssize_t (*readop)(struct pstore_zone *zone,  in psz_pstore_read()
1063 	struct pstore_zone *zone;  in psz_pstore_read()  local
1072 	zone = psz_read_next_zone(cxt);  in psz_pstore_read()
1073 	if (!zone)  in psz_pstore_read()
1076 	record->type = zone->type;  in psz_pstore_read()
1093 	ret = readop(zone, record);  in psz_pstore_read()
1115 	struct pstore_zone *zone = *pszone;  in psz_free_zone()  local
1117 	if (!zone)  in psz_free_zone()
1120 	kfree(zone->buffer);  in psz_free_zone()
1121 	kfree(zone);  in psz_free_zone()
1156 	struct pstore_zone *zone;  in psz_init_zone()  local
1168 	zone = kzalloc(sizeof(struct pstore_zone), GFP_KERNEL);  in psz_init_zone()
1169 	if (!zone)  in psz_init_zone()
1172 	zone->buffer = kmalloc(size, GFP_KERNEL);  in psz_init_zone()
1173 	if (!zone->buffer) {  in psz_init_zone()
1174 		kfree(zone);  in psz_init_zone()
1177 	memset(zone->buffer, 0xFF, size);  in psz_init_zone()
1178 	zone->off = *off;  in psz_init_zone()
1179 	zone->name = name;  in psz_init_zone()
1180 	zone->type = type;  in psz_init_zone()
1181 	zone->buffer_size = size - sizeof(struct psz_buffer);  in psz_init_zone()
1182 	zone->buffer->sig = type ^ PSZ_SIG;  in psz_init_zone()
1183 	zone->oldbuf = NULL;  in psz_init_zone()
1184 	atomic_set(&zone->dirty, 0);  in psz_init_zone()
1185 	atomic_set(&zone->buffer->datalen, 0);  in psz_init_zone()
1186 	atomic_set(&zone->buffer->start, 0);  in psz_init_zone()
1190 	pr_debug("pszone %s: off 0x%llx, %zu header, %zu data\n", zone->name,  in psz_init_zone()
1191 			zone->off, sizeof(*zone->buffer), zone->buffer_size);  in psz_init_zone()
1192 	return zone;  in psz_init_zone()
1200 	struct pstore_zone **zones, *zone;  in psz_init_zones()  local
1223 		zone = psz_init_zone(type, off, record_size);  in psz_init_zones()
1224 		if (!zone || IS_ERR(zone)) {  in psz_init_zones()
1227 			return (void *)zone;  in psz_init_zones()
1229 		zones[i] = zone;  in psz_init_zones()