1From 41574b628ec4229c24dfe289af7b6978edcca4ed Mon Sep 17 00:00:00 2001 2From: Sean Anderson <sean.anderson@seco.com> 3Date: Thu, 30 Dec 2021 15:19:41 -0500 4Subject: [PATCH] libsparse: Add "hole" mode to sparse_file_read 5 6This adds support for filesystem-level sparse files. These files have 7holes which are not stored in the filesystem and when read are full of 8zeros. While these zeros may be significant in some types of files, 9other types of files may not care about the contents of holes. For 10example, most filesystem creation tools write to all the blocks they 11care about. Those blocks not written to will remain holes, and can be 12safely represented by "don't care" chunks. Using "don't care" chunks 13instead of fill chunks can result in a substantial reduction of the time 14it takes to program a sparse image. 15 16To accomplish this, we extend the existing "sparse" boolean parameter to 17be an enum of mode types. This enum represents the strategy we take when 18reading in a file. For the most part the implementation is 19straightforward. We use lseek to determine where the holes in the file 20are, and then use do_sparse_file_read_normal to create chunks for the 21data section. Note that every file has an implicit hole at its end. 22 23Change-Id: I0cfbf08886fca9a91cb753ec8734c84fcbe52c9f 24Upstream-Status: Backport [f96466b05543b984ef7315d830bab4a409228d35] 25Signed-off-by: Sean Anderson <sean.anderson@seco.com> 26--- 27 libsparse/img2simg.c | 2 +- 28 libsparse/include/sparse/sparse.h | 32 +++++++++++--- 29 libsparse/sparse_read.c | 71 +++++++++++++++++++++++++++++-- 30 3 files changed, 93 insertions(+), 12 deletions(-) 31 32diff --git a/libsparse/img2simg.c b/libsparse/img2simg.c 33index a0db36f45..2e171b613 100644 34--- a/libsparse/img2simg.c 35+++ b/libsparse/img2simg.c 36@@ -96,7 +96,7 @@ int main(int argc, char *argv[]) 37 } 38 39 sparse_file_verbose(s); 40- ret = sparse_file_read(s, in, false, false); 41+ ret = sparse_file_read(s, in, SPARSE_READ_MODE_NORMAL, false); 42 if (ret) { 43 fprintf(stderr, "Failed to read file\n"); 44 exit(-1); 45diff --git a/libsparse/include/sparse/sparse.h b/libsparse/include/sparse/sparse.h 46index 8b757d22a..b68aa21a8 100644 47--- a/libsparse/include/sparse/sparse.h 48+++ b/libsparse/include/sparse/sparse.h 49@@ -196,23 +196,41 @@ int64_t sparse_file_len(struct sparse_file *s, bool sparse, bool crc); 50 int sparse_file_callback(struct sparse_file *s, bool sparse, bool crc, 51 int (*write)(void *priv, const void *data, int len), void *priv); 52 53+/** 54+ * enum sparse_read_mode - The method to use when reading in files 55+ * @SPARSE_READ_MODE_NORMAL: The input is a regular file. Constant chunks of 56+ * data (including holes) will be be converted to 57+ * fill chunks. 58+ * @SPARSE_READ_MODE_SPARSE: The input is an Android sparse file. 59+ * @SPARSE_READ_MODE_HOLE: The input is a regular file. Holes will be converted 60+ * to "don't care" chunks. Other constant chunks will 61+ * be converted to fill chunks. 62+ */ 63+enum sparse_read_mode { 64+ SPARSE_READ_MODE_NORMAL = false, 65+ SPARSE_READ_MODE_SPARSE = true, 66+ SPARSE_READ_MODE_HOLE, 67+}; 68+ 69 /** 70 * sparse_file_read - read a file into a sparse file cookie 71 * 72 * @s - sparse file cookie 73 * @fd - file descriptor to read from 74- * @sparse - read a file in the Android sparse file format 75+ * @mode - mode to use when reading the input file 76 * @crc - verify the crc of a file in the Android sparse file format 77 * 78- * Reads a file into a sparse file cookie. If sparse is true, the file is 79- * assumed to be in the Android sparse file format. If sparse is false, the 80- * file will be sparsed by looking for block aligned chunks of all zeros or 81- * another 32 bit value. If crc is true, the crc of the sparse file will be 82- * verified. 83+ * Reads a file into a sparse file cookie. If @mode is 84+ * %SPARSE_READ_MODE_SPARSE, the file is assumed to be in the Android sparse 85+ * file format. If @mode is %SPARSE_READ_MODE_NORMAL, the file will be sparsed 86+ * by looking for block aligned chunks of all zeros or another 32 bit value. If 87+ * @mode is %SPARSE_READ_MODE_HOLE, the file will be sparsed like 88+ * %SPARSE_READ_MODE_NORMAL, but holes in the file will be converted to "don't 89+ * care" chunks. If crc is true, the crc of the sparse file will be verified. 90 * 91 * Returns 0 on success, negative errno on error. 92 */ 93-int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc); 94+int sparse_file_read(struct sparse_file *s, int fd, enum sparse_read_mode mode, bool crc); 95 96 /** 97 * sparse_file_import - import an existing sparse file 98diff --git a/libsparse/sparse_read.c b/libsparse/sparse_read.c 99index ee4abd86a..81f48cc13 100644 100--- a/libsparse/sparse_read.c 101+++ b/libsparse/sparse_read.c 102@@ -414,16 +414,79 @@ static int sparse_file_read_normal(struct sparse_file* s, int fd) 103 return ret; 104 } 105 106-int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc) 107+#ifdef __linux__ 108+static int sparse_file_read_hole(struct sparse_file* s, int fd) 109 { 110- if (crc && !sparse) { 111+ int ret; 112+ uint32_t* buf = (uint32_t*)malloc(s->block_size); 113+ int64_t end = 0; 114+ int64_t start = 0; 115+ 116+ if (!buf) { 117+ return -ENOMEM; 118+ } 119+ 120+ do { 121+ start = lseek(fd, end, SEEK_DATA); 122+ if (start < 0) { 123+ if (errno == ENXIO) 124+ /* The rest of the file is a hole */ 125+ break; 126+ 127+ error("could not seek to data"); 128+ free(buf); 129+ return -errno; 130+ } else if (start > s->len) { 131+ break; 132+ } 133+ 134+ end = lseek(fd, start, SEEK_HOLE); 135+ if (end < 0) { 136+ error("could not seek to end"); 137+ free(buf); 138+ return -errno; 139+ } 140+ end = min(end, s->len); 141+ 142+ start = ALIGN_DOWN(start, s->block_size); 143+ end = ALIGN(end, s->block_size); 144+ if (lseek(fd, start, SEEK_SET) < 0) { 145+ free(buf); 146+ return -errno; 147+ } 148+ 149+ ret = do_sparse_file_read_normal(s, fd, buf, start, end - start); 150+ if (ret) { 151+ free(buf); 152+ return ret; 153+ } 154+ } while (end < s->len); 155+ 156+ free(buf); 157+ return 0; 158+} 159+#else 160+static int sparse_file_read_hole(struct sparse_file* s __unused, int fd __unused) 161+{ 162+ return -ENOTSUP; 163+} 164+#endif 165+ 166+int sparse_file_read(struct sparse_file *s, int fd, enum sparse_read_mode mode, bool crc) 167+{ 168+ if (crc && mode != SPARSE_READ_MODE_SPARSE) { 169 return -EINVAL; 170 } 171 172- if (sparse) { 173+ switch (mode) { 174+ case SPARSE_READ_MODE_SPARSE: 175 return sparse_file_read_sparse(s, fd, crc); 176- } else { 177+ case SPARSE_READ_MODE_NORMAL: 178 return sparse_file_read_normal(s, fd); 179+ case SPARSE_READ_MODE_HOLE: 180+ return sparse_file_read_hole(s, fd); 181+ default: 182+ return -EINVAL; 183 } 184 } 185 186-- 1872.35.1.1320.gc452695387.dirty 188 189