mirror of
				https://github.com/eledio-devices/thirdparty-littlefs.git
				synced 2025-10-31 08:42:40 +01:00 
			
		
		
		
	Initially, littlefs relied entirely on bad-block detection for wear-leveling. Conceptually, at the end of a devices lifespan, all blocks would be worn evenly, even if they weren't worn out at the same time. However, this doesn't work for all devices, rather than causing corruption during writes, wear reduces a devices "sticking power", causing bits to flip over time. This means for many devices, true wear-leveling (dynamic or static) is required. Fortunately, way back at the beginning, littlefs was designed to do full dynamic wear-leveling, only dropping it when making the retrospectively short-sighted realization that bad-block detection is theoretically sufficient. We can enable dynamic wear-leveling with only a few tweaks to littlefs. These can be implemented without breaking backwards compatibility. 1. Evict metadata-pairs after a certain number of writes. Eviction in this case is identical to a relocation to recover from a bad block. We move our data and stick the old block back into our pool of blocks. For knowing when to evict, we already have a revision count for each metadata-pair which gives us enough information. We add the configuration option block_cycles and evict when our revision count is a multiple of this value. 2. Now all blocks participate in COW behaviour. However we don't store the state of our allocator, so every boot cycle we reuse the first blocks on storage. This is very bad on a microcontroller, where we may reboot often. We need a way to spread our usage across the disk. To pull this off, we can simply randomize which block we start our allocator at. But we need a random number generator that is different on each boot. Fortunately we have a great source of entropy, our filesystem. So we seed our block allocator with a simple hash of the CRCs on our metadata-pairs. This can be done for free since we already need to scan the metadata-pairs during mount. What we end up with is a uniform distribution of wear on storage. The wear is not perfect, if a block is used for metadata it gets more wear, and the randomization may not be exact. But we can never actually get perfect wear-leveling, since we're already resigned to dynamic wear-leveling at the file level. With the addition of metadata logging, we end up with a really interesting two-stage wear-leveling algorithm. At the low-level, metadata is statically wear-leveled. At the high-level, blocks are dynamically wear-leveled. --- This specific commit implements the first step, eviction of metadata pairs. Entertwining this into the already complicated compact logic was a bit annoying, however we can combine the logic for superblock expansion with the logic for metadata-pair eviction.
		
			
				
	
	
		
			217 lines
		
	
	
		
			7.3 KiB
		
	
	
	
		
			YAML
		
	
	
	
	
	
			
		
		
	
	
			217 lines
		
	
	
		
			7.3 KiB
		
	
	
	
		
			YAML
		
	
	
	
	
	
| # Environment variables
 | |
| env:
 | |
|   global:
 | |
|     - CFLAGS=-Werror
 | |
| 
 | |
| # Common test script
 | |
| script:
 | |
|   # make sure example can at least compile
 | |
|   - sed -n '/``` c/,/```/{/```/d; p;}' README.md > test.c &&
 | |
|     make all CFLAGS+="
 | |
|         -Duser_provided_block_device_read=NULL
 | |
|         -Duser_provided_block_device_prog=NULL
 | |
|         -Duser_provided_block_device_erase=NULL
 | |
|         -Duser_provided_block_device_sync=NULL
 | |
|         -include stdio.h"
 | |
| 
 | |
|   # run tests
 | |
|   - make test QUIET=1
 | |
| 
 | |
|   # run tests with a few different configurations
 | |
|   - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=1      -DLFS_CACHE_SIZE=4"
 | |
|   - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512    -DLFS_CACHE_SIZE=512 -DLFS_BLOCK_CYCLES=16"
 | |
|   - make test QUIET=1 CFLAGS+="-DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD=2048"
 | |
| 
 | |
|   - make clean test QUIET=1 CFLAGS+="-DLFS_INLINE_MAX=0"
 | |
|   - make clean test QUIET=1 CFLAGS+="-DLFS_NO_INTRINSICS"
 | |
| 
 | |
|   # compile and find the code size with the smallest configuration
 | |
|   - make clean size
 | |
|         OBJ="$(ls lfs*.o | tr '\n' ' ')"
 | |
|         CFLAGS+="-DLFS_NO_ASSERT -DLFS_NO_DEBUG -DLFS_NO_WARN -DLFS_NO_ERROR"
 | |
|         | tee sizes
 | |
| 
 | |
|   # update status if we succeeded, compare with master if possible
 | |
|   - |
 | |
|     if [ "$TRAVIS_TEST_RESULT" -eq 0 ]
 | |
|     then
 | |
|         CURR=$(tail -n1 sizes | awk '{print $1}')
 | |
|         PREV=$(curl -u $GEKY_BOT_STATUSES https://api.github.com/repos/$TRAVIS_REPO_SLUG/status/master \
 | |
|             | jq -re "select(.sha != \"$TRAVIS_COMMIT\")
 | |
|                 | .statuses[] | select(.context == \"$STAGE/$NAME\").description
 | |
|                 | capture(\"code size is (?<size>[0-9]+)\").size" \
 | |
|             || echo 0)
 | |
| 
 | |
|         STATUS="Passed, code size is ${CURR}B"
 | |
|         if [ "$PREV" -ne 0 ]
 | |
|         then
 | |
|             STATUS="$STATUS ($(python -c "print '%+.2f' % (100*($CURR-$PREV)/$PREV.0)")%)"
 | |
|         fi
 | |
|     fi
 | |
| 
 | |
| # CI matrix
 | |
| jobs:
 | |
|   include:
 | |
|     # native testing
 | |
|     - stage: test
 | |
|       env:
 | |
|         - STAGE=test
 | |
|         - NAME=littlefs-x86
 | |
| 
 | |
|     # cross-compile with ARM (thumb mode)
 | |
|     - stage: test
 | |
|       env:
 | |
|         - STAGE=test
 | |
|         - NAME=littlefs-arm
 | |
|         - CC="arm-linux-gnueabi-gcc --static -mthumb"
 | |
|         - EXEC="qemu-arm"
 | |
|       install:
 | |
|         - sudo apt-get install gcc-arm-linux-gnueabi qemu-user
 | |
|         - arm-linux-gnueabi-gcc --version
 | |
|         - qemu-arm -version
 | |
| 
 | |
|     # cross-compile with PowerPC
 | |
|     - stage: test
 | |
|       env:
 | |
|         - STAGE=test
 | |
|         - NAME=littlefs-powerpc
 | |
|         - CC="powerpc-linux-gnu-gcc --static"
 | |
|         - EXEC="qemu-ppc"
 | |
|       install:
 | |
|         - sudo apt-get install gcc-powerpc-linux-gnu qemu-user
 | |
|         - powerpc-linux-gnu-gcc --version
 | |
|         - qemu-ppc -version
 | |
| 
 | |
|     # cross-compile with MIPS
 | |
|     - stage: test
 | |
|       env:
 | |
|         - STAGE=test
 | |
|         - NAME=littlefs-mips
 | |
|         - CC="mips-linux-gnu-gcc --static"
 | |
|         - EXEC="qemu-mips"
 | |
|       install:
 | |
|         - sudo add-apt-repository -y "deb http://archive.ubuntu.com/ubuntu/ xenial main universe"
 | |
|         - sudo apt-get -qq update
 | |
|         - sudo apt-get install gcc-mips-linux-gnu qemu-user
 | |
|         - mips-linux-gnu-gcc --version
 | |
|         - qemu-mips -version
 | |
| 
 | |
|     # self-host with littlefs-fuse for fuzz test
 | |
|     - stage: test
 | |
|       env:
 | |
|         - STAGE=test
 | |
|         - NAME=littlefs-fuse
 | |
|       install:
 | |
|         - sudo apt-get install libfuse-dev
 | |
|         - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v2-alpha
 | |
|         - fusermount -V
 | |
|         - gcc --version
 | |
|       before_script:
 | |
|         # setup disk for littlefs-fuse
 | |
|         - rm -rf littlefs-fuse/littlefs/*
 | |
|         - cp -r $(git ls-tree --name-only HEAD) littlefs-fuse/littlefs
 | |
| 
 | |
|         - mkdir mount
 | |
|         - sudo chmod a+rw /dev/loop0
 | |
|         - dd if=/dev/zero bs=512 count=2048 of=disk
 | |
|         - losetup /dev/loop0 disk
 | |
|       script:
 | |
|         # self-host test
 | |
|         - make -C littlefs-fuse
 | |
| 
 | |
|         - littlefs-fuse/lfs --format /dev/loop0
 | |
|         - littlefs-fuse/lfs /dev/loop0 mount
 | |
| 
 | |
|         - ls mount
 | |
|         - mkdir mount/littlefs
 | |
|         - cp -r $(git ls-tree --name-only HEAD) mount/littlefs
 | |
|         - cd mount/littlefs
 | |
|         - ls
 | |
|         - make -B test_dirs test_files QUIET=1
 | |
| 
 | |
|       # Automatically update releases
 | |
|     - stage: deploy
 | |
|       env:
 | |
|         - STAGE=deploy
 | |
|         - NAME=deploy
 | |
|       script:
 | |
|         # Find version defined in lfs.h
 | |
|         - LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3)
 | |
|         - LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16)))
 | |
|         - LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >>  0)))
 | |
|         # Grab latests patch from repo tags, default to 0
 | |
|         - LFS_VERSION_PATCH=$(curl -f -u "$GEKY_BOT_RELEASES"
 | |
|                 https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs
 | |
|                 | jq 'map(.ref | match(
 | |
|                     "refs/tags/v'"$LFS_VERSION_MAJOR"'\\.'"$LFS_VERSION_MINOR"'\\.(.*)$")
 | |
|                     .captures[].string | tonumber + 1) | max // 0')
 | |
|         # We have our new version
 | |
|         - LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.$LFS_VERSION_PATCH"
 | |
|         - echo "VERSION $LFS_VERSION"
 | |
|         - |
 | |
|           # Check that we're the most recent commit
 | |
|           CURRENT_COMMIT=$(curl -f -u "$GEKY_BOT_RELEASES" \
 | |
|                 https://api.github.com/repos/$TRAVIS_REPO_SLUG/commits/master \
 | |
|                 | jq -re '.sha')
 | |
|           if [ "$TRAVIS_COMMIT" == "$CURRENT_COMMIT" ]
 | |
|           then
 | |
|             # Build release notes
 | |
|             PREV=$(git tag --sort=-v:refname -l "v*" | head -1)
 | |
|             if [ ! -z "$PREV" ]
 | |
|             then
 | |
|                 echo "PREV $PREV"
 | |
|                 CHANGES=$'### Changes\n\n'$( \
 | |
|                     git log --oneline $PREV.. --grep='^Merge' --invert-grep)
 | |
|                 printf "CHANGES\n%s\n\n" "$CHANGES"
 | |
|             fi
 | |
|             # Create the release
 | |
|             curl -f -u "$GEKY_BOT_RELEASES" -X POST \
 | |
|                 https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \
 | |
|                 -d "{
 | |
|                     \"tag_name\": \"$LFS_VERSION\",
 | |
|                     \"target_commitish\": \"$TRAVIS_COMMIT\",
 | |
|                     \"name\": \"${LFS_VERSION%.0}\",
 | |
|                     \"body\": $(jq -sR '.' <<< "$CHANGES")
 | |
|                 }"
 | |
|           fi
 | |
| 
 | |
| # Manage statuses
 | |
| before_install:
 | |
|   - |
 | |
|     curl -u $GEKY_BOT_STATUSES -X POST \
 | |
|         https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
 | |
|         -d "{
 | |
|             \"context\": \"$STAGE/$NAME\",
 | |
|             \"state\": \"pending\",
 | |
|             \"description\": \"${STATUS:-In progress}\",
 | |
|             \"target_url\": \"https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID\"
 | |
|         }"
 | |
| 
 | |
| after_failure:
 | |
|   - |
 | |
|     curl -u $GEKY_BOT_STATUSES -X POST \
 | |
|         https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
 | |
|         -d "{
 | |
|             \"context\": \"$STAGE/$NAME\",
 | |
|             \"state\": \"failure\",
 | |
|             \"description\": \"${STATUS:-Failed}\",
 | |
|             \"target_url\": \"https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID\"
 | |
|         }"
 | |
| 
 | |
| after_success:
 | |
|   - |
 | |
|     curl -u $GEKY_BOT_STATUSES -X POST \
 | |
|         https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
 | |
|         -d "{
 | |
|             \"context\": \"$STAGE/$NAME\",
 | |
|             \"state\": \"success\",
 | |
|             \"description\": \"${STATUS:-Passed}\",
 | |
|             \"target_url\": \"https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID\"
 | |
|         }"
 | |
| 
 | |
| # Job control
 | |
| stages:
 | |
|     - name: test
 | |
|     - name: deploy
 | |
|       if: branch = master AND type = push
 |