Added BasicJsonDocument::shrinkToFit()

CHANGELOG.md

@@ -1,6 +1,11 @@
 ArduinoJson: change log
 =======================
 
+HEAD
+----
+
+* Added `BasicJsonDocument::shrinkToFit()`
+
 v6.13.0 (2019-11-01)
 -------
 

extras/tests/JsonDocument/CMakeLists.txt

@@ -12,6 +12,7 @@ add_executable(JsonDocumentTests
 	isNull.cpp
 	nesting.cpp
 	remove.cpp
+	shrinkToFit.cpp
 	size.cpp
 	StaticJsonDocument.cpp
 	subscript.cpp

extras/tests/JsonDocument/shrinkToFit.cpp

@@ -0,0 +1,151 @@
+// ArduinoJson - arduinojson.org
+// Copyright Benoit Blanchon 2014-2019
+// MIT License
+
+#include <ArduinoJson.h>
+#include <catch.hpp>
+
+#include <stdlib.h>  // malloc, free
+#include <string>
+
+using ARDUINOJSON_NAMESPACE::addPadding;
+
+class ArmoredAllocator {
+ public:
+  ArmoredAllocator() : _ptr(0), _size(0) {}
+
+  void* allocate(size_t size) {
+    _ptr = malloc(size);
+    _size = size;
+    return _ptr;
+  }
+
+  void deallocate(void* ptr) {
+    REQUIRE(ptr == _ptr);
+    free(ptr);
+    _ptr = 0;
+    _size = 0;
+  }
+
+  void* reallocate(void* ptr, size_t new_size) {
+    REQUIRE(ptr == _ptr);
+    // don't call realloc, instead alloc a new buffer and erase the old one
+    // this way we make sure we support relocation
+    void* new_ptr = malloc(new_size);
+    memcpy(new_ptr, _ptr, std::min(new_size, _size));
+    memset(_ptr, '#', _size);  // erase
+    free(_ptr);
+    _ptr = new_ptr;
+    return new_ptr;
+  }
+
+ private:
+  void* _ptr;
+  size_t _size;
+};
+
+typedef BasicJsonDocument<ArmoredAllocator> ShrinkToFitTestDocument;
+
+void testShrinkToFit(ShrinkToFitTestDocument& doc, std::string expected_json,
+                     size_t expected_size) {
+  doc.shrinkToFit();
+
+  REQUIRE(doc.capacity() == expected_size);
+  REQUIRE(doc.memoryUsage() == expected_size);
+
+  std::string json;
+  serializeJson(doc, json);
+  REQUIRE(json == expected_json);
+}
+
+TEST_CASE("BasicJsonDocument::shrinkToFit()") {
+  ShrinkToFitTestDocument doc(4096);
+
+  SECTION("null") {
+    testShrinkToFit(doc, "null", 0);
+  }
+
+  SECTION("empty object") {
+    deserializeJson(doc, "{}");
+    testShrinkToFit(doc, "{}", JSON_OBJECT_SIZE(0));
+  }
+
+  SECTION("empty array") {
+    deserializeJson(doc, "[]");
+    testShrinkToFit(doc, "[]", JSON_ARRAY_SIZE(0));
+  }
+
+  SECTION("linked string") {
+    doc.set("hello");
+    testShrinkToFit(doc, "\"hello\"", 0);
+  }
+
+  SECTION("owned string") {
+    doc.set(std::string("abcdefg"));
+    testShrinkToFit(doc, "\"abcdefg\"", 8);
+  }
+
+  SECTION("linked raw") {
+    doc.set(serialized("[{},123]"));
+    testShrinkToFit(doc, "[{},123]", 0);
+  }
+
+  SECTION("owned raw") {
+    doc.set(serialized(std::string("[{},123]")));
+    testShrinkToFit(doc, "[{},123]", 8);
+  }
+
+  SECTION("linked key") {
+    doc["key"] = 42;
+    testShrinkToFit(doc, "{\"key\":42}", JSON_OBJECT_SIZE(1));
+  }
+
+  SECTION("owned key") {
+    doc[std::string("abcdefg")] = 42;
+    testShrinkToFit(doc, "{\"abcdefg\":42}", JSON_OBJECT_SIZE(1) + 8);
+  }
+
+  SECTION("linked string in array") {
+    doc.add("hello");
+    testShrinkToFit(doc, "[\"hello\"]", JSON_ARRAY_SIZE(1));
+  }
+
+  SECTION("owned string in array") {
+    doc.add(std::string("abcdefg"));
+    testShrinkToFit(doc, "[\"abcdefg\"]", JSON_ARRAY_SIZE(1) + 8);
+  }
+
+  SECTION("linked string in object") {
+    doc["key"] = "hello";
+    testShrinkToFit(doc, "{\"key\":\"hello\"}", JSON_OBJECT_SIZE(1));
+  }
+
+  SECTION("owned string in object") {
+    doc["key"] = std::string("abcdefg");
+    testShrinkToFit(doc, "{\"key\":\"abcdefg\"}", JSON_ARRAY_SIZE(1) + 8);
+  }
+
+  SECTION("unaligned") {
+    doc.add(std::string("?"));  // two bytes in the string pool
+    REQUIRE(doc.memoryUsage() == JSON_OBJECT_SIZE(1) + 2);
+
+    doc.shrinkToFit();
+
+    // the new capacity should be padded to align the pointers
+    REQUIRE(doc.capacity() == JSON_OBJECT_SIZE(1) + sizeof(void*));
+    REQUIRE(doc.memoryUsage() == JSON_OBJECT_SIZE(1) + 2);
+    REQUIRE(doc[0] == "?");
+  }
+}
+
+TEST_CASE("DynamicJsonDocument::shrinkToFit()") {
+  DynamicJsonDocument doc(4096);
+
+  deserializeJson(doc, "{\"hello\":[\"world\"]");
+
+  doc.shrinkToFit();
+
+  std::string json;
+  serializeJson(doc, json);
+  REQUIRE(json == "{\"hello\":[\"world\"]}");
+}

src/ArduinoJson/Collection/CollectionData.hpp

@@ -5,6 +5,7 @@
 #pragma once
 
 #include <ArduinoJson/Namespace.hpp>
+#include <ArduinoJson/Polyfills/assert.hpp>
 
 #include <stddef.h>  // size_t
 
@@ -63,6 +64,8 @@ class CollectionData {
   size_t nesting() const;
   size_t size() const;
 
+  void movePointers(ptrdiff_t stringDistance, ptrdiff_t variantDistance);
+
  private:
   VariantSlot *getSlot(size_t index) const;
 

src/ArduinoJson/Collection/CollectionImpl.hpp

@@ -160,4 +160,20 @@ inline size_t CollectionData::size() const {
   return slotSize(_head);
 }
 
+template <typename T>
+inline void movePointer(T*& p, ptrdiff_t offset) {
+  if (!p) return;
+  p = reinterpret_cast<T*>(
+      reinterpret_cast<void*>(reinterpret_cast<char*>(p) + offset));
+  ARDUINOJSON_ASSERT(isAligned(p));
+}
+
+inline void CollectionData::movePointers(ptrdiff_t stringDistance,
+                                         ptrdiff_t variantDistance) {
+  movePointer(_head, variantDistance);
+  movePointer(_tail, variantDistance);
+  for (VariantSlot* slot = _head; slot; slot = slot->next())
+    slot->movePointers(stringDistance, variantDistance);
+}
+
 }  // namespace ARDUINOJSON_NAMESPACE

src/ArduinoJson/Document/BasicJsonDocument.hpp

@@ -8,6 +8,8 @@
 
 namespace ARDUINOJSON_NAMESPACE {
 
+// Helper to implement the "base-from-member" idiom
+// (we need to store the allocator before constructing JsonDocument)
 template <typename TAllocator>
 class AllocatorOwner {
  protected:
@@ -15,12 +17,16 @@ class AllocatorOwner {
   AllocatorOwner(const AllocatorOwner& src) : _allocator(src._allocator) {}
   AllocatorOwner(TAllocator allocator) : _allocator(allocator) {}
 
-  void* allocate(size_t n) {
-    return _allocator.allocate(n);
+  void* allocate(size_t size) {
+    return _allocator.allocate(size);
   }
 
-  void deallocate(void* p) {
-    _allocator.deallocate(p);
+  void deallocate(void* ptr) {
+    _allocator.deallocate(ptr);
+  }
+
+  void* reallocate(void* ptr, size_t new_size) {
+    return _allocator.reallocate(ptr, new_size);
   }
 
  private:
@@ -69,6 +75,20 @@ class BasicJsonDocument : AllocatorOwner<TAllocator>, public JsonDocument {
     return *this;
   }
 
+  void shrinkToFit() {
+    ptrdiff_t bytes_reclaimed = _pool.squash();
+    if (bytes_reclaimed == 0) return;
+
+    void* old_ptr = _pool.buffer();
+    void* new_ptr = this->reallocate(old_ptr, _pool.capacity());
+
+    ptrdiff_t ptr_offset =
+        static_cast<char*>(new_ptr) - static_cast<char*>(old_ptr);
+
+    _pool.movePointers(ptr_offset);
+    _data.movePointers(ptr_offset, ptr_offset - bytes_reclaimed);
+  }
+
  private:
   MemoryPool allocPool(size_t requiredSize) {
     size_t capa = addPadding(requiredSize);

src/ArduinoJson/Document/DynamicJsonDocument.hpp

@@ -11,12 +11,16 @@
 namespace ARDUINOJSON_NAMESPACE {
 
 struct DefaultAllocator {
-  void* allocate(size_t n) {
-    return malloc(n);
+  void* allocate(size_t size) {
+    return malloc(size);
   }
 
-  void deallocate(void* p) {
-    free(p);
+  void deallocate(void* ptr) {
+    free(ptr);
+  }
+
+  void* reallocate(void* ptr, size_t new_size) {
+    return realloc(ptr, new_size);
   }
 };
 

src/ArduinoJson/Document/JsonDocument.hpp

@@ -301,7 +301,6 @@ class JsonDocument : public Visitable {
     _pool = pool;
   }
 
- private:
   VariantRef getVariant() {
     return VariantRef(&_pool, &_data);
   }

src/ArduinoJson/Memory/Alignment.hpp

@@ -21,6 +21,12 @@ inline size_t addPadding(size_t bytes) {
   return (bytes + mask) & ~mask;
 }
 
+template <typename T>
+inline T *addPadding(T *p) {
+  size_t address = addPadding(reinterpret_cast<size_t>(p));
+  return reinterpret_cast<T *>(address);
+}
+
 template <size_t bytes>
 struct AddPadding {
   static const size_t mask = sizeof(void *) - 1;

src/ArduinoJson/Memory/MemoryPool.hpp

@@ -10,13 +10,15 @@
 #include <ArduinoJson/Polyfills/mpl/max.hpp>
 #include <ArduinoJson/Variant/VariantSlot.hpp>
 
+#include <string.h>  // memmove
+
 namespace ARDUINOJSON_NAMESPACE {
 
-// _begin                                _end
-// v                                        v
-// +-------------+--------------+-----------+
-// | strings...  |   (free)     |  ...slots |
-// +-------------+--------------+-----------+
+// _begin                                   _end
+// v                                           v
+// +-------------+--------------+--------------+
+// | strings...  |   (free)     |  ...variants |
+// +-------------+--------------+--------------+
 //               ^              ^
 //             _left          _right
 
@@ -101,6 +103,39 @@ class MemoryPool {
     return p;
   }
 
+  // Squash the free space between strings and variants
+  //
+  // _begin                    _end
+  // v                            v
+  // +-------------+--------------+
+  // | strings...  |  ...variants |
+  // +-------------+--------------+
+  //               ^
+  //          _left _right
+  //
+  // This funcion is called before a realloc.
+  ptrdiff_t squash() {
+    char* new_right = addPadding(_left);
+    if (new_right >= _right) return 0;
+
+    size_t right_size = static_cast<size_t>(_end - _right);
+    memmove(new_right, _right, right_size);
+
+    ptrdiff_t bytes_reclaimed = _right - new_right;
+    _right = new_right;
+    _end = new_right + right_size;
+    return bytes_reclaimed;
+  }
+
+  // Move all pointers together
+  // This funcion is called after a realloc.
+  void movePointers(ptrdiff_t offset) {
+    _begin += offset;
+    _left += offset;
+    _right += offset;
+    _end += offset;
+  }
+
  private:
   StringSlot* allocStringSlot() {
     return allocRight<StringSlot>();

src/ArduinoJson/Variant/VariantContent.hpp

@@ -16,14 +16,14 @@ namespace ARDUINOJSON_NAMESPACE {
 enum {
   VALUE_MASK = 0x7F,
 
-  OWNERSHIP_BIT = 0x01,
+  VALUE_IS_OWNED = 0x01,
   VALUE_IS_NULL = 0,
   VALUE_IS_LINKED_RAW = 0x02,
   VALUE_IS_OWNED_RAW = 0x03,
   VALUE_IS_LINKED_STRING = 0x04,
   VALUE_IS_OWNED_STRING = 0x05,
 
-  // CAUTION: no OWNERSHIP_BIT below
+  // CAUTION: no VALUE_IS_OWNED below
   VALUE_IS_BOOLEAN = 0x06,
   VALUE_IS_POSITIVE_INTEGER = 0x08,
   VALUE_IS_NEGATIVE_INTEGER = 0x0A,

src/ArduinoJson/Variant/VariantData.hpp

@@ -103,7 +103,7 @@ class VariantData {
 
   bool equals(const VariantData &other) const {
     // Check that variant have the same type, but ignore string ownership
-    if ((type() | OWNERSHIP_BIT) != (other.type() | OWNERSHIP_BIT))
+    if ((type() | VALUE_IS_OWNED) != (other.type() | VALUE_IS_OWNED))
       return false;
 
     switch (type()) {
@@ -352,6 +352,12 @@ class VariantData {
     return _content.asCollection.add(key, pool);
   }
 
+  void movePointers(ptrdiff_t stringDistance, ptrdiff_t variantDistance) {
+    if (_flags & VALUE_IS_OWNED) _content.asString += stringDistance;
+    if (_flags & COLLECTION_MASK)
+      _content.asCollection.movePointers(stringDistance, variantDistance);
+  }
+
  private:
   uint8_t type() const {
     return _flags & VALUE_MASK;

src/ArduinoJson/Variant/VariantSlot.hpp

@@ -14,8 +14,6 @@ namespace ARDUINOJSON_NAMESPACE {
 
 typedef conditional<sizeof(void*) <= 2, int8_t, int16_t>::type VariantSlotDiff;
 
-class VairantData;
-
 class VariantSlot {
   // CAUTION: same layout as VariantData
   // we cannot use composition because it adds padding
@@ -93,6 +91,13 @@ class VariantSlot {
     _flags = 0;
     _key = 0;
   }
+
+  void movePointers(ptrdiff_t stringDistance, ptrdiff_t variantDistance) {
+    if (_flags & KEY_IS_OWNED) _key += stringDistance;
+    if (_flags & VALUE_IS_OWNED) _content.asString += stringDistance;
+    if (_flags & COLLECTION_MASK)
+      _content.asCollection.movePointers(stringDistance, variantDistance);
+  }
 };
 
 }  // namespace ARDUINOJSON_NAMESPACE