Added string deduplication (closes #1303)

extras/tests/Helpers/WString.h

@@ -9,6 +9,9 @@
 // Reproduces Arduino's String class
 class String {
  public:
+  String() {}
+  explicit String(const char* s) : _str(s) {}
+
   String& operator+=(const char* rhs) {
     _str += rhs;
     return *this;

extras/tests/JsonDeserializer/filter.cpp

@@ -239,7 +239,7 @@ TEST_CASE("Filtering") {
       10,
       DeserializationError::Ok,
       "[{\"example\":1},{\"example\":3}]",
-      JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 16
+      JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 8
     },
     {
       "[',2,3]",

extras/tests/JsonDeserializer/string.cpp

@@ -74,16 +74,16 @@ TEST_CASE("Invalid JSON string") {
   }
 }
 
-TEST_CASE("Not enough room to duplicate the string") {
-  DynamicJsonDocument doc(JSON_OBJECT_SIZE(0));
+TEST_CASE("Not enough room to save the key") {
+  DynamicJsonDocument doc(JSON_OBJECT_SIZE(1) + 8);
 
   SECTION("Quoted string") {
-    REQUIRE(deserializeJson(doc, "{\"example\":1}") ==
+    REQUIRE(deserializeJson(doc, "{\"accuracy\":1}") ==
             DeserializationError::NoMemory);
   }
 
   SECTION("Non-quoted string") {
-    REQUIRE(deserializeJson(doc, "{example:1}") ==
+    REQUIRE(deserializeJson(doc, "{accuracy:1}") ==
             DeserializationError::NoMemory);
   }
 }

extras/tests/JsonDocument/StaticJsonDocument.cpp

@@ -212,7 +212,7 @@ TEST_CASE("StaticJsonDocument") {
 
   SECTION("garbageCollect()") {
     StaticJsonDocument<256> doc;
-    doc[std::string("example")] = std::string("example");
+    doc[std::string("example")] = std::string("jukebox");
     doc.remove("example");
     REQUIRE(doc.memoryUsage() == JSON_OBJECT_SIZE(1) + 16);
 

extras/tests/MemoryPool/CMakeLists.txt

@@ -4,8 +4,8 @@
 
 add_executable(MemoryPoolTests 
 	allocVariant.cpp
-	allocString.cpp
 	clear.cpp
+	saveString.cpp
 	size.cpp
 	StringCopier.cpp
 )

extras/tests/MemoryPool/StringCopier.cpp

@@ -38,8 +38,47 @@ TEST_CASE("StringCopier") {
 
     str.startString(&pool);
     str.append('h');
-    str.commit(&pool);
+    str.save(&pool);
 
     REQUIRE(1 == pool.size());
   }
 }
+
+static const char* addStringToPool(MemoryPool* pool, const char* s) {
+  StringCopier str;
+  str.startString(pool);
+  str.append(s);
+  str.append('\0');
+  return str.save(pool);
+}
+
+TEST_CASE("StringCopier::save() deduplicates strings") {
+  char buffer[4096];
+  MemoryPool pool(buffer, 4096);
+
+  SECTION("Basic") {
+    const char* s1 = addStringToPool(&pool, "hello");
+    const char* s2 = addStringToPool(&pool, "world");
+    const char* s3 = addStringToPool(&pool, "hello");
+
+    REQUIRE(s1 == s3);
+    REQUIRE(s2 != s3);
+    REQUIRE(pool.size() == 12);
+  }
+
+  SECTION("Requires terminator") {
+    const char* s1 = addStringToPool(&pool, "hello world");
+    const char* s2 = addStringToPool(&pool, "hello");
+
+    REQUIRE(s2 != s1);
+    REQUIRE(pool.size() == 12 + 6);
+  }
+
+  SECTION("Don't overrun") {
+    const char* s1 = addStringToPool(&pool, "hello world");
+    const char* s2 = addStringToPool(&pool, "wor");
+
+    REQUIRE(s2 != s1);
+    REQUIRE(pool.size() == 12 + 4);
+  }
+}

extras/tests/MemoryPool/allocString.cpp

@@ -1,67 +0,0 @@
-// ArduinoJson - arduinojson.org
-// Copyright Benoit Blanchon 2014-2020
-// MIT License
-
-#include <ArduinoJson/Memory/MemoryPool.hpp>
-#include <catch.hpp>
-
-using namespace ARDUINOJSON_NAMESPACE;
-
-TEST_CASE("MemoryPool::allocFrozenString()") {
-  const size_t poolCapacity = 64;
-  const size_t longestString = poolCapacity;
-  char buffer[poolCapacity];
-  MemoryPool pool(buffer, poolCapacity);
-
-  SECTION("Returns different addresses") {
-    char *a = pool.allocFrozenString(1);
-    char *b = pool.allocFrozenString(1);
-    REQUIRE(a != b);
-  }
-
-  SECTION("Returns NULL when full") {
-    void *p1 = pool.allocFrozenString(longestString);
-    REQUIRE(p1 != 0);
-
-    void *p2 = pool.allocFrozenString(1);
-    REQUIRE(p2 == 0);
-  }
-
-  SECTION("Returns NULL when pool is too small") {
-    void *p = pool.allocFrozenString(longestString + 1);
-    REQUIRE(0 == p);
-  }
-
-  SECTION("Returns NULL when buffer is NULL") {
-    MemoryPool pool2(0, poolCapacity);
-    REQUIRE(0 == pool2.allocFrozenString(2));
-  }
-
-  SECTION("Returns NULL when capacity is 0") {
-    MemoryPool pool2(buffer, 0);
-    REQUIRE(0 == pool2.allocFrozenString(2));
-  }
-
-  SECTION("Returns same address after clear()") {
-    void *a = pool.allocFrozenString(1);
-    pool.clear();
-    void *b = pool.allocFrozenString(1);
-
-    REQUIRE(a == b);
-  }
-
-  SECTION("Can use full capacity when fresh") {
-    void *a = pool.allocFrozenString(longestString);
-
-    REQUIRE(a != 0);
-  }
-
-  SECTION("Can use full capacity after clear") {
-    pool.allocFrozenString(longestString);
-    pool.clear();
-
-    void *a = pool.allocFrozenString(longestString);
-
-    REQUIRE(a != 0);
-  }
-}

extras/tests/MemoryPool/clear.cpp

@@ -3,6 +3,7 @@
 // MIT License
 
 #include <ArduinoJson/Memory/MemoryPool.hpp>
+#include <ArduinoJson/Strings/StringAdapters.hpp>
 #include <catch.hpp>
 
 using namespace ARDUINOJSON_NAMESPACE;
@@ -21,8 +22,8 @@ TEST_CASE("MemoryPool::clear()") {
   }
 
   SECTION("Discards allocated strings") {
-    pool.allocFrozenString(10);
-    REQUIRE(pool.size() > 0);
+    pool.saveString(adaptString(const_cast<char *>("123456789")));
+    REQUIRE(pool.size() == 10);
 
     pool.clear();
 

extras/tests/MemoryPool/saveString.cpp

@@ -0,0 +1,81 @@
+// ArduinoJson - arduinojson.org
+// Copyright Benoit Blanchon 2014-2020
+// MIT License
+
+#include <ArduinoJson/Memory/MemoryPool.hpp>
+#include <ArduinoJson/Strings/StringAdapters.hpp>
+#include <catch.hpp>
+
+using namespace ARDUINOJSON_NAMESPACE;
+
+static const char *saveString(MemoryPool &pool, const char *s) {
+  return pool.saveString(adaptString(const_cast<char *>(s)));
+}
+
+TEST_CASE("MemoryPool::saveString()") {
+  char buffer[32];
+  MemoryPool pool(buffer, 32);
+
+  SECTION("Duplicates different strings") {
+    const char *a = saveString(pool, "hello");
+    const char *b = saveString(pool, "world");
+    REQUIRE(a != b);
+    REQUIRE(pool.size() == 6 + 6);
+  }
+
+  SECTION("Deduplicates identical strings") {
+    const char *a = saveString(pool, "hello");
+    const char *b = saveString(pool, "hello");
+    REQUIRE(a == b);
+    REQUIRE(pool.size() == 6);
+  }
+
+  SECTION("Returns NULL when full") {
+    REQUIRE(pool.capacity() == 32);
+
+    const void *p1 = saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+    REQUIRE(p1 != 0);
+    REQUIRE(pool.size() == 32);
+
+    const void *p2 = saveString(pool, "b");
+    REQUIRE(p2 == 0);
+  }
+
+  SECTION("Returns NULL when pool is too small") {
+    const void *p = saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+    REQUIRE(0 == p);
+  }
+
+  SECTION("Returns NULL when buffer is NULL") {
+    MemoryPool pool2(0, 32);
+    REQUIRE(0 == saveString(pool2, "a"));
+  }
+
+  SECTION("Returns NULL when capacity is 0") {
+    MemoryPool pool2(buffer, 0);
+    REQUIRE(0 == saveString(pool2, "a"));
+  }
+
+  SECTION("Returns same address after clear()") {
+    const void *a = saveString(pool, "hello");
+    pool.clear();
+    const void *b = saveString(pool, "world");
+
+    REQUIRE(a == b);
+  }
+
+  SECTION("Can use full capacity when fresh") {
+    const void *a = saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+
+    REQUIRE(a != 0);
+  }
+
+  SECTION("Can use full capacity after clear") {
+    saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+    pool.clear();
+
+    const void *a = saveString(pool, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb");
+
+    REQUIRE(a != 0);
+  }
+}

extras/tests/MemoryPool/size.cpp

@@ -22,24 +22,6 @@ TEST_CASE("MemoryPool::size()") {
     REQUIRE(0 == pool.size());
   }
 
-  SECTION("Decreases after freezeString()") {
-    StringSlot a = pool.allocExpandableString();
-    pool.freezeString(a, 1);
-    REQUIRE(pool.size() == 1);
-
-    StringSlot b = pool.allocExpandableString();
-    pool.freezeString(b, 1);
-    REQUIRE(pool.size() == 2);
-  }
-
-  SECTION("Increases after allocFrozenString()") {
-    pool.allocFrozenString(1);
-    REQUIRE(pool.size() == 1);
-
-    pool.allocFrozenString(2);
-    REQUIRE(pool.size() == 3);
-  }
-
   SECTION("Doesn't grow when memory pool is full") {
     const size_t variantCount = sizeof(buffer) / sizeof(VariantSlot);
 

extras/tests/Misc/StringAdapters.cpp

@@ -6,6 +6,7 @@
 #include "progmem_emulation.hpp"
 #include "weird_strcmp.hpp"
 
+#include <ArduinoJson/Strings/ArduinoStringAdapter.hpp>
 #include <ArduinoJson/Strings/ConstRamStringAdapter.hpp>
 #include <ArduinoJson/Strings/FlashStringAdapter.hpp>
 #include <ArduinoJson/Strings/SizedRamStringAdapter.hpp>
@@ -114,6 +115,21 @@ TEST_CASE("std::string") {
   CHECK(adapter.size() == 5);
 }
 
+TEST_CASE("Arduino String") {
+  ::String str("bravo");
+  ArduinoStringAdapter adapter = adaptString(str);
+
+  CHECK(adapter.compare(NULL) > 0);
+  CHECK(adapter.compare("alpha") > 0);
+  CHECK(adapter.compare("bravo") == 0);
+  CHECK(adapter.compare("charlie") < 0);
+
+  CHECK(adapter.equals("bravo"));
+  CHECK_FALSE(adapter.equals("charlie"));
+
+  CHECK(adapter.size() == 5);
+}
+
 TEST_CASE("custom_string") {
   custom_string str("bravo");
   StlStringAdapter<custom_string> adapter = adaptString(str);

extras/tests/MixedConfiguration/CMakeLists.txt

@@ -18,6 +18,8 @@ add_executable(MixedConfigurationTests
 	enable_nan_0.cpp
 	enable_nan_1.cpp
 	enable_progmem_1.cpp
+	enable_string_deduplication_0.cpp
+	enable_string_deduplication_1.cpp
 	use_double_0.cpp
 	use_double_1.cpp
 	use_long_long_0.cpp

extras/tests/MixedConfiguration/enable_string_deduplication_0.cpp

@@ -0,0 +1,125 @@
+// ArduinoJson - arduinojson.org
+// Copyright Benoit Blanchon 2014-2020
+// MIT License
+
+#include "progmem_emulation.hpp"
+
+#define ARDUINOJSON_ENABLE_ARDUINO_STRING 1
+#define ARDUINOJSON_ENABLE_PROGMEM 1
+#define ARDUINOJSON_ENABLE_STRING_DEDUPLICATION 0
+#include <ArduinoJson.h>
+
+#include <catch.hpp>
+
+TEST_CASE("ARDUINOJSON_ENABLE_STRING_DEDUPLICATION = 0") {
+  StaticJsonDocument<1024> doc;
+
+  SECTION("deserializeJson()") {
+    SECTION("Deduplicate values") {
+      deserializeJson(doc, "[\"example\",\"example\"]");
+
+      CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 16);
+      CHECK(doc[0].as<char*>() != doc[1].as<char*>());
+    }
+
+    SECTION("Deduplicate keys") {
+      deserializeJson(doc, "[{\"example\":1},{\"example\":2}]");
+
+      CHECK(doc.memoryUsage() ==
+            2 * JSON_OBJECT_SIZE(1) + JSON_ARRAY_SIZE(2) + 16);
+
+      const char* key1 = doc[0].as<JsonObject>().begin()->key().c_str();
+      const char* key2 = doc[1].as<JsonObject>().begin()->key().c_str();
+
+      CHECK(key1 != key2);
+    }
+  }
+
+  SECTION("JsonDocument") {
+    SECTION("values") {
+      SECTION("std::string") {
+        doc.add(std::string("example"));
+        doc.add(std::string("example"));
+
+        CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 16);
+        CHECK(doc[0].as<char*>() != doc[1].as<char*>());
+      }
+
+      SECTION("char*") {
+        char value[] = "example";
+        doc.add(value);
+        doc.add(value);
+
+        CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 16);
+        CHECK(doc[0].as<char*>() != doc[1].as<char*>());
+      }
+
+      SECTION("Arduino String") {
+        doc.add(String("example"));
+        doc.add(String("example"));
+
+        CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 16);
+        CHECK(doc[0].as<char*>() != doc[1].as<char*>());
+      }
+
+      SECTION("Flash string") {
+        doc.add(F("example"));
+        doc.add(F("example"));
+
+        CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 16);
+        CHECK(doc[0].as<char*>() != doc[1].as<char*>());
+      }
+    }
+
+    SECTION("keys") {
+      SECTION("std::string") {
+        doc[0][std::string("example")] = 1;
+        doc[1][std::string("example")] = 2;
+
+        CHECK(doc.memoryUsage() ==
+              JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 16);
+
+        const char* key1 = doc[0].as<JsonObject>().begin()->key().c_str();
+        const char* key2 = doc[1].as<JsonObject>().begin()->key().c_str();
+        CHECK(key1 != key2);
+      }
+
+      SECTION("char*") {
+        char key[] = "example";
+        doc[0][key] = 1;
+        doc[1][key] = 2;
+
+        CHECK(doc.memoryUsage() ==
+              JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 16);
+
+        const char* key1 = doc[0].as<JsonObject>().begin()->key().c_str();
+        const char* key2 = doc[1].as<JsonObject>().begin()->key().c_str();
+        CHECK(key1 != key2);
+      }
+
+      SECTION("Arduino String") {
+        doc[0][String("example")] = 1;
+        doc[1][String("example")] = 2;
+
+        CHECK(doc.memoryUsage() ==
+              JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 16);
+
+        const char* key1 = doc[0].as<JsonObject>().begin()->key().c_str();
+        const char* key2 = doc[1].as<JsonObject>().begin()->key().c_str();
+        CHECK(key1 != key2);
+      }
+
+      SECTION("Flash string") {
+        doc[0][F("example")] = 1;
+        doc[1][F("example")] = 2;
+
+        CHECK(doc.memoryUsage() ==
+              JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 16);
+
+        const char* key1 = doc[0].as<JsonObject>().begin()->key().c_str();
+        const char* key2 = doc[1].as<JsonObject>().begin()->key().c_str();
+        CHECK(key1 != key2);
+      }
+    }
+  }
+}

extras/tests/MixedConfiguration/enable_string_deduplication_1.cpp

@@ -0,0 +1,124 @@
+// ArduinoJson - arduinojson.org
+// Copyright Benoit Blanchon 2014-2020
+// MIT License
+
+#include "progmem_emulation.hpp"
+
+#define ARDUINOJSON_ENABLE_ARDUINO_STRING 1
+#define ARDUINOJSON_ENABLE_PROGMEM 1
+#define ARDUINOJSON_ENABLE_STRING_DEDUPLICATION 1
+#include <ArduinoJson.h>
+
+#include <catch.hpp>
+
+TEST_CASE("ARDUINOJSON_ENABLE_STRING_DEDUPLICATION = 1") {
+  StaticJsonDocument<1024> doc;
+
+  SECTION("deserializeJson()") {
+    SECTION("Deduplicate values") {
+      deserializeJson(doc, "[\"example\",\"example\"]");
+
+      CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 8);
+      CHECK(doc[0].as<char*>() == doc[1].as<char*>());
+    }
+
+    SECTION("Deduplicate keys") {
+      deserializeJson(doc, "[{\"example\":1},{\"example\":2}]");
+
+      CHECK(doc.memoryUsage() ==
+            2 * JSON_OBJECT_SIZE(1) + JSON_ARRAY_SIZE(2) + 8);
+
+      const char* key1 = doc[0].as<JsonObject>().begin()->key().c_str();
+      const char* key2 = doc[1].as<JsonObject>().begin()->key().c_str();
+      CHECK(key1 == key2);
+    }
+  }
+
+  SECTION("JsonDocument") {
+    SECTION("values") {
+      SECTION("std::string") {
+        doc.add(std::string("example"));
+        doc.add(std::string("example"));
+
+        CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 8);
+        CHECK(doc[0].as<char*>() == doc[1].as<char*>());
+      }
+
+      SECTION("char*") {
+        char value[] = "example";
+        doc.add(value);
+        doc.add(value);
+
+        CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 8);
+        CHECK(doc[0].as<char*>() == doc[1].as<char*>());
+      }
+
+      SECTION("Arduino String") {
+        doc.add(String("example"));
+        doc.add(String("example"));
+
+        CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 8);
+        CHECK(doc[0].as<char*>() == doc[1].as<char*>());
+      }
+
+      SECTION("Flash string") {
+        doc.add(F("example"));
+        doc.add(F("example"));
+
+        CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 8);
+        CHECK(doc[0].as<char*>() == doc[1].as<char*>());
+      }
+    }
+
+    SECTION("keys") {
+      SECTION("std::string") {
+        doc[0][std::string("example")] = 1;
+        doc[1][std::string("example")] = 2;
+
+        CHECK(doc.memoryUsage() ==
+              JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 8);
+
+        const char* key1 = doc[0].as<JsonObject>().begin()->key().c_str();
+        const char* key2 = doc[1].as<JsonObject>().begin()->key().c_str();
+        CHECK(key1 == key2);
+      }
+
+      SECTION("char*") {
+        char key[] = "example";
+        doc[0][key] = 1;
+        doc[1][key] = 2;
+
+        CHECK(doc.memoryUsage() ==
+              JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 8);
+
+        const char* key1 = doc[0].as<JsonObject>().begin()->key().c_str();
+        const char* key2 = doc[1].as<JsonObject>().begin()->key().c_str();
+        CHECK(key1 == key2);
+      }
+
+      SECTION("Arduino String") {
+        doc[0][String("example")] = 1;
+        doc[1][String("example")] = 2;
+
+        CHECK(doc.memoryUsage() ==
+              JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 8);
+
+        const char* key1 = doc[0].as<JsonObject>().begin()->key().c_str();
+        const char* key2 = doc[1].as<JsonObject>().begin()->key().c_str();
+        CHECK(key1 == key2);
+      }
+
+      SECTION("Flash string") {
+        doc[0][F("example")] = 1;
+        doc[1][F("example")] = 2;
+
+        CHECK(doc.memoryUsage() ==
+              JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 8);
+
+        const char* key1 = doc[0].as<JsonObject>().begin()->key().c_str();
+        const char* key2 = doc[1].as<JsonObject>().begin()->key().c_str();
+        CHECK(key1 == key2);
+      }
+    }
+  }
+}