From 9c3baa7f14c7756e03d38491d3e58414b02c247a Mon Sep 17 00:00:00 2001 From: ljacqu Date: Mon, 13 Mar 2017 20:29:08 +0100 Subject: [PATCH] #1125 Create persistence of LimboPlayers in segment files (work in progress) - Instead of one huge file or a file for each player, allow the user to define how many files he wants to distribute the LimboPlayers over. This is based on a function that creates a String (segment ID) based on the player's UUID. --- .../persistence/LimboPersistenceType.java | 2 + .../persistence/SegmentConfiguration.java | 94 +++++++++++ .../SegmentFilesPersistenceHolder.java | 132 +++++++++++++++ .../limbo/persistence/SegmentNameBuilder.java | 60 +++++++ .../settings/properties/LimboSettings.java | 20 ++- .../persistence/SegmentConfigurationTest.java | 47 ++++++ .../persistence/SegmentNameBuilderTest.java | 153 ++++++++++++++++++ 7 files changed, 507 insertions(+), 1 deletion(-) create mode 100644 src/main/java/fr/xephi/authme/data/limbo/persistence/SegmentConfiguration.java create mode 100644 src/main/java/fr/xephi/authme/data/limbo/persistence/SegmentFilesPersistenceHolder.java create mode 100644 src/main/java/fr/xephi/authme/data/limbo/persistence/SegmentNameBuilder.java create mode 100644 src/test/java/fr/xephi/authme/data/limbo/persistence/SegmentConfigurationTest.java create mode 100644 src/test/java/fr/xephi/authme/data/limbo/persistence/SegmentNameBuilderTest.java diff --git a/src/main/java/fr/xephi/authme/data/limbo/persistence/LimboPersistenceType.java b/src/main/java/fr/xephi/authme/data/limbo/persistence/LimboPersistenceType.java index 294ccbff..c998f95a 100644 --- a/src/main/java/fr/xephi/authme/data/limbo/persistence/LimboPersistenceType.java +++ b/src/main/java/fr/xephi/authme/data/limbo/persistence/LimboPersistenceType.java @@ -9,6 +9,8 @@ public enum LimboPersistenceType { SINGLE_FILE(SingleFilePersistenceHandler.class), + SEGMENT_FILES(SegmentFilesPersistenceHolder.class), + DISABLED(NoOpPersistenceHandler.class); private final Class implementationClass; diff --git a/src/main/java/fr/xephi/authme/data/limbo/persistence/SegmentConfiguration.java b/src/main/java/fr/xephi/authme/data/limbo/persistence/SegmentConfiguration.java new file mode 100644 index 00000000..d2f0d202 --- /dev/null +++ b/src/main/java/fr/xephi/authme/data/limbo/persistence/SegmentConfiguration.java @@ -0,0 +1,94 @@ +package fr.xephi.authme.data.limbo.persistence; + +/** + * Configuration for the total number of segments to use. + *

+ * The {@link SegmentFilesPersistenceHolder} reduces the number of files by assigning each UUID + * to a segment. This enum allows to define how many segments the UUIDs should be distributed in. + *

+ * Segments are defined by a distribution and a length. The distribution defines + * to how many outputs a single hexadecimal characters should be mapped. So e.g. a distribution + * of 3 means that all hexadecimal characters 0-f should be distributed over three different + * outputs evenly. The {@link SegmentNameBuilder} simply uses hexadecimal characters as outputs, + * so e.g. with a distribution of 3 all hex characters 0-f are mapped to 0, 1, or 2. + *

+ * To ensure an even distribution the segments must be powers of 2. Trivially, to implement a + * distribution of 16, the same character may be returned as was input (since 0-f make up 16 + * characters). A distribution of 1, on the other hand, means that the same output is returned + * regardless of the input character. + *

+ * The length parameter defines how many characters of a player's UUID should be used to + * create the segment ID. In other words, with a distribution of 2 and a length of 3, the first + * three characters of the UUID are taken into consideration, each mapped to one of two possible + * characters. For instance, a UUID starting with "0f5c9321" may yield the segment ID "010." + * Such a segment ID defines in which file the given UUID can be found and stored. + *

+ * The number of segments such a configuration yields is computed as {@code distribution ^ length}, + * since distribution defines how many outputs there are per digit, and length defines the number + * of digits. For instance, a distribution of 2 and a length of 3 will yield segment IDs 000, 001, + * 010, 011, 100, 101, 110 and 111 (i.e. all binary numbers from 0 to 7). + *

+ * There are multiple possibilities to achieve certain segment totals, e.g. 8 different segments + * may be created by setting distribution to 8 and length to 1, or distr. to 2 and length to 3. + * Where possible, prefer a length of 1 (no string concatenation required) or a distribution of + * 16 (no remapping of the characters required). + */ +public enum SegmentConfiguration { + + /** 1. */ + ONE(1, 1), + + /** 2. */ + TWO(2, 1), + + /** 4. */ + FOUR(4, 1), + + /** 8. */ + EIGHT(8, 1), + + /** 16. */ + SIXTEEN(16, 1), + + /** 32. */ + THIRTY_TWO(2, 5), + + /** 64. */ + SIXTY_FOUR(4, 3), + + /** 128. */ + ONE_TWENTY(2, 7), + + /** 256. */ + TWO_FIFTY(16, 2); + + private final int distribution; + private final int length; + + SegmentConfiguration(int distribution, int length) { + this.distribution = distribution; + this.length = length; + } + + /** + * @return the distribution size per character, i.e. how many possible outputs there are + * for any hexadecimal character + */ + public int getDistribution() { + return distribution; + } + + /** + * @return number of characters from a UUID that should be used to create a segment ID + */ + public int getLength() { + return length; + } + + /** + * @return number of segments to which this configuration will distribute UUIDs + */ + public int getTotalSegments() { + return (int) Math.pow(distribution, length); + } +} diff --git a/src/main/java/fr/xephi/authme/data/limbo/persistence/SegmentFilesPersistenceHolder.java b/src/main/java/fr/xephi/authme/data/limbo/persistence/SegmentFilesPersistenceHolder.java new file mode 100644 index 00000000..24751fe0 --- /dev/null +++ b/src/main/java/fr/xephi/authme/data/limbo/persistence/SegmentFilesPersistenceHolder.java @@ -0,0 +1,132 @@ +package fr.xephi.authme.data.limbo.persistence; + +import com.google.common.io.Files; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import fr.xephi.authme.ConsoleLogger; +import fr.xephi.authme.data.limbo.LimboPlayer; +import fr.xephi.authme.initialization.DataFolder; +import fr.xephi.authme.service.BukkitService; +import fr.xephi.authme.settings.Settings; +import fr.xephi.authme.settings.properties.LimboSettings; +import fr.xephi.authme.util.FileUtils; +import fr.xephi.authme.util.PlayerUtils; +import org.bukkit.entity.Player; + +import javax.inject.Inject; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.lang.reflect.Type; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +/** + * Persistence handler for LimboPlayer objects by distributing the objects to store + * in various segments (buckets) based on the start of the player's UUID. + */ +class SegmentFilesPersistenceHolder implements LimboPersistenceHandler { + + private static final Type LIMBO_MAP_TYPE = new TypeToken>(){}.getType(); + + private final File cacheFolder; + private final Gson gson; + private final SegmentNameBuilder segmentNameBuilder; + + @Inject + SegmentFilesPersistenceHolder(@DataFolder File dataFolder, BukkitService bukkitService, Settings settings) { + cacheFolder = new File(dataFolder, "playerdata"); + if (!cacheFolder.exists()) { + // TODO ljacqu 20170313: Create FileUtils#mkdirs + cacheFolder.mkdirs(); + } + + gson = new GsonBuilder() + .registerTypeAdapter(LimboPlayer.class, new LimboPlayerSerializer()) + .registerTypeAdapter(LimboPlayer.class, new LimboPlayerDeserializer(bukkitService)) + .setPrettyPrinting() + .create(); + + segmentNameBuilder = new SegmentNameBuilder(settings.getProperty(LimboSettings.SEGMENT_DISTRIBUTION)); + + // TODO #1125: Check for other segment files and attempt to convert? + } + + @Override + public LimboPlayer getLimboPlayer(Player player) { + String uuid = PlayerUtils.getUUIDorName(player); + File file = getPlayerSegmentFile(uuid); + Map entries = readLimboPlayers(file); + return entries == null ? null : entries.get(uuid); + } + + @Override + public void saveLimboPlayer(Player player, LimboPlayer limbo) { + String uuid = PlayerUtils.getUUIDorName(player); + File file = getPlayerSegmentFile(uuid); + + Map entries = null; + if (file.exists()) { + entries = readLimboPlayers(file); + } else { + FileUtils.create(file); + } + /* intentionally separate if */ + if (entries == null) { + entries = new HashMap<>(); + } + + entries.put(PlayerUtils.getUUIDorName(player), limbo); + saveEntries(entries, file); + } + + @Override + public void removeLimboPlayer(Player player) { + String uuid = PlayerUtils.getUUIDorName(player); + File file = getPlayerSegmentFile(uuid); + if (file.exists()) { + Map entries = readLimboPlayers(file); + if (entries != null && entries.remove(PlayerUtils.getUUIDorName(player)) != null) { + saveEntries(entries, file); + } + } + } + + private void saveEntries(Map entries, File file) { + if (entries.isEmpty()) { + // TODO #1125: Probably should do a sweep of empty files on startup / shutdown, but not all the time + FileUtils.delete(file); + } else { + try (FileWriter fw = new FileWriter(file)) { + gson.toJson(entries, fw); + } catch (IOException e) { + ConsoleLogger.logException("Could not write to '" + file + "':", e); + } + } + } + + private Map readLimboPlayers(File file) { + if (!file.exists()) { + return null; + } + + try { + return gson.fromJson(Files.toString(file, StandardCharsets.UTF_8), LIMBO_MAP_TYPE); + } catch (IOException e) { + ConsoleLogger.logException("Failed reading '" + file + "':", e); + } + return null; + } + + private File getPlayerSegmentFile(String uuid) { + String segment = segmentNameBuilder.createSegmentName(uuid); + return new File(cacheFolder, segment + "-limbo.json"); + } + + @Override + public LimboPersistenceType getType() { + return LimboPersistenceType.SINGLE_FILE; + } +} diff --git a/src/main/java/fr/xephi/authme/data/limbo/persistence/SegmentNameBuilder.java b/src/main/java/fr/xephi/authme/data/limbo/persistence/SegmentNameBuilder.java new file mode 100644 index 00000000..df517fba --- /dev/null +++ b/src/main/java/fr/xephi/authme/data/limbo/persistence/SegmentNameBuilder.java @@ -0,0 +1,60 @@ +package fr.xephi.authme.data.limbo.persistence; + +import java.util.HashMap; +import java.util.Map; + +/** + * Creates segment names for {@link SegmentFilesPersistenceHolder}. + */ +class SegmentNameBuilder { + + private final int length; + private final int distribution; + private final String prefix; + private final Map charToSegmentChar; + + /** + * Constructor. + * + * @param partition the segment configuration + */ + SegmentNameBuilder(SegmentConfiguration partition) { + this.length = partition.getLength(); + this.distribution = partition.getDistribution(); + this.prefix = "seg" + partition.getTotalSegments() + "-"; + this.charToSegmentChar = buildCharMap(distribution); + } + + String createSegmentName(String uuid) { + if (distribution == 16) { + return prefix + uuid.substring(0, length); + } else { + return prefix + createSegmentName(uuid.substring(0, length).toCharArray()); + } + } + + private String createSegmentName(char[] chars) { + if (chars.length == 1) { + return String.valueOf(charToSegmentChar.get(chars[0])); + } + + StringBuilder sb = new StringBuilder(chars.length); + for (char chr : chars) { + sb.append(charToSegmentChar.get(chr)); + } + return sb.toString(); + } + + private static Map buildCharMap(int distribution) { + final char[] hexChars = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + final int divisor = 16 / distribution; + + Map charToSegmentChar = new HashMap<>(); + for (int i = 0; i < hexChars.length; ++i) { + int mappedChar = i / divisor; + charToSegmentChar.put(hexChars[i], hexChars[mappedChar]); + } + return charToSegmentChar; + } + +} diff --git a/src/main/java/fr/xephi/authme/settings/properties/LimboSettings.java b/src/main/java/fr/xephi/authme/settings/properties/LimboSettings.java index 1d7a29d7..d8d7104e 100644 --- a/src/main/java/fr/xephi/authme/settings/properties/LimboSettings.java +++ b/src/main/java/fr/xephi/authme/settings/properties/LimboSettings.java @@ -8,6 +8,7 @@ import com.google.common.collect.ImmutableMap; import fr.xephi.authme.data.limbo.AllowFlightRestoreType; import fr.xephi.authme.data.limbo.WalkFlySpeedRestoreType; import fr.xephi.authme.data.limbo.persistence.LimboPersistenceType; +import fr.xephi.authme.data.limbo.persistence.SegmentConfiguration; import java.util.Map; @@ -23,11 +24,28 @@ public final class LimboSettings implements SettingsHolder { "on disk. This is useful in case of a server crash, so next time the server starts we can", "properly restore things like OP status, ability to fly, and walk/fly speed.", "DISABLED: no disk storage, INDIVIDUAL_FILES: each player data in its own file,", - "SINGLE_FILE: all data in one single file (only if you have a small server!)" + "SINGLE_FILE: all data in one single file (only if you have a small server!)", + "SEGMENT_FILES: distributes players into different buckets based on their UUID. See below." }) public static final Property LIMBO_PERSISTENCE_TYPE = newProperty(LimboPersistenceType.class, "limbo.persistence.type", LimboPersistenceType.INDIVIDUAL_FILES); + @Comment({ + "This setting only affects SEGMENT_FILES persistence. The segment file", + "persistence attempts to reduce the number of files by distributing players into various", + "buckets based on their UUID. This setting defines into how many files the players should", + "be distributed. Possible values: ONE, TWO, FOUR, EIGHT, SIXTEEN, THIRTY_TWO, SIXTY_FOUR,", + "ONE_TWENTY for 128, TWO_FIFTY for 256.", + "For example, if you expect 100 non-logged in players, setting to SIXTEEN will average", + "6.25 players per file (100 / 16). If you set to ONE, like persistence SINGLE_FILE, only", + "one file will be used. Contrary to SINGLE_FILE, it won't keep the entries in cache, which", + "may deliver different results in terms of performance.", + "Note: if you change this setting you lose all stored LimboPlayer data because the", + "distribution of players will be different." + }) + public static final Property SEGMENT_DISTRIBUTION = + newProperty(SegmentConfiguration.class, "limbo.persistence.segmentDistribution", SegmentConfiguration.SIXTEEN); + @Comment({ "Whether the player is allowed to fly: RESTORE, ENABLE, DISABLE.", "RESTORE sets back the old property from the player." diff --git a/src/test/java/fr/xephi/authme/data/limbo/persistence/SegmentConfigurationTest.java b/src/test/java/fr/xephi/authme/data/limbo/persistence/SegmentConfigurationTest.java new file mode 100644 index 00000000..f5e5e124 --- /dev/null +++ b/src/test/java/fr/xephi/authme/data/limbo/persistence/SegmentConfigurationTest.java @@ -0,0 +1,47 @@ +package fr.xephi.authme.data.limbo.persistence; + +import com.google.common.collect.ImmutableSet; +import org.junit.Test; + +import java.util.HashSet; +import java.util.Set; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.junit.Assert.fail; + +/** + * Test for {@link SegmentConfiguration}. + */ +public class SegmentConfigurationTest { + + @Test + public void shouldHaveDistributionThatIsPowerOf2() { + // given + Set allowedDistributions = ImmutableSet.of(1, 2, 4, 8, 16); + + // when / then + for (SegmentConfiguration entry : SegmentConfiguration.values()) { + if (!allowedDistributions.contains(entry.getDistribution())) { + fail("Distribution must be a power of 2 and within [1, 16]. Offending item: " + entry); + } + } + } + + @Test + public void shouldHaveDifferentSegmentSizes() { + // given + Set sizes = new HashSet<>(); + + // when / then + for (SegmentConfiguration entry : SegmentConfiguration.values()) { + int segSize = (int) Math.pow(entry.getDistribution(), entry.getLength()); + assertThat(entry + " must have a positive segment size", + segSize, greaterThan(0)); + + assertThat(entry + " has a segment size that was already encountered (" + segSize + ")", + sizes.add(segSize), equalTo(true)); + } + } +} diff --git a/src/test/java/fr/xephi/authme/data/limbo/persistence/SegmentNameBuilderTest.java b/src/test/java/fr/xephi/authme/data/limbo/persistence/SegmentNameBuilderTest.java new file mode 100644 index 00000000..41c3b0ff --- /dev/null +++ b/src/test/java/fr/xephi/authme/data/limbo/persistence/SegmentNameBuilderTest.java @@ -0,0 +1,153 @@ +package fr.xephi.authme.data.limbo.persistence; + +import org.junit.Test; + +import java.util.HashSet; +import java.util.Set; + +import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.EIGHT; +import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.FOUR; +import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.ONE; +import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.SIXTEEN; +import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.SIXTY_FOUR; +import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.THIRTY_TWO; +import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.TWO; +import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.TWO_FIFTY; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.junit.Assert.assertThat; + +/** + * Test for {@link SegmentNameBuilder}. + */ +public class SegmentNameBuilderTest { + + /** + * Checks that using a given segment size really produces as many segments as defined. + * E.g. if we partition with {@link SegmentConfiguration#EIGHT} we expect eight different buckets. + */ + @Test + public void shouldCreatePromisedSizeOfSegments() { + for (SegmentConfiguration part : SegmentConfiguration.values()) { + // Perform this check only for `length` <= 5 because the test creates all hex numbers with `length` digits. + if (part.getLength() <= 5) { + checkTotalSegmentsProduced(part); + } + } + } + + private void checkTotalSegmentsProduced(SegmentConfiguration part) { + // given + SegmentNameBuilder nameBuilder = new SegmentNameBuilder(part); + Set encounteredSegments = new HashSet<>(); + int shift = part.getLength() * 4; + // e.g. (1 << 16) - 1 = 0xFFFF. (Number of digits = shift/4, since 16 = 2^4) + int max = (1 << shift) - 1; + + // when + for (int i = 0; i <= max; ++i) { + String uuid = toPaddedHex(i, part.getLength()); + encounteredSegments.add(nameBuilder.createSegmentName(uuid)); + } + + // then + assertThat(encounteredSegments, hasSize(part.getTotalSegments())); + } + + private static String toPaddedHex(int dec, int padLength) { + String hexResult = Integer.toString(dec, 16); + while (hexResult.length() < padLength) { + hexResult = "0" + hexResult; + } + return hexResult; + } + + @Test + public void shouldCreateOneSegment() { + // given + SegmentNameBuilder nameBuilder = new SegmentNameBuilder(ONE); + + // when / then + assertThat(nameBuilder.createSegmentName("abc"), equalTo("seg1-0")); + assertThat(nameBuilder.createSegmentName("f0e"), equalTo("seg1-0")); + assertThat(nameBuilder.createSegmentName("329"), equalTo("seg1-0")); + } + + @Test + public void shouldCreateTwoSegments() { + // given + SegmentNameBuilder nameBuilder = new SegmentNameBuilder(TWO); + + // when / then + assertThat(nameBuilder.createSegmentName("f6c"), equalTo("seg2-1")); + assertThat(nameBuilder.createSegmentName("29f"), equalTo("seg2-0")); + assertThat(nameBuilder.createSegmentName("983"), equalTo("seg2-1")); + } + + @Test + public void shouldCreateFourSegments() { + // given + SegmentNameBuilder nameBuilder = new SegmentNameBuilder(FOUR); + + // when / then + assertThat(nameBuilder.createSegmentName("f9cc"), equalTo("seg4-3")); + assertThat(nameBuilder.createSegmentName("84c9"), equalTo("seg4-2")); + assertThat(nameBuilder.createSegmentName("3799"), equalTo("seg4-0")); + } + + @Test + public void shouldCreateEightSegments() { + // given + SegmentNameBuilder nameBuilder = new SegmentNameBuilder(EIGHT); + + // when / then + assertThat(nameBuilder.createSegmentName("fc9c"), equalTo("seg8-7")); + assertThat(nameBuilder.createSegmentName("90ad"), equalTo("seg8-4")); + assertThat(nameBuilder.createSegmentName("35e4"), equalTo("seg8-1")); + assertThat(nameBuilder.createSegmentName("a39f"), equalTo("seg8-5")); + } + + @Test + public void shouldCreateSixteenSegments() { + // given + SegmentNameBuilder nameBuilder = new SegmentNameBuilder(SIXTEEN); + + // when / then + assertThat(nameBuilder.createSegmentName("fc9a054"), equalTo("seg16-f")); + assertThat(nameBuilder.createSegmentName("b0a945e"), equalTo("seg16-b")); + assertThat(nameBuilder.createSegmentName("7afebab"), equalTo("seg16-7")); + } + + @Test + public void shouldCreateThirtyTwoSegments() { + // given + SegmentNameBuilder nameBuilder = new SegmentNameBuilder(THIRTY_TWO); + + // when / then + assertThat(nameBuilder.createSegmentName("f890c9"), equalTo("seg32-11101")); + assertThat(nameBuilder.createSegmentName("49c39a"), equalTo("seg32-01101")); + assertThat(nameBuilder.createSegmentName("b75d09"), equalTo("seg32-10010")); + } + + @Test + public void shouldCreateSixtyFourSegments() { + // given + SegmentNameBuilder nameBuilder = new SegmentNameBuilder(SIXTY_FOUR); + + // when / then + assertThat(nameBuilder.createSegmentName("82f"), equalTo("seg64-203")); + assertThat(nameBuilder.createSegmentName("9b4"), equalTo("seg64-221")); + assertThat(nameBuilder.createSegmentName("068"), equalTo("seg64-012")); + } + + @Test + public void shouldCreate256Segments() { + // given + SegmentNameBuilder nameBuilder = new SegmentNameBuilder(TWO_FIFTY); + + // when / then + assertThat(nameBuilder.createSegmentName("a813c"), equalTo("seg256-a8")); + assertThat(nameBuilder.createSegmentName("b4d01"), equalTo("seg256-b4")); + assertThat(nameBuilder.createSegmentName("7122f"), equalTo("seg256-71")); + } +}