#1125 Create persistence of LimboPlayers in segment files (work in progress)

- Instead of one huge file or a file for each player, allow the user to define how many files he wants to distribute the LimboPlayers over. This is based on a function that creates a String (segment ID) based on the player's UUID.
This commit is contained in:
ljacqu 2017-03-13 20:29:08 +01:00
parent 710198833c
commit 9c3baa7f14
7 changed files with 507 additions and 1 deletions

View File

@ -9,6 +9,8 @@ public enum LimboPersistenceType {
SINGLE_FILE(SingleFilePersistenceHandler.class),
SEGMENT_FILES(SegmentFilesPersistenceHolder.class),
DISABLED(NoOpPersistenceHandler.class);
private final Class<? extends LimboPersistenceHandler> implementationClass;

View File

@ -0,0 +1,94 @@
package fr.xephi.authme.data.limbo.persistence;
/**
* Configuration for the total number of segments to use.
* <p>
* The {@link SegmentFilesPersistenceHolder} reduces the number of files by assigning each UUID
* to a segment. This enum allows to define how many segments the UUIDs should be distributed in.
* <p>
* Segments are defined by a <b>distribution</b> and a <b>length.</b> The distribution defines
* to how many outputs a single hexadecimal characters should be mapped. So e.g. a distribution
* of 3 means that all hexadecimal characters 0-f should be distributed over three different
* outputs evenly. The {@link SegmentNameBuilder} simply uses hexadecimal characters as outputs,
* so e.g. with a distribution of 3 all hex characters 0-f are mapped to 0, 1, or 2.
* <p>
* To ensure an even distribution the segments must be powers of 2. Trivially, to implement a
* distribution of 16, the same character may be returned as was input (since 0-f make up 16
* characters). A distribution of 1, on the other hand, means that the same output is returned
* regardless of the input character.
* <p>
* The <b>length</b> parameter defines how many characters of a player's UUID should be used to
* create the segment ID. In other words, with a distribution of 2 and a length of 3, the first
* three characters of the UUID are taken into consideration, each mapped to one of two possible
* characters. For instance, a UUID starting with "0f5c9321" may yield the segment ID "010."
* Such a segment ID defines in which file the given UUID can be found and stored.
* <p>
* The number of segments such a configuration yields is computed as {@code distribution ^ length},
* since distribution defines how many outputs there are per digit, and length defines the number
* of digits. For instance, a distribution of 2 and a length of 3 will yield segment IDs 000, 001,
* 010, 011, 100, 101, 110 and 111 (i.e. all binary numbers from 0 to 7).
* <p>
* There are multiple possibilities to achieve certain segment totals, e.g. 8 different segments
* may be created by setting distribution to 8 and length to 1, or distr. to 2 and length to 3.
* Where possible, prefer a length of 1 (no string concatenation required) or a distribution of
* 16 (no remapping of the characters required).
*/
public enum SegmentConfiguration {
/** 1. */
ONE(1, 1),
/** 2. */
TWO(2, 1),
/** 4. */
FOUR(4, 1),
/** 8. */
EIGHT(8, 1),
/** 16. */
SIXTEEN(16, 1),
/** 32. */
THIRTY_TWO(2, 5),
/** 64. */
SIXTY_FOUR(4, 3),
/** 128. */
ONE_TWENTY(2, 7),
/** 256. */
TWO_FIFTY(16, 2);
private final int distribution;
private final int length;
SegmentConfiguration(int distribution, int length) {
this.distribution = distribution;
this.length = length;
}
/**
* @return the distribution size per character, i.e. how many possible outputs there are
* for any hexadecimal character
*/
public int getDistribution() {
return distribution;
}
/**
* @return number of characters from a UUID that should be used to create a segment ID
*/
public int getLength() {
return length;
}
/**
* @return number of segments to which this configuration will distribute UUIDs
*/
public int getTotalSegments() {
return (int) Math.pow(distribution, length);
}
}

View File

@ -0,0 +1,132 @@
package fr.xephi.authme.data.limbo.persistence;
import com.google.common.io.Files;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import fr.xephi.authme.ConsoleLogger;
import fr.xephi.authme.data.limbo.LimboPlayer;
import fr.xephi.authme.initialization.DataFolder;
import fr.xephi.authme.service.BukkitService;
import fr.xephi.authme.settings.Settings;
import fr.xephi.authme.settings.properties.LimboSettings;
import fr.xephi.authme.util.FileUtils;
import fr.xephi.authme.util.PlayerUtils;
import org.bukkit.entity.Player;
import javax.inject.Inject;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.reflect.Type;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
/**
* Persistence handler for LimboPlayer objects by distributing the objects to store
* in various segments (buckets) based on the start of the player's UUID.
*/
class SegmentFilesPersistenceHolder implements LimboPersistenceHandler {
private static final Type LIMBO_MAP_TYPE = new TypeToken<Map<String, LimboPlayer>>(){}.getType();
private final File cacheFolder;
private final Gson gson;
private final SegmentNameBuilder segmentNameBuilder;
@Inject
SegmentFilesPersistenceHolder(@DataFolder File dataFolder, BukkitService bukkitService, Settings settings) {
cacheFolder = new File(dataFolder, "playerdata");
if (!cacheFolder.exists()) {
// TODO ljacqu 20170313: Create FileUtils#mkdirs
cacheFolder.mkdirs();
}
gson = new GsonBuilder()
.registerTypeAdapter(LimboPlayer.class, new LimboPlayerSerializer())
.registerTypeAdapter(LimboPlayer.class, new LimboPlayerDeserializer(bukkitService))
.setPrettyPrinting()
.create();
segmentNameBuilder = new SegmentNameBuilder(settings.getProperty(LimboSettings.SEGMENT_DISTRIBUTION));
// TODO #1125: Check for other segment files and attempt to convert?
}
@Override
public LimboPlayer getLimboPlayer(Player player) {
String uuid = PlayerUtils.getUUIDorName(player);
File file = getPlayerSegmentFile(uuid);
Map<String, LimboPlayer> entries = readLimboPlayers(file);
return entries == null ? null : entries.get(uuid);
}
@Override
public void saveLimboPlayer(Player player, LimboPlayer limbo) {
String uuid = PlayerUtils.getUUIDorName(player);
File file = getPlayerSegmentFile(uuid);
Map<String, LimboPlayer> entries = null;
if (file.exists()) {
entries = readLimboPlayers(file);
} else {
FileUtils.create(file);
}
/* intentionally separate if */
if (entries == null) {
entries = new HashMap<>();
}
entries.put(PlayerUtils.getUUIDorName(player), limbo);
saveEntries(entries, file);
}
@Override
public void removeLimboPlayer(Player player) {
String uuid = PlayerUtils.getUUIDorName(player);
File file = getPlayerSegmentFile(uuid);
if (file.exists()) {
Map<String, LimboPlayer> entries = readLimboPlayers(file);
if (entries != null && entries.remove(PlayerUtils.getUUIDorName(player)) != null) {
saveEntries(entries, file);
}
}
}
private void saveEntries(Map<String, LimboPlayer> entries, File file) {
if (entries.isEmpty()) {
// TODO #1125: Probably should do a sweep of empty files on startup / shutdown, but not all the time
FileUtils.delete(file);
} else {
try (FileWriter fw = new FileWriter(file)) {
gson.toJson(entries, fw);
} catch (IOException e) {
ConsoleLogger.logException("Could not write to '" + file + "':", e);
}
}
}
private Map<String, LimboPlayer> readLimboPlayers(File file) {
if (!file.exists()) {
return null;
}
try {
return gson.fromJson(Files.toString(file, StandardCharsets.UTF_8), LIMBO_MAP_TYPE);
} catch (IOException e) {
ConsoleLogger.logException("Failed reading '" + file + "':", e);
}
return null;
}
private File getPlayerSegmentFile(String uuid) {
String segment = segmentNameBuilder.createSegmentName(uuid);
return new File(cacheFolder, segment + "-limbo.json");
}
@Override
public LimboPersistenceType getType() {
return LimboPersistenceType.SINGLE_FILE;
}
}

View File

@ -0,0 +1,60 @@
package fr.xephi.authme.data.limbo.persistence;
import java.util.HashMap;
import java.util.Map;
/**
* Creates segment names for {@link SegmentFilesPersistenceHolder}.
*/
class SegmentNameBuilder {
private final int length;
private final int distribution;
private final String prefix;
private final Map<Character, Character> charToSegmentChar;
/**
* Constructor.
*
* @param partition the segment configuration
*/
SegmentNameBuilder(SegmentConfiguration partition) {
this.length = partition.getLength();
this.distribution = partition.getDistribution();
this.prefix = "seg" + partition.getTotalSegments() + "-";
this.charToSegmentChar = buildCharMap(distribution);
}
String createSegmentName(String uuid) {
if (distribution == 16) {
return prefix + uuid.substring(0, length);
} else {
return prefix + createSegmentName(uuid.substring(0, length).toCharArray());
}
}
private String createSegmentName(char[] chars) {
if (chars.length == 1) {
return String.valueOf(charToSegmentChar.get(chars[0]));
}
StringBuilder sb = new StringBuilder(chars.length);
for (char chr : chars) {
sb.append(charToSegmentChar.get(chr));
}
return sb.toString();
}
private static Map<Character, Character> buildCharMap(int distribution) {
final char[] hexChars = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
final int divisor = 16 / distribution;
Map<Character, Character> charToSegmentChar = new HashMap<>();
for (int i = 0; i < hexChars.length; ++i) {
int mappedChar = i / divisor;
charToSegmentChar.put(hexChars[i], hexChars[mappedChar]);
}
return charToSegmentChar;
}
}

View File

@ -8,6 +8,7 @@ import com.google.common.collect.ImmutableMap;
import fr.xephi.authme.data.limbo.AllowFlightRestoreType;
import fr.xephi.authme.data.limbo.WalkFlySpeedRestoreType;
import fr.xephi.authme.data.limbo.persistence.LimboPersistenceType;
import fr.xephi.authme.data.limbo.persistence.SegmentConfiguration;
import java.util.Map;
@ -23,11 +24,28 @@ public final class LimboSettings implements SettingsHolder {
"on disk. This is useful in case of a server crash, so next time the server starts we can",
"properly restore things like OP status, ability to fly, and walk/fly speed.",
"DISABLED: no disk storage, INDIVIDUAL_FILES: each player data in its own file,",
"SINGLE_FILE: all data in one single file (only if you have a small server!)"
"SINGLE_FILE: all data in one single file (only if you have a small server!)",
"SEGMENT_FILES: distributes players into different buckets based on their UUID. See below."
})
public static final Property<LimboPersistenceType> LIMBO_PERSISTENCE_TYPE =
newProperty(LimboPersistenceType.class, "limbo.persistence.type", LimboPersistenceType.INDIVIDUAL_FILES);
@Comment({
"This setting only affects SEGMENT_FILES persistence. The segment file",
"persistence attempts to reduce the number of files by distributing players into various",
"buckets based on their UUID. This setting defines into how many files the players should",
"be distributed. Possible values: ONE, TWO, FOUR, EIGHT, SIXTEEN, THIRTY_TWO, SIXTY_FOUR,",
"ONE_TWENTY for 128, TWO_FIFTY for 256.",
"For example, if you expect 100 non-logged in players, setting to SIXTEEN will average",
"6.25 players per file (100 / 16). If you set to ONE, like persistence SINGLE_FILE, only",
"one file will be used. Contrary to SINGLE_FILE, it won't keep the entries in cache, which",
"may deliver different results in terms of performance.",
"Note: if you change this setting you lose all stored LimboPlayer data because the",
"distribution of players will be different."
})
public static final Property<SegmentConfiguration> SEGMENT_DISTRIBUTION =
newProperty(SegmentConfiguration.class, "limbo.persistence.segmentDistribution", SegmentConfiguration.SIXTEEN);
@Comment({
"Whether the player is allowed to fly: RESTORE, ENABLE, DISABLE.",
"RESTORE sets back the old property from the player."

View File

@ -0,0 +1,47 @@
package fr.xephi.authme.data.limbo.persistence;
import com.google.common.collect.ImmutableSet;
import org.junit.Test;
import java.util.HashSet;
import java.util.Set;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.fail;
/**
* Test for {@link SegmentConfiguration}.
*/
public class SegmentConfigurationTest {
@Test
public void shouldHaveDistributionThatIsPowerOf2() {
// given
Set<Integer> allowedDistributions = ImmutableSet.of(1, 2, 4, 8, 16);
// when / then
for (SegmentConfiguration entry : SegmentConfiguration.values()) {
if (!allowedDistributions.contains(entry.getDistribution())) {
fail("Distribution must be a power of 2 and within [1, 16]. Offending item: " + entry);
}
}
}
@Test
public void shouldHaveDifferentSegmentSizes() {
// given
Set<Integer> sizes = new HashSet<>();
// when / then
for (SegmentConfiguration entry : SegmentConfiguration.values()) {
int segSize = (int) Math.pow(entry.getDistribution(), entry.getLength());
assertThat(entry + " must have a positive segment size",
segSize, greaterThan(0));
assertThat(entry + " has a segment size that was already encountered (" + segSize + ")",
sizes.add(segSize), equalTo(true));
}
}
}

View File

@ -0,0 +1,153 @@
package fr.xephi.authme.data.limbo.persistence;
import org.junit.Test;
import java.util.HashSet;
import java.util.Set;
import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.EIGHT;
import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.FOUR;
import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.ONE;
import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.SIXTEEN;
import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.SIXTY_FOUR;
import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.THIRTY_TWO;
import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.TWO;
import static fr.xephi.authme.data.limbo.persistence.SegmentConfiguration.TWO_FIFTY;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.assertThat;
/**
* Test for {@link SegmentNameBuilder}.
*/
public class SegmentNameBuilderTest {
/**
* Checks that using a given segment size really produces as many segments as defined.
* E.g. if we partition with {@link SegmentConfiguration#EIGHT} we expect eight different buckets.
*/
@Test
public void shouldCreatePromisedSizeOfSegments() {
for (SegmentConfiguration part : SegmentConfiguration.values()) {
// Perform this check only for `length` <= 5 because the test creates all hex numbers with `length` digits.
if (part.getLength() <= 5) {
checkTotalSegmentsProduced(part);
}
}
}
private void checkTotalSegmentsProduced(SegmentConfiguration part) {
// given
SegmentNameBuilder nameBuilder = new SegmentNameBuilder(part);
Set<String> encounteredSegments = new HashSet<>();
int shift = part.getLength() * 4;
// e.g. (1 << 16) - 1 = 0xFFFF. (Number of digits = shift/4, since 16 = 2^4)
int max = (1 << shift) - 1;
// when
for (int i = 0; i <= max; ++i) {
String uuid = toPaddedHex(i, part.getLength());
encounteredSegments.add(nameBuilder.createSegmentName(uuid));
}
// then
assertThat(encounteredSegments, hasSize(part.getTotalSegments()));
}
private static String toPaddedHex(int dec, int padLength) {
String hexResult = Integer.toString(dec, 16);
while (hexResult.length() < padLength) {
hexResult = "0" + hexResult;
}
return hexResult;
}
@Test
public void shouldCreateOneSegment() {
// given
SegmentNameBuilder nameBuilder = new SegmentNameBuilder(ONE);
// when / then
assertThat(nameBuilder.createSegmentName("abc"), equalTo("seg1-0"));
assertThat(nameBuilder.createSegmentName("f0e"), equalTo("seg1-0"));
assertThat(nameBuilder.createSegmentName("329"), equalTo("seg1-0"));
}
@Test
public void shouldCreateTwoSegments() {
// given
SegmentNameBuilder nameBuilder = new SegmentNameBuilder(TWO);
// when / then
assertThat(nameBuilder.createSegmentName("f6c"), equalTo("seg2-1"));
assertThat(nameBuilder.createSegmentName("29f"), equalTo("seg2-0"));
assertThat(nameBuilder.createSegmentName("983"), equalTo("seg2-1"));
}
@Test
public void shouldCreateFourSegments() {
// given
SegmentNameBuilder nameBuilder = new SegmentNameBuilder(FOUR);
// when / then
assertThat(nameBuilder.createSegmentName("f9cc"), equalTo("seg4-3"));
assertThat(nameBuilder.createSegmentName("84c9"), equalTo("seg4-2"));
assertThat(nameBuilder.createSegmentName("3799"), equalTo("seg4-0"));
}
@Test
public void shouldCreateEightSegments() {
// given
SegmentNameBuilder nameBuilder = new SegmentNameBuilder(EIGHT);
// when / then
assertThat(nameBuilder.createSegmentName("fc9c"), equalTo("seg8-7"));
assertThat(nameBuilder.createSegmentName("90ad"), equalTo("seg8-4"));
assertThat(nameBuilder.createSegmentName("35e4"), equalTo("seg8-1"));
assertThat(nameBuilder.createSegmentName("a39f"), equalTo("seg8-5"));
}
@Test
public void shouldCreateSixteenSegments() {
// given
SegmentNameBuilder nameBuilder = new SegmentNameBuilder(SIXTEEN);
// when / then
assertThat(nameBuilder.createSegmentName("fc9a054"), equalTo("seg16-f"));
assertThat(nameBuilder.createSegmentName("b0a945e"), equalTo("seg16-b"));
assertThat(nameBuilder.createSegmentName("7afebab"), equalTo("seg16-7"));
}
@Test
public void shouldCreateThirtyTwoSegments() {
// given
SegmentNameBuilder nameBuilder = new SegmentNameBuilder(THIRTY_TWO);
// when / then
assertThat(nameBuilder.createSegmentName("f890c9"), equalTo("seg32-11101"));
assertThat(nameBuilder.createSegmentName("49c39a"), equalTo("seg32-01101"));
assertThat(nameBuilder.createSegmentName("b75d09"), equalTo("seg32-10010"));
}
@Test
public void shouldCreateSixtyFourSegments() {
// given
SegmentNameBuilder nameBuilder = new SegmentNameBuilder(SIXTY_FOUR);
// when / then
assertThat(nameBuilder.createSegmentName("82f"), equalTo("seg64-203"));
assertThat(nameBuilder.createSegmentName("9b4"), equalTo("seg64-221"));
assertThat(nameBuilder.createSegmentName("068"), equalTo("seg64-012"));
}
@Test
public void shouldCreate256Segments() {
// given
SegmentNameBuilder nameBuilder = new SegmentNameBuilder(TWO_FIFTY);
// when / then
assertThat(nameBuilder.createSegmentName("a813c"), equalTo("seg256-a8"));
assertThat(nameBuilder.createSegmentName("b4d01"), equalTo("seg256-b4"));
assertThat(nameBuilder.createSegmentName("7122f"), equalTo("seg256-71"));
}
}