Meishin/LoginSystem
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#1125 Limbo persistence: convert old segments, add tests

Browse Source 
- On startup / reload the playerdata folder is scanned for old segment files, whose data is migrated before they are deleted
- Add tests for segment files persistence holder
This commit is contained in:
ljacqu
2017-03-14 20:46:28 +01:00
parent 9c3baa7f14
commit 84acc4557a
16 changed files with 574 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/fr/xephi/authme/data/limbo/persistence/LimboPersistence.java
+4 -7
View File
@@ -70,13 +70,10 @@ public class LimboPersistence implements SettingsDependent {
@Override
public void reload(Settings settings) {
LimboPersistenceType persistenceType = settings.getProperty(LimboSettings.LIMBO_PERSISTENCE_TYPE);
if (handler == null || handler.getType() != persistenceType) {
// If we're changing from an existing handler, output a quick hint that nothing is converted.
if (handler != null) {
ConsoleLogger.info("Limbo persistence type has changed! Note that the data is not converted.");
}
handler = handlerFactory.newInstance(persistenceType.getImplementationClass());
// If we're changing from an existing handler, output a quick hint that nothing is converted.
if (handler != null && handler.getType() != persistenceType) {
ConsoleLogger.info("Limbo persistence type has changed! Note that the data is not converted.");
}
handler = handlerFactory.newInstance(persistenceType.getImplementationClass());
}
}
src/main/java/fr/xephi/authme/data/limbo/persistence/LimboPersistenceType.java
+4
View File
@@ -5,12 +5,16 @@ package fr.xephi.authme.data.limbo.persistence;
*/
public enum LimboPersistenceType {
/** Store each LimboPlayer in a separate file. */
INDIVIDUAL_FILES(SeparateFilePersistenceHandler.class),
/** Store all LimboPlayers in the same file. */
SINGLE_FILE(SingleFilePersistenceHandler.class),
/** Distribute LimboPlayers by segments into a set number of files. */
SEGMENT_FILES(SegmentFilesPersistenceHolder.class),
/** No persistence to disk. */
DISABLED(NoOpPersistenceHandler.class);
private final Class<? extends LimboPersistenceHandler> implementationClass;
src/main/java/fr/xephi/authme/data/limbo/persistence/SegmentConfiguration.java
+2 -2
View File
@@ -38,8 +38,8 @@ public enum SegmentConfiguration {
/** 1. */
ONE(1, 1),
/** 2. */
TWO(2, 1),
///** 2. */
//TWO(2, 1),
/** 4. */
FOUR(4, 1),
src/main/java/fr/xephi/authme/data/limbo/persistence/SegmentFilesPersistenceHolder.java
+109 -15
View File
@@ -17,11 +17,14 @@ import org.bukkit.entity.Player;
import javax.inject.Inject;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.reflect.Type;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
/**
* Persistence handler for LimboPlayer objects by distributing the objects to store
@@ -51,7 +54,8 @@ class SegmentFilesPersistenceHolder implements LimboPersistenceHandler {
segmentNameBuilder = new SegmentNameBuilder(settings.getProperty(LimboSettings.SEGMENT_DISTRIBUTION));
// TODO #1125: Check for other segment files and attempt to convert?
convertOldDataToCurrentSegmentScheme();
deleteEmptyFiles();
}
@Override
@@ -94,16 +98,16 @@ class SegmentFilesPersistenceHolder implements LimboPersistenceHandler {
}
}
@Override
public LimboPersistenceType getType() {
return LimboPersistenceType.SEGMENT_FILES;
}
private void saveEntries(Map<String, LimboPlayer> entries, File file) {
if (entries.isEmpty()) {
// TODO #1125: Probably should do a sweep of empty files on startup / shutdown, but not all the time
FileUtils.delete(file);
} else {
try (FileWriter fw = new FileWriter(file)) {
gson.toJson(entries, fw);
} catch (IOException e) {
ConsoleLogger.logException("Could not write to '" + file + "':", e);
}
try (FileWriter fw = new FileWriter(file)) {
gson.toJson(entries, fw);
} catch (Exception e) {
ConsoleLogger.logException("Could not write to '" + file + "':", e);
}
}
@@ -114,7 +118,7 @@ class SegmentFilesPersistenceHolder implements LimboPersistenceHandler {
try {
return gson.fromJson(Files.toString(file, StandardCharsets.UTF_8), LIMBO_MAP_TYPE);
} catch (IOException e) {
} catch (Exception e) {
ConsoleLogger.logException("Failed reading '" + file + "':", e);
}
return null;
@@ -125,8 +129,98 @@ class SegmentFilesPersistenceHolder implements LimboPersistenceHandler {
return new File(cacheFolder, segment + "-limbo.json");
}
@Override
public LimboPersistenceType getType() {
return LimboPersistenceType.SINGLE_FILE;
/**
* Loads segment files in the cache folder that don't correspond to the current segmenting scheme
* and migrates the data into files of the current segments. This allows a player to change the
* segment size without any loss of data.
*/
private void convertOldDataToCurrentSegmentScheme() {
String currentPrefix = segmentNameBuilder.getPrefix();
File[] files = listFiles(cacheFolder);
Map<String, LimboPlayer> allLimboPlayers = new HashMap<>();
List<File> migratedFiles = new ArrayList<>();
for (File file : files) {
if (isLimboJsonFile(file) && !file.getName().startsWith(currentPrefix)) {
Map<String, LimboPlayer> data = readLimboPlayers(file);
if (data != null) {
allLimboPlayers.putAll(data);
migratedFiles.add(file);
}
}
}
if (!allLimboPlayers.isEmpty()) {
saveToNewSegments(allLimboPlayers);
migratedFiles.forEach(FileUtils::delete);
}
}
/**
* Saves the LimboPlayer data read from old segmenting schemes into the current segmenting scheme.
*
* @param limbosFromOldSegments the limbo players to store into the current segment files
*/
private void saveToNewSegments(Map<String, LimboPlayer> limbosFromOldSegments) {
Map<String, Map<String, LimboPlayer>> limboBySegment = groupBySegment(limbosFromOldSegments);
ConsoleLogger.info("Saving " + limbosFromOldSegments.size() + " LimboPlayers from old segments into "
+ limboBySegment.size() + " current segments");
for (Map.Entry<String, Map<String, LimboPlayer>> entry : limboBySegment.entrySet()) {
File file = new File(cacheFolder, entry.getKey() + "-limbo.json");
Map<String, LimboPlayer> limbosToSave = Optional.ofNullable(readLimboPlayers(file))
.orElseGet(HashMap::new);
limbosToSave.putAll(entry.getValue());
saveEntries(limbosToSave, file);
}
}
/**
* Converts a Map of UUID to LimboPlayers to a 2-dimensional Map of LimboPlayers by segment ID and UUID.
* {@code Map(uuid -> LimboPlayer) to Map(segment -> Map(uuid -> LimboPlayer))}
*
* @param readLimboPlayers the limbo players to order by segment
* @return limbo players ordered by segment ID and associated player UUID
*/
private Map<String, Map<String, LimboPlayer>> groupBySegment(Map<String, LimboPlayer> readLimboPlayers) {
Map<String, Map<String, LimboPlayer>> limboBySegment = new HashMap<>();
for (Map.Entry<String, LimboPlayer> entry : readLimboPlayers.entrySet()) {
String segmentId = segmentNameBuilder.createSegmentName(entry.getKey());
limboBySegment.computeIfAbsent(segmentId, s -> new HashMap<>())
.put(entry.getKey(), entry.getValue());
}
return limboBySegment;
}
/**
* Deletes files from the current segmenting scheme that are empty.
*/
private void deleteEmptyFiles() {
File[] files = listFiles(cacheFolder);
long deletedFiles = Arrays.stream(files)
// typically the size is 2 because there's an empty JSON map: {}
.filter(f -> isLimboJsonFile(f) && f.length() < 3)
.peek(FileUtils::delete)
.count();
ConsoleLogger.debug("Limbo: Deleted {0} empty segment files", deletedFiles);
}
/**
* @param file the file to check
* @return true if it is a segment file storing Limbo data, false otherwise
*/
private static boolean isLimboJsonFile(File file) {
String name = file.getName();
return name.startsWith("seg") && name.endsWith("-limbo.json");
}
private static File[] listFiles(File folder) {
File[] files = folder.listFiles();
if (files == null) {
ConsoleLogger.warning("Could not get files of '" + folder + "'");
return new File[0];
}
return files;
}
}
src/main/java/fr/xephi/authme/data/limbo/persistence/SegmentNameBuilder.java
+15 -2
View File
@@ -25,15 +25,28 @@ class SegmentNameBuilder {
this.charToSegmentChar = buildCharMap(distribution);
}
/**
* Returns the segment ID for the given UUID.
*
* @param uuid the player's uuid to get the segment for
* @return id the uuid belongs to
*/
String createSegmentName(String uuid) {
if (distribution == 16) {
return prefix + uuid.substring(0, length);
} else {
return prefix + createSegmentName(uuid.substring(0, length).toCharArray());
return prefix + buildSegmentName(uuid.substring(0, length).toCharArray());
}
}
private String createSegmentName(char[] chars) {
/**
* @return the prefix used for the current segment configuration
*/
String getPrefix() {
return prefix;
}
private String buildSegmentName(char[] chars) {
if (chars.length == 1) {
return String.valueOf(charToSegmentChar.get(chars[0]));
}
src/main/java/fr/xephi/authme/settings/properties/LimboSettings.java
+3 -3
View File
@@ -34,14 +34,14 @@ public final class LimboSettings implements SettingsHolder {
"This setting only affects SEGMENT_FILES persistence. The segment file",
"persistence attempts to reduce the number of files by distributing players into various",
"buckets based on their UUID. This setting defines into how many files the players should",
"be distributed. Possible values: ONE, TWO, FOUR, EIGHT, SIXTEEN, THIRTY_TWO, SIXTY_FOUR,",
"be distributed. Possible values: ONE, FOUR, EIGHT, SIXTEEN, THIRTY_TWO, SIXTY_FOUR,",
"ONE_TWENTY for 128, TWO_FIFTY for 256.",
"For example, if you expect 100 non-logged in players, setting to SIXTEEN will average",
"6.25 players per file (100 / 16). If you set to ONE, like persistence SINGLE_FILE, only",
"one file will be used. Contrary to SINGLE_FILE, it won't keep the entries in cache, which",
"may deliver different results in terms of performance.",
"Note: if you change this setting you lose all stored LimboPlayer data because the",
"distribution of players will be different."
"Note: if you change this setting all data will be migrated. If you have a lot of data,",
"change this setting only on server restart, not with /authme reload."
})
public static final Property<SegmentConfiguration> SEGMENT_DISTRIBUTION =
newProperty(SegmentConfiguration.class, "limbo.persistence.segmentDistribution", SegmentConfiguration.SIXTEEN);