major refactor of the purging process

This commit is contained in:
Gnat008 2016-06-16 12:28:42 -04:00
parent 3a102c324e
commit af1520802d
10 changed files with 499 additions and 89 deletions

View File

@ -36,6 +36,7 @@ import fr.xephi.authme.output.Messages;
import fr.xephi.authme.permission.PermissionsManager;
import fr.xephi.authme.permission.PermissionsSystemType;
import fr.xephi.authme.process.Management;
import fr.xephi.authme.process.purge.PurgeService;
import fr.xephi.authme.security.PasswordSecurity;
import fr.xephi.authme.security.crypts.SHA256;
import fr.xephi.authme.settings.NewSetting;
@ -133,6 +134,7 @@ public class AuthMe extends JavaPlugin {
private boolean autoPurging;
private BukkitService bukkitService;
private AuthMeServiceInitializer initializer;
private PurgeService purgeService;
/**
* Get the plugin's instance.
@ -255,6 +257,7 @@ public class AuthMe extends JavaPlugin {
api = initializer.get(NewAPI.class);
management = initializer.get(Management.class);
dataManager = initializer.get(DataManager.class);
purgeService = initializer.get(PurgeService.class);
initializer.get(API.class);
// Set up Metrics
@ -310,7 +313,7 @@ public class AuthMe extends JavaPlugin {
}
// Purge on start if enabled
runAutoPurge();
purgeService.runAutoPurge();
}
/**
@ -643,29 +646,6 @@ public class AuthMe extends JavaPlugin {
return pluginHooks != null && pluginHooks.isNpc(player) || player.hasMetadata("NPC");
}
// Purge inactive players from the database, as defined in the configuration
private void runAutoPurge() {
if (!newSettings.getProperty(PurgeSettings.USE_AUTO_PURGE) || autoPurging) {
return;
}
autoPurging = true;
ConsoleLogger.info("AutoPurging the Database...");
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DATE, -newSettings.getProperty(PurgeSettings.DAYS_BEFORE_REMOVE_PLAYER));
long until = calendar.getTimeInMillis();
Set<String> cleared = database.autoPurgeDatabase(until);
if (CollectionUtils.isEmpty(cleared)) {
return;
}
ConsoleLogger.info("AutoPurging the Database: " + cleared.size() + " accounts removed!");
ConsoleLogger.info("Purging user accounts...");
new PurgeTask(plugin, Bukkit.getConsoleSender(), cleared, true, Bukkit.getOfflinePlayers())
.runTaskTimer(plugin, 0, 1);
}
// Return the spawn location of a player
@Deprecated
public Location getSpawnLocation(Player player) {

View File

@ -3,6 +3,7 @@ package fr.xephi.authme.command.executable.authme;
import fr.xephi.authme.AuthMe;
import fr.xephi.authme.command.ExecutableCommand;
import fr.xephi.authme.datasource.DataSource;
import fr.xephi.authme.process.purge.PurgeService;
import fr.xephi.authme.task.PurgeTask;
import fr.xephi.authme.util.BukkitService;
import org.bukkit.ChatColor;
@ -21,10 +22,7 @@ import java.util.Set;
public class PurgeBannedPlayersCommand implements ExecutableCommand {
@Inject
private DataSource dataSource;
@Inject
private AuthMe plugin;
private PurgeService purgeService;
@Inject
private BukkitService bukkitService;
@ -38,12 +36,6 @@ public class PurgeBannedPlayersCommand implements ExecutableCommand {
namedBanned.add(offlinePlayer.getName().toLowerCase());
}
//todo: note this should may run async because it may executes a SQL-Query
// Purge the banned players
dataSource.purgeBanned(namedBanned);
// Show a status message
sender.sendMessage(ChatColor.GOLD + "Purging user accounts...");
new PurgeTask(plugin, sender, namedBanned, bannedPlayers).runTaskTimer(plugin, 0, 1);
purgeService.purgeBanned(sender, namedBanned, bannedPlayers);
}
}

View File

@ -3,6 +3,7 @@ package fr.xephi.authme.command.executable.authme;
import fr.xephi.authme.AuthMe;
import fr.xephi.authme.command.ExecutableCommand;
import fr.xephi.authme.datasource.DataSource;
import fr.xephi.authme.process.purge.PurgeService;
import fr.xephi.authme.task.PurgeTask;
import org.bukkit.ChatColor;
import org.bukkit.command.CommandSender;
@ -21,7 +22,7 @@ public class PurgeCommand implements ExecutableCommand {
private static final int MINIMUM_LAST_SEEN_DAYS = 30;
@Inject
private DataSource dataSource;
private PurgeService purgeService;
@Inject
private AuthMe plugin;
@ -52,13 +53,7 @@ public class PurgeCommand implements ExecutableCommand {
calendar.add(Calendar.DATE, -days);
long until = calendar.getTimeInMillis();
//todo: note this should may run async because it may executes a SQL-Query
// Purge the data, get the purged values
Set<String> purged = dataSource.autoPurgeDatabase(until);
// Show a status message
sender.sendMessage(ChatColor.GOLD + "Deleted " + purged.size() + " user accounts");
sender.sendMessage(ChatColor.GOLD + "Purging user accounts...");
new PurgeTask(plugin, sender, purged).runTaskTimer(plugin, 0, 1);
// Run the purge
purgeService.runPurge(sender, until);
}
}

View File

@ -148,6 +148,18 @@ public class CacheDataSource implements DataSource {
return cleared;
}
@Override
public Set<String> getRecordsToPurge(long until) {
return source.getRecordsToPurge(until);
}
@Override
public void purgeRecords(Set<String> toPurge) {
for (String name : toPurge) {
cachedAuths.invalidate(name);
}
}
@Override
public boolean removeAuth(String name) {
name = name.toLowerCase();

View File

@ -71,13 +71,28 @@ public interface DataSource extends Reloadable {
/**
* Purge all records in the database whose last login was longer ago than
* the given time.
* the given time if they do not have the bypass permission.
*
* @param until The minimum last login
* @return The account names that have been removed
*/
Set<String> autoPurgeDatabase(long until);
/**
* Get all records in the database whose last login was before the given time.
*
* @param until The minimum last login
* @return The account names selected to purge
*/
Set<String> getRecordsToPurge(long until);
/**
* Purge the given players from the database.
*
* @param toPurge The players to purge
*/
void purgeRecords(Set<String> toPurge);
/**
* Remove a user record from the database.
*

View File

@ -8,6 +8,7 @@ import fr.xephi.authme.cache.auth.PlayerCache;
import fr.xephi.authme.security.crypts.HashedPassword;
import fr.xephi.authme.settings.Settings;
import javax.swing.*;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.Closeable;
@ -262,6 +263,65 @@ public class FlatFile implements DataSource {
return cleared;
}
@Override
public Set<String> getRecordsToPurge(long until) {
BufferedReader br = null;
Set<String> list = new HashSet<>();
try {
br = new BufferedReader(new FileReader(source));
String line;
while ((line = br.readLine()) != null) {
String[] args = line.split(":");
if (args.length >= 4) {
if (Long.parseLong(args[3]) >= until) {
list.add(args[0]);
continue;
}
}
}
} catch (IOException ex) {
ConsoleLogger.showError(ex.getMessage());
return list;
} finally {
silentClose(br);
}
return list;
}
@Override
public void purgeRecords(Set<String> toPurge) {
BufferedReader br = null;
BufferedWriter bw = null;
ArrayList<String> lines = new ArrayList<>();
try {
br = new BufferedReader(new FileReader(source));
bw = new BufferedWriter(new FileWriter(source));
String line;
while ((line = br.readLine()) != null) {
String[] args = line.split(":");
if (args.length >= 4) {
if (toPurge.contains(args[0])) {
lines.add(line);
continue;
}
}
}
for (String l : lines) {
bw.write(l + "\n");
}
} catch (IOException ex) {
ConsoleLogger.showError(ex.getMessage());
return;
} finally {
silentClose(br);
silentClose(bw);
}
}
@Override
public synchronized boolean removeAuth(String user) {
if (!isAuthAvailable(user)) {

View File

@ -635,6 +635,39 @@ public class MySQL implements DataSource {
return list;
}
@Override
public Set<String> getRecordsToPurge(long until) {
Set<String> list = new HashSet<>();
String select = "SELECT " + col.NAME + " FROM " + tableName + " WHERE " + col.LAST_LOGIN + "<?;";
try (Connection con = getConnection();
PreparedStatement selectPst = con.prepareStatement(select)) {
selectPst.setLong(1, until);
try (ResultSet rs = selectPst.executeQuery()) {
while (rs.next()) {
list.add(rs.getString(col.NAME));
}
}
} catch (SQLException ex) {
logSqlException(ex);
}
return list;
}
@Override
public void purgeRecords(Set<String> toPurge) {
String delete = "DELETE FROM " + tableName + " WHERE " + col.NAME + "=?;";
try (Connection con = getConnection(); PreparedStatement deletePst = con.prepareStatement(delete)) {
for (String name : toPurge) {
deletePst.setString(1, name);
deletePst.executeUpdate();
}
} catch (SQLException ex) {
logSqlException(ex);
}
}
@Override
public boolean removeAuth(String user) {
user = user.toLowerCase();

View File

@ -314,6 +314,38 @@ public class SQLite implements DataSource {
return list;
}
@Override
public Set<String> getRecordsToPurge(long until) {
Set<String> list = new HashSet<>();
String select = "SELECT " + col.NAME + " FROM " + tableName + " WHERE " + col.LAST_LOGIN + "<?;";
try (PreparedStatement selectPst = con.prepareStatement(select)) {
selectPst.setLong(1, until);
try (ResultSet rs = selectPst.executeQuery()) {
while (rs.next()) {
list.add(rs.getString(col.NAME));
}
}
} catch (SQLException ex) {
logSqlException(ex);
}
return list;
}
@Override
public void purgeRecords(Set<String> toPurge) {
String delete = "DELETE FROM " + tableName + " WHERE " + col.NAME + "=?;";
for (String name : toPurge) {
try (PreparedStatement deletePst = con.prepareStatement(delete)) {
deletePst.setString(1, name);
deletePst.executeUpdate();
} catch (SQLException ex) {
logSqlException(ex);
}
}
}
@Override
public boolean removeAuth(String user) {
PreparedStatement pst = null;

View File

@ -0,0 +1,310 @@
package fr.xephi.authme.process.purge;
import fr.xephi.authme.ConsoleLogger;
import fr.xephi.authme.datasource.DataSource;
import fr.xephi.authme.hooks.PluginHooks;
import fr.xephi.authme.initialization.Reloadable;
import fr.xephi.authme.permission.PermissionsManager;
import fr.xephi.authme.permission.PlayerStatePermission;
import fr.xephi.authme.settings.NewSetting;
import fr.xephi.authme.settings.properties.PurgeSettings;
import fr.xephi.authme.task.PurgeTask;
import fr.xephi.authme.util.BukkitService;
import fr.xephi.authme.util.CollectionUtils;
import fr.xephi.authme.util.Utils;
import org.bukkit.Bukkit;
import org.bukkit.ChatColor;
import org.bukkit.OfflinePlayer;
import org.bukkit.Server;
import org.bukkit.command.CommandSender;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import java.io.File;
import java.util.Calendar;
import java.util.HashSet;
import java.util.Set;
import static fr.xephi.authme.util.StringUtils.makePath;
public class PurgeService implements Reloadable {
@Inject
private BukkitService bukkitService;
@Inject
private DataSource dataSource;
@Inject
private NewSetting settings;
@Inject
private PermissionsManager permissionsManager;
@Inject
private PluginHooks pluginHooks;
@Inject
private Server server;
private boolean autoPurging = false;
// Settings
private boolean useAutoPurge;
private boolean removeEssentialsFiles;
private boolean removePlayerDat;
private boolean removeLimitedCreativeInventories;
private boolean removeAntiXrayFiles;
private boolean removePermissions;
private int daysBeforePurge;
/**
* Return whether an automatic purge is in progress.
*
* @return True if purging.
*/
public boolean isAutoPurging() {
return this.autoPurging;
}
/**
* Set if an automatic purge is currently in progress.
*
* @param autoPurging True if automatically purging.
*/
public void setAutoPurging(boolean autoPurging) {
this.autoPurging = autoPurging;
}
/**
* Purges players from the database. Ran on startup.
*/
public void runAutoPurge() {
if (!useAutoPurge || autoPurging) {
return;
}
this.autoPurging = true;
// Get the initial list of players to purge
ConsoleLogger.info("Automatically purging the database...");
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DATE, daysBeforePurge);
long until = calendar.getTimeInMillis();
Set<String> initialPurge = dataSource.getRecordsToPurge(until);
if (CollectionUtils.isEmpty(initialPurge)) {
return;
}
// Remove players from the purge list if they have bypass permission
Set<String> toPurge = getFinalPurgeList(initialPurge);
// Purge players from the database
dataSource.purgeRecords(toPurge);
ConsoleLogger.info("Purged the database: " + toPurge.size() + " accounts removed!");
ConsoleLogger.info("Purging user accounts...");
// Schedule a PurgeTask
PurgeTask purgeTask = new PurgeTask(this, Bukkit.getConsoleSender(), toPurge, Bukkit.getOfflinePlayers(), true);
bukkitService.runTaskAsynchronously(purgeTask);
}
/**
* Run a purge with a specified time.
*
* @param sender Sender running the command.
* @param until The minimum last login.
*/
public void runPurge(CommandSender sender, long until) {
//todo: note this should may run async because it may executes a SQL-Query
Set<String> initialPurge = dataSource.getRecordsToPurge(until);
if (CollectionUtils.isEmpty(initialPurge)) {
return;
}
Set<String> toPurge = getFinalPurgeList(initialPurge);
// Purge records from the database
dataSource.purgeRecords(toPurge);
sender.sendMessage(ChatColor.GOLD + "Deleted " + toPurge.size() + " user accounts");
sender.sendMessage(ChatColor.GOLD + "Purging user accounts...");
// Schedule a PurgeTask
PurgeTask purgeTask = new PurgeTask(this, sender, toPurge, Bukkit.getOfflinePlayers(), false);
bukkitService.runTaskAsynchronously(purgeTask);
}
public void purgeBanned(CommandSender sender, Set<String> bannedNames, Set<OfflinePlayer> bannedPlayers) {
//todo: note this should may run async because it may executes a SQL-Query
dataSource.purgeBanned(bannedNames);
OfflinePlayer[] bannedPlayersArray = new OfflinePlayer[bannedPlayers.size()];
bannedPlayers.toArray(bannedPlayersArray);
PurgeTask purgeTask = new PurgeTask(this, sender, bannedNames, bannedPlayersArray, false);
bukkitService.runTaskAsynchronously(purgeTask);
}
/**
* Check each name in the initial purge findings to remove any player from the purge list
* that has the bypass permission.
*
* @param initial The initial list of players to purge.
*
* @return The list of players to purge after permission check.
*/
private Set<String> getFinalPurgeList(Set<String> initial) {
Set<String> toPurge = new HashSet<>();
for (String name : initial) {
if (!permissionsManager.hasPermission(name, PlayerStatePermission.BYPASS_PURGE)) {
toPurge.add(name);
}
}
return toPurge;
}
public synchronized void purgeAntiXray(Set<String> cleared) {
if (!removeAntiXrayFiles) {
return;
}
int i = 0;
File dataFolder = new File("." + File.separator + "plugins" + File.separator + "AntiXRayData"
+ File.separator + "PlayerData");
if (!dataFolder.exists() || !dataFolder.isDirectory()) {
return;
}
for (String file : dataFolder.list()) {
if (cleared.contains(file.toLowerCase())) {
File playerFile = new File(dataFolder, file);
if (playerFile.exists() && playerFile.delete()) {
i++;
}
}
}
ConsoleLogger.info("AutoPurge: Removed " + i + " AntiXRayData Files");
}
public synchronized void purgeLimitedCreative(Set<String> cleared) {
if (!removeLimitedCreativeInventories) {
return;
}
int i = 0;
File dataFolder = new File("." + File.separator + "plugins" + File.separator + "LimitedCreative"
+ File.separator + "inventories");
if (!dataFolder.exists() || !dataFolder.isDirectory()) {
return;
}
for (String file : dataFolder.list()) {
String name = file;
int idx;
idx = file.lastIndexOf("_creative.yml");
if (idx != -1) {
name = name.substring(0, idx);
} else {
idx = file.lastIndexOf("_adventure.yml");
if (idx != -1) {
name = name.substring(0, idx);
} else {
idx = file.lastIndexOf(".yml");
if (idx != -1) {
name = name.substring(0, idx);
}
}
}
if (name.equals(file)) {
continue;
}
if (cleared.contains(name.toLowerCase())) {
File dataFile = new File(dataFolder, file);
if (dataFile.exists() && dataFile.delete()) {
i++;
}
}
}
ConsoleLogger.info("AutoPurge: Removed " + i + " LimitedCreative Survival, Creative and Adventure files");
}
public synchronized void purgeDat(Set<OfflinePlayer> cleared) {
if (!removePlayerDat) {
return;
}
int i = 0;
File dataFolder = new File(server.getWorldContainer()
, makePath(settings.getProperty(PurgeSettings.DEFAULT_WORLD), "players"));
for (OfflinePlayer offlinePlayer : cleared) {
File playerFile = new File(dataFolder, Utils.getUUIDorName(offlinePlayer) + ".dat");
if (playerFile.delete()) {
i++;
}
}
ConsoleLogger.info("AutoPurge: Removed " + i + " .dat Files");
}
/**
* Method purgeEssentials.
*
* @param cleared List of String
*/
public synchronized void purgeEssentials(Set<OfflinePlayer> cleared) {
if (!removeEssentialsFiles && !pluginHooks.isEssentialsAvailable()) {
return;
}
int i = 0;
File essentialsDataFolder = pluginHooks.getEssentialsDataFolder();
if (essentialsDataFolder == null) {
ConsoleLogger.info("Cannot purge Essentials: plugin is not loaded");
return;
}
final File userDataFolder = new File(essentialsDataFolder, "userdata");
if (!userDataFolder.exists() || !userDataFolder.isDirectory()) {
return;
}
for (OfflinePlayer offlinePlayer : cleared) {
File playerFile = new File(userDataFolder, Utils.getUUIDorName(offlinePlayer) + ".yml");
if (playerFile.exists() && playerFile.delete()) {
i++;
}
}
ConsoleLogger.info("AutoPurge: Removed " + i + " EssentialsFiles");
}
// TODO: What is this method for? Is it correct?
// TODO: Make it work with OfflinePlayers group data.
public synchronized void purgePermissions(Set<OfflinePlayer> cleared) {
if (!removePermissions) {
return;
}
for (OfflinePlayer offlinePlayer : cleared) {
String name = offlinePlayer.getName();
permissionsManager.removeAllGroups(bukkitService.getPlayerExact(name));
}
ConsoleLogger.info("AutoPurge: Removed permissions from " + cleared.size() + " player(s).");
}
@PostConstruct
@Override
public void reload() {
this.useAutoPurge = settings.getProperty(PurgeSettings.USE_AUTO_PURGE);
this.removeEssentialsFiles = settings.getProperty(PurgeSettings.REMOVE_ESSENTIALS_FILES);
this.removePlayerDat = settings.getProperty(PurgeSettings.REMOVE_PLAYER_DAT);
this.removeAntiXrayFiles = settings.getProperty(PurgeSettings.REMOVE_ANTI_XRAY_FILE);
this.removeLimitedCreativeInventories = settings.getProperty(PurgeSettings.REMOVE_LIMITED_CREATIVE_INVENTORIES);
this.removePermissions = settings.getProperty(PurgeSettings.REMOVE_PERMISSIONS);
this.daysBeforePurge = settings.getProperty(PurgeSettings.DAYS_BEFORE_REMOVE_PLAYER);
}
}

View File

@ -1,12 +1,7 @@
package fr.xephi.authme.task;
import fr.xephi.authme.AuthMe;
import fr.xephi.authme.ConsoleLogger;
import fr.xephi.authme.permission.PermissionNode;
import fr.xephi.authme.permission.PermissionsManager;
import fr.xephi.authme.permission.PlayerPermission;
import fr.xephi.authme.settings.NewSetting;
import fr.xephi.authme.settings.properties.PurgeSettings;
import fr.xephi.authme.process.purge.PurgeService;
import java.util.HashSet;
import java.util.Set;
@ -19,16 +14,13 @@ import org.bukkit.command.CommandSender;
import org.bukkit.entity.Player;
import org.bukkit.scheduler.BukkitRunnable;
import javax.inject.Inject;
public class PurgeTask extends BukkitRunnable {
private PurgeService purgeService;
//how many players we should check for each tick
private static final int INTERVALL_CHECK = 5;
private final AuthMe plugin;
private final NewSetting newSetting;
private final UUID sender;
private final Set<String> toPurge;
@ -39,19 +31,8 @@ public class PurgeTask extends BukkitRunnable {
private int currentPage = 0;
public PurgeTask(AuthMe plugin, CommandSender sender, Set<String> purged) {
this(plugin, sender, purged, false, Bukkit.getOfflinePlayers());
}
public PurgeTask(AuthMe plugin, CommandSender sender, Set<String> purged, Set<OfflinePlayer> offlinePlayers) {
this(plugin, sender, purged, false
, offlinePlayers.toArray(new OfflinePlayer[offlinePlayers.size()]));
}
public PurgeTask(AuthMe plugin, CommandSender sender, Set<String> purged
public PurgeTask(CommandSender sender, Set<String> purged
, boolean autoPurge, OfflinePlayer[] offlinePlayers) {
this.plugin = plugin;
this.newSetting = plugin.getSettings();
if (sender instanceof Player) {
this.sender = ((Player) sender).getUniqueId();
@ -72,6 +53,21 @@ public class PurgeTask extends BukkitRunnable {
// }
}
public PurgeTask(PurgeService service, CommandSender sender, Set<String> toPurge, OfflinePlayer[] offlinePlayers,
boolean autoPurging) {
this.purgeService = service;
if (sender instanceof Player) {
this.sender = ((Player) sender).getUniqueId();
} else {
this.sender = null;
}
this.toPurge = toPurge;
this.totalPurgeCount = toPurge.size();
this.autoPurging = autoPurging;
this.offlinePlayers = offlinePlayers;
}
@Override
public void run() {
if (toPurge.isEmpty()) {
@ -116,26 +112,11 @@ public class PurgeTask extends BukkitRunnable {
private void purgeData(Set<OfflinePlayer> playerPortion, Set<String> namePortion) {
// Purge other data
if (newSetting.getProperty(PurgeSettings.REMOVE_ESSENTIALS_FILES)
&& plugin.getPluginHooks().isEssentialsAvailable()) {
plugin.dataManager.purgeEssentials(playerPortion);
}
if (newSetting.getProperty(PurgeSettings.REMOVE_PLAYER_DAT)) {
plugin.dataManager.purgeDat(playerPortion);
}
if (newSetting.getProperty(PurgeSettings.REMOVE_LIMITED_CREATIVE_INVENTORIES)) {
plugin.dataManager.purgeLimitedCreative(namePortion);
}
if (newSetting.getProperty(PurgeSettings.REMOVE_ANTI_XRAY_FILE)) {
plugin.dataManager.purgeAntiXray(namePortion);
}
if (newSetting.getProperty(PurgeSettings.REMOVE_PERMISSIONS)) {
plugin.dataManager.purgePermissions(playerPortion);
}
purgeService.purgeEssentials(playerPortion);
purgeService.purgeDat(playerPortion);
purgeService.purgeLimitedCreative(namePortion);
purgeService.purgeAntiXray(namePortion);
purgeService.purgePermissions(playerPortion);
}
private void finish() {
@ -146,7 +127,7 @@ public class PurgeTask extends BukkitRunnable {
ConsoleLogger.info("AutoPurge Finished!");
if (autoPurging) {
plugin.notifyAutoPurgeEnd();
purgeService.setAutoPurging(false);
}
}