#792 Force migration of SQLite when necessary (#1371)

- Detect if a migration is necessary
- Create a backup
- Perform the migration
This commit is contained in:
ljacqu
2017-10-22 09:16:48 +02:00
committed by GitHub
parent c37c0ce436
commit d6e2369f36
12 changed files with 234 additions and 260 deletions
@@ -21,7 +21,7 @@ public class DebugCommand implements ExecutableCommand {
private static final Set<Class<? extends DebugSection>> SECTION_CLASSES = ImmutableSet.of(
PermissionGroups.class, DataStatistics.class, CountryLookup.class, PlayerAuthViewer.class, InputValidator.class,
LimboPlayerViewer.class, CountryLookup.class, HasPermissionChecker.class, TestEmailSender.class,
SpawnLocationViewer.class, MySqlDefaultChanger.class, SqliteMigrater.class);
SpawnLocationViewer.class, MySqlDefaultChanger.class);
@Inject
private Factory<DebugSection> debugSectionFactory;
@@ -1,178 +0,0 @@
package fr.xephi.authme.command.executable.authme.debug;
import fr.xephi.authme.ConsoleLogger;
import fr.xephi.authme.datasource.Columns;
import fr.xephi.authme.datasource.DataSource;
import fr.xephi.authme.datasource.SQLite;
import fr.xephi.authme.permission.DebugSectionPermissions;
import fr.xephi.authme.permission.PermissionNode;
import fr.xephi.authme.settings.Settings;
import fr.xephi.authme.settings.properties.DatabaseSettings;
import fr.xephi.authme.util.RandomStringUtils;
import org.bukkit.ChatColor;
import org.bukkit.command.CommandSender;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import java.lang.reflect.Field;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import static fr.xephi.authme.command.executable.authme.debug.DebugSectionUtils.castToTypeOrNull;
import static fr.xephi.authme.command.executable.authme.debug.DebugSectionUtils.unwrapSourceFromCacheDataSource;
import static org.bukkit.ChatColor.BOLD;
import static org.bukkit.ChatColor.GOLD;
/**
* Performs a migration on the SQLite data source if necessary.
*/
class SqliteMigrater implements DebugSection {
@Inject
private DataSource dataSource;
@Inject
private Settings settings;
private SQLite sqLite;
private String confirmationCode;
@PostConstruct
void setSqLiteField() {
this.sqLite = castToTypeOrNull(unwrapSourceFromCacheDataSource(this.dataSource), SQLite.class);
}
@Override
public String getName() {
return "migratesqlite";
}
@Override
public String getDescription() {
return "Migrates the SQLite database";
}
// A migration can be forced even if SQLite says it doesn't need a migration by adding "force" as second argument
@Override
public void execute(CommandSender sender, List<String> arguments) {
if (sqLite == null) {
sender.sendMessage("This command migrates SQLite. You are currently not using a SQLite database.");
return;
}
if (!isMigrationRequired() && !isMigrationForced(arguments)) {
sender.sendMessage("Good news! No migration is required of your database");
} else if (checkConfirmationCodeAndInformSenderOnMismatch(sender, arguments)) {
final String tableName = settings.getProperty(DatabaseSettings.MYSQL_TABLE);
final Columns columns = new Columns(settings);
try {
recreateDatabaseWithNewDefinitions(tableName, columns);
sender.sendMessage(ChatColor.GREEN + "Successfully migrated your SQLite database!");
} catch (SQLException e) {
ConsoleLogger.logException("Failed to migrate SQLite database", e);
sender.sendMessage(ChatColor.RED
+ "An error occurred during SQLite migration. Please check the logs!");
}
}
}
private boolean checkConfirmationCodeAndInformSenderOnMismatch(CommandSender sender, List<String> arguments) {
boolean isMatch = !arguments.isEmpty() && arguments.get(0).equalsIgnoreCase(confirmationCode);
if (isMatch) {
confirmationCode = null;
return true;
} else {
confirmationCode = RandomStringUtils.generate(4).toUpperCase();
sender.sendMessage(new String[]{
BOLD.toString() + GOLD + "Please create a backup of your SQLite database before running this command!",
"Either copy your DB file or run /authme backup. Afterwards,",
String.format("run '/authme debug %s %s' to perform the migration. "
+ "The code confirms that you've made a backup!", getName(), confirmationCode)
});
return false;
}
}
@Override
public PermissionNode getRequiredPermission() {
return DebugSectionPermissions.MIGRATE_SQLITE;
}
private boolean isMigrationRequired() {
Connection connection = getConnection(sqLite);
try {
DatabaseMetaData metaData = connection.getMetaData();
return sqLite.isMigrationRequired(metaData);
} catch (SQLException e) {
throw new IllegalStateException("Could not check if SQLite migration is required", e);
}
}
private static boolean isMigrationForced(List<String> arguments) {
return arguments.size() >= 2 && "force".equals(arguments.get(1));
}
// Cannot rename or remove a column from SQLite, so we have to rename the table and create an updated one
// cf. https://stackoverflow.com/questions/805363/how-do-i-rename-a-column-in-a-sqlite-database-table
private void recreateDatabaseWithNewDefinitions(String tableName, Columns col) throws SQLException {
Connection connection = getConnection(sqLite);
String tempTable = "tmp_" + tableName;
try (Statement st = connection.createStatement()) {
st.execute("ALTER TABLE " + tableName + " RENAME TO " + tempTable + ";");
}
sqLite.reload();
connection = getConnection(sqLite);
try (Statement st = connection.createStatement()) {
String copySql = "INSERT INTO $table ($id, $name, $realName, $password, $lastIp, $lastLogin, $regIp, "
+ "$regDate, $locX, $locY, $locZ, $locWorld, $locPitch, $locYaw, $email, $isLogged)"
+ "SELECT $id, $name, $realName,"
+ " $password, CASE WHEN $lastIp = '127.0.0.1' OR $lastIp = '' THEN NULL else $lastIp END,"
+ " $lastLogin, $regIp, $regDate, $locX, $locY, $locZ, $locWorld, $locPitch, $locYaw,"
+ " CASE WHEN $email = 'your@email.com' THEN NULL ELSE $email END, $isLogged"
+ " FROM " + tempTable + ";";
int insertedEntries = st.executeUpdate(replaceColumnVariables(copySql, tableName, col));
ConsoleLogger.info("Copied over " + insertedEntries + " from the old table to the new one");
st.execute("DROP TABLE " + tempTable + ";");
}
}
private String replaceColumnVariables(String sql, String tableName, Columns col) {
String replacedSql = sql.replace("$table", tableName).replace("$id", col.ID)
.replace("$name", col.NAME).replace("$realName", col.REAL_NAME)
.replace("$password", col.PASSWORD).replace("$lastIp", col.LAST_IP)
.replace("$lastLogin", col.LAST_LOGIN).replace("$regIp", col.REGISTRATION_IP)
.replace("$regDate", col.REGISTRATION_DATE).replace("$locX", col.LASTLOC_X)
.replace("$locY", col.LASTLOC_Y).replace("$locZ", col.LASTLOC_Z)
.replace("$locWorld", col.LASTLOC_WORLD).replace("$locPitch", col.LASTLOC_PITCH)
.replace("$locYaw", col.LASTLOC_YAW).replace("$email", col.EMAIL)
.replace("$isLogged", col.IS_LOGGED);
if (replacedSql.contains("$")) {
throw new IllegalStateException("SQL still statement still has '$' in it - was a tag not replaced?"
+ " Replacement result: " + replacedSql);
}
return replacedSql;
}
/**
* Returns the connection from the given SQLite instance.
*
* @param sqLite the SQLite instance to process
* @return the connection to the SQLite database
*/
private static Connection getConnection(SQLite sqLite) {
try {
Field connectionField = SQLite.class.getDeclaredField("con");
connectionField.setAccessible(true);
return (Connection) connectionField.get(sqLite);
} catch (NoSuchFieldException | IllegalAccessException e) {
throw new IllegalStateException("Failed to get the connection from SQLite", e);
}
}
}
@@ -8,6 +8,7 @@ import fr.xephi.authme.settings.Settings;
import fr.xephi.authme.settings.properties.DatabaseSettings;
import fr.xephi.authme.util.StringUtils;
import java.io.File;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
@@ -29,6 +30,8 @@ import static fr.xephi.authme.datasource.SqlDataSourceUtils.logSqlException;
*/
public class SQLite implements DataSource {
private final Settings settings;
private final File dataFolder;
private final String database;
private final String tableName;
private final Columns col;
@@ -39,10 +42,11 @@ public class SQLite implements DataSource {
*
* @param settings The settings instance
*
* @throws ClassNotFoundException if no driver could be found for the datasource
* @throws SQLException when initialization of a SQL datasource failed
* @throws SQLException when initialization of a SQL datasource failed
*/
public SQLite(Settings settings) throws ClassNotFoundException, SQLException {
public SQLite(Settings settings, File dataFolder) throws SQLException {
this.settings = settings;
this.dataFolder = dataFolder;
this.database = settings.getProperty(DatabaseSettings.MYSQL_DATABASE);
this.tableName = settings.getProperty(DatabaseSettings.MYSQL_TABLE);
this.col = new Columns(settings);
@@ -50,26 +54,40 @@ public class SQLite implements DataSource {
try {
this.connect();
this.setup();
} catch (ClassNotFoundException | SQLException ex) {
this.migrateIfNeeded();
} catch (Exception ex) {
ConsoleLogger.logException("Error during SQLite initialization:", ex);
throw ex;
}
}
@VisibleForTesting
SQLite(Settings settings, Connection connection) {
SQLite(Settings settings, File dataFolder, Connection connection) {
this.settings = settings;
this.dataFolder = dataFolder;
this.database = settings.getProperty(DatabaseSettings.MYSQL_DATABASE);
this.tableName = settings.getProperty(DatabaseSettings.MYSQL_TABLE);
this.col = new Columns(settings);
this.con = connection;
}
private void connect() throws ClassNotFoundException, SQLException {
Class.forName("org.sqlite.JDBC");
ConsoleLogger.info("SQLite driver loaded");
/**
* Initializes the connection to the SQLite database.
*/
protected void connect() throws SQLException {
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {
throw new IllegalStateException("Failed to load SQLite JDBC class", e);
}
ConsoleLogger.debug("SQLite driver loaded");
this.con = DriverManager.getConnection("jdbc:sqlite:plugins/AuthMe/" + database + ".db");
}
/**
* Creates the table if necessary, or adds any missing columns to the table.
*/
@VisibleForTesting
protected void setup() throws SQLException {
try (Statement st = con.createStatement()) {
@@ -152,28 +170,18 @@ public class SQLite implements DataSource {
st.executeUpdate("ALTER TABLE " + tableName
+ " ADD COLUMN " + col.HAS_SESSION + " INT NOT NULL DEFAULT '0';");
}
if (isMigrationRequired(md)) {
ConsoleLogger.warning("READ ME! Your SQLite database is outdated and cannot save new players.");
ConsoleLogger.warning("Run /authme debug migratesqlite after making a backup");
}
}
ConsoleLogger.info("SQLite Setup finished");
}
/**
* Returns whether the database needs to be migrated.
* <p>
* Background: Before commit 22911a0 (July 2016), new SQLite databases initialized the last IP column to be NOT NULL
* without a default value. Allowing the last IP to be null (#792) is therefore not compatible.
*
* @param metaData the database meta data
* @return true if a migration is necessary, false otherwise
* @throws SQLException .
*/
public boolean isMigrationRequired(DatabaseMetaData metaData) throws SQLException {
return SqlDataSourceUtils.isNotNullColumn(metaData, tableName, col.LAST_IP)
&& SqlDataSourceUtils.getColumnDefaultValue(metaData, tableName, col.LAST_IP) == null;
protected void migrateIfNeeded() throws SQLException {
DatabaseMetaData metaData = con.getMetaData();
if (SqLiteMigrater.isMigrationRequired(metaData, tableName, col)) {
new SqLiteMigrater(settings, dataFolder).performMigration(this);
// Migration deletes the table and recreates it, therefore call connect() again
// to get an up-to-date Connection to the database
connect();
}
}
private boolean isColumnMissing(DatabaseMetaData metaData, String columnName) throws SQLException {
@@ -188,8 +196,9 @@ public class SQLite implements DataSource {
try {
this.connect();
this.setup();
} catch (ClassNotFoundException | SQLException ex) {
ConsoleLogger.logException("Error during SQLite initialization:", ex);
this.migrateIfNeeded();
} catch (SQLException ex) {
ConsoleLogger.logException("Error while reloading SQLite:", ex);
}
}
@@ -0,0 +1,154 @@
package fr.xephi.authme.datasource;
import fr.xephi.authme.ConsoleLogger;
import fr.xephi.authme.initialization.DataFolder;
import fr.xephi.authme.settings.Settings;
import fr.xephi.authme.settings.properties.DatabaseSettings;
import fr.xephi.authme.util.FileUtils;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.file.Files;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
/**
* Migrates the SQLite database when necessary.
*/
class SqLiteMigrater {
@DataFolder
private final File dataFolder;
private final String databaseName;
private final String tableName;
private final Columns col;
@Inject
SqLiteMigrater(Settings settings, @DataFolder File dataFolder) {
this.dataFolder = dataFolder;
this.databaseName = settings.getProperty(DatabaseSettings.MYSQL_DATABASE);
this.tableName = settings.getProperty(DatabaseSettings.MYSQL_TABLE);
this.col = new Columns(settings);
}
/**
* Returns whether the database needs to be migrated.
* <p>
* Background: Before commit 22911a0 (July 2016), new SQLite databases initialized the last IP column to be NOT NULL
* without a default value. Allowing the last IP to be null (#792) is therefore not compatible.
*
* @param metaData the database meta data
* @param tableName the table name (SQLite file name)
* @param col column names configuration
* @return true if a migration is necessary, false otherwise
*/
static boolean isMigrationRequired(DatabaseMetaData metaData, String tableName, Columns col) throws SQLException {
return SqlDataSourceUtils.isNotNullColumn(metaData, tableName, col.LAST_IP)
&& SqlDataSourceUtils.getColumnDefaultValue(metaData, tableName, col.LAST_IP) == null;
}
/**
* Migrates the given SQLite instance.
*
* @param sqLite the instance to migrate
*/
void performMigration(SQLite sqLite) throws SQLException {
ConsoleLogger.warning("YOUR SQLITE DATABASE NEEDS MIGRATING! DO NOT TURN OFF YOUR SERVER");
String backupName = createBackup();
ConsoleLogger.info("Made a backup of your database at 'backups/" + backupName + "'");
recreateDatabaseWithNewDefinitions(sqLite);
ConsoleLogger.info("SQLite database migrated successfully");
}
private String createBackup() {
File sqLite = new File(dataFolder, databaseName + ".db");
File backupDirectory = new File(dataFolder, "backups");
FileUtils.createDirectory(backupDirectory);
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd_HH-mm");
String backupName = "backup-" + databaseName + dateFormat.format(new Date()) + ".db";
File backup = new File(backupDirectory, backupName);
try {
Files.copy(sqLite.toPath(), backup.toPath());
return backupName;
} catch (IOException e) {
throw new IllegalStateException("Failed to create SQLite backup before migration", e);
}
}
/**
* Renames the current database, creates a new database under the name and copies the data
* from the renamed database to the newly created one. This is necessary because SQLite
* does not support dropping or modifying a column.
*
* @param sqLite the SQLite instance to migrate
*/
// cf. https://stackoverflow.com/questions/805363/how-do-i-rename-a-column-in-a-sqlite-database-table
private void recreateDatabaseWithNewDefinitions(SQLite sqLite) throws SQLException {
Connection connection = getConnection(sqLite);
String tempTable = "tmp_" + tableName;
try (Statement st = connection.createStatement()) {
st.execute("ALTER TABLE " + tableName + " RENAME TO " + tempTable + ";");
}
sqLite.reload();
connection = getConnection(sqLite);
try (Statement st = connection.createStatement()) {
String copySql = "INSERT INTO $table ($id, $name, $realName, $password, $lastIp, $lastLogin, $regIp, "
+ "$regDate, $locX, $locY, $locZ, $locWorld, $locPitch, $locYaw, $email, $isLogged)"
+ "SELECT $id, $name, $realName,"
+ " $password, CASE WHEN $lastIp = '127.0.0.1' OR $lastIp = '' THEN NULL else $lastIp END,"
+ " $lastLogin, $regIp, $regDate, $locX, $locY, $locZ, $locWorld, $locPitch, $locYaw,"
+ " CASE WHEN $email = 'your@email.com' THEN NULL ELSE $email END, $isLogged"
+ " FROM " + tempTable + ";";
int insertedEntries = st.executeUpdate(replaceColumnVariables(copySql));
ConsoleLogger.info("Copied over " + insertedEntries + " from the old table to the new one");
st.execute("DROP TABLE " + tempTable + ";");
}
}
private String replaceColumnVariables(String sql) {
String replacedSql = sql.replace("$table", tableName).replace("$id", col.ID)
.replace("$name", col.NAME).replace("$realName", col.REAL_NAME)
.replace("$password", col.PASSWORD).replace("$lastIp", col.LAST_IP)
.replace("$lastLogin", col.LAST_LOGIN).replace("$regIp", col.REGISTRATION_IP)
.replace("$regDate", col.REGISTRATION_DATE).replace("$locX", col.LASTLOC_X)
.replace("$locY", col.LASTLOC_Y).replace("$locZ", col.LASTLOC_Z)
.replace("$locWorld", col.LASTLOC_WORLD).replace("$locPitch", col.LASTLOC_PITCH)
.replace("$locYaw", col.LASTLOC_YAW).replace("$email", col.EMAIL)
.replace("$isLogged", col.IS_LOGGED);
if (replacedSql.contains("$")) {
throw new IllegalStateException("SQL still statement still has '$' in it - was a tag not replaced?"
+ " Replacement result: " + replacedSql);
}
return replacedSql;
}
/**
* Returns the connection from the given SQLite instance.
*
* @param sqLite the SQLite instance to process
* @return the connection to the SQLite database
*/
private static Connection getConnection(SQLite sqLite) {
try {
Field connectionField = SQLite.class.getDeclaredField("con");
connectionField.setAccessible(true);
return (Connection) connectionField.get(sqLite);
} catch (NoSuchFieldException | IllegalAccessException e) {
throw new IllegalStateException("Failed to get the connection from SQLite", e);
}
}
}
@@ -3,9 +3,11 @@ package fr.xephi.authme.datasource.converter;
import fr.xephi.authme.datasource.DataSource;
import fr.xephi.authme.datasource.DataSourceType;
import fr.xephi.authme.datasource.SQLite;
import fr.xephi.authme.initialization.DataFolder;
import fr.xephi.authme.settings.Settings;
import javax.inject.Inject;
import java.io.File;
import java.sql.SQLException;
/**
@@ -14,15 +16,17 @@ import java.sql.SQLException;
public class SqliteToSql extends AbstractDataSourceConverter<SQLite> {
private final Settings settings;
private final File dataFolder;
@Inject
SqliteToSql(Settings settings, DataSource dataSource) {
SqliteToSql(Settings settings, DataSource dataSource, @DataFolder File dataFolder) {
super(dataSource, DataSourceType.MYSQL);
this.settings = settings;
this.dataFolder = dataFolder;
}
@Override
protected SQLite getSource() throws SQLException, ClassNotFoundException {
return new SQLite(settings);
protected SQLite getSource() throws SQLException {
return new SQLite(settings, dataFolder);
}
}
@@ -57,11 +57,10 @@ public class DataSourceProvider implements Provider<DataSource> {
* Sets up the data source.
*
* @return the constructed datasource
* @throws ClassNotFoundException if no driver could be found for the datasource
* @throws SQLException when initialization of a SQL datasource failed
* @throws IOException if flat file cannot be read
*/
private DataSource createDataSource() throws ClassNotFoundException, SQLException, IOException {
private DataSource createDataSource() throws SQLException, IOException {
DataSourceType dataSourceType = settings.getProperty(DatabaseSettings.BACKEND);
DataSource dataSource;
switch (dataSourceType) {
@@ -73,7 +72,7 @@ public class DataSourceProvider implements Provider<DataSource> {
dataSource = new MySQL(settings, mySqlExtensionsFactory);
break;
case SQLITE:
dataSource = new SQLite(settings);
dataSource = new SQLite(settings, dataFolder);
break;
default:
throw new UnsupportedOperationException("Unknown data source type '" + dataSourceType + "'");
@@ -113,7 +112,7 @@ public class DataSourceProvider implements Provider<DataSource> {
+ "to SQLite... Connection will be impossible until conversion is done!");
FlatFile flatFile = (FlatFile) dataSource;
try {
SQLite sqlite = new SQLite(settings);
SQLite sqlite = new SQLite(settings, dataFolder);
ForceFlatToSqlite converter = new ForceFlatToSqlite(flatFile, sqlite);
converter.execute(null);
settings.setProperty(DatabaseSettings.BACKEND, DataSourceType.SQLITE);
@@ -32,9 +32,6 @@ public enum DebugSectionPermissions implements PermissionNode {
/** Permission to change nullable status of MySQL columns. */
MYSQL_DEFAULT_CHANGER("authme.debug.mysqldef"),
/** Permission to perform a migration of SQLite. */
MIGRATE_SQLITE("authme.debug.migratesqlite"),
/** Permission to view spawn information. */
SPAWN_LOCATION("authme.debug.spawn"),