From 189647d9f2365c843fe5ea420e251b7c2002dc11 Mon Sep 17 00:00:00 2001 From: ljacqu Date: Sun, 11 Feb 2018 09:22:42 +0100 Subject: [PATCH] #1467 Fix character issues by always using UTF-8 when reading and writing - Change usages of Bukkit's FileResource to a ConfigMe PropertyReader - Specify UTF-8 for reading and writing --- .../message/updater/JarMessageSource.java | 28 ++--- .../MessageMigraterPropertyReader.java | 47 +++++--- .../message/updater/MessageUpdater.java | 28 ----- .../updater/MigraterYamlFileResource.java | 103 ++++++++++++++++++ .../xephi/authme/ClassesConsistencyTest.java | 2 + .../updater/MigraterYamlFileResourceTest.java | 70 ++++++++++++ .../tools/messages/MessagesFileWriter.java | 2 +- .../fr/xephi/authme/message/chinese_texts.yml | 3 + 8 files changed, 224 insertions(+), 59 deletions(-) create mode 100644 src/main/java/fr/xephi/authme/message/updater/MigraterYamlFileResource.java create mode 100644 src/test/java/fr/xephi/authme/message/updater/MigraterYamlFileResourceTest.java create mode 100644 src/test/resources/fr/xephi/authme/message/chinese_texts.yml diff --git a/src/main/java/fr/xephi/authme/message/updater/JarMessageSource.java b/src/main/java/fr/xephi/authme/message/updater/JarMessageSource.java index c4da59c7..34c95a6c 100644 --- a/src/main/java/fr/xephi/authme/message/updater/JarMessageSource.java +++ b/src/main/java/fr/xephi/authme/message/updater/JarMessageSource.java @@ -1,14 +1,12 @@ package fr.xephi.authme.message.updater; import ch.jalu.configme.properties.Property; +import ch.jalu.configme.resource.PropertyReader; import fr.xephi.authme.ConsoleLogger; import fr.xephi.authme.util.FileUtils; -import org.bukkit.configuration.file.FileConfiguration; -import org.bukkit.configuration.file.YamlConfiguration; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; /** * Returns messages from the JAR's message files. Favors a local JAR (e.g. messages_nl.yml) @@ -16,8 +14,8 @@ import java.io.InputStreamReader; */ public class JarMessageSource { - private final FileConfiguration localJarConfiguration; - private final FileConfiguration defaultJarConfiguration; + private final PropertyReader localJarMessages; + private final PropertyReader defaultJarMessages; /** * Constructor. @@ -26,29 +24,31 @@ public class JarMessageSource { * @param defaultJarPath path to the default messages file in the JAR (must exist) */ public JarMessageSource(String localJarPath, String defaultJarPath) { - localJarConfiguration = localJarPath.equals(defaultJarPath) ? null : loadJarFile(localJarPath); - defaultJarConfiguration = loadJarFile(defaultJarPath); + localJarMessages = localJarPath.equals(defaultJarPath) ? null : loadJarFile(localJarPath); + defaultJarMessages = loadJarFile(defaultJarPath); - if (defaultJarConfiguration == null) { + if (defaultJarMessages == null) { throw new IllegalStateException("Default JAR file '" + defaultJarPath + "' could not be loaded"); } } public String getMessageFromJar(Property property) { String key = property.getPath(); - String message = localJarConfiguration == null ? null : localJarConfiguration.getString(key); - return message == null ? defaultJarConfiguration.getString(key) : message; + String message = getString(key, localJarMessages); + return message == null ? getString(key, defaultJarMessages) : message; } - private static YamlConfiguration loadJarFile(String jarPath) { + private static String getString(String path, PropertyReader reader) { + return reader == null ? null : reader.getTypedObject(path, String.class); + } + + private static MessageMigraterPropertyReader loadJarFile(String jarPath) { try (InputStream stream = FileUtils.getResourceFromJar(jarPath)) { if (stream == null) { ConsoleLogger.debug("Could not load '" + jarPath + "' from JAR"); return null; } - try (InputStreamReader isr = new InputStreamReader(stream)) { - return YamlConfiguration.loadConfiguration(isr); - } + return MessageMigraterPropertyReader.loadFromStream(stream); } catch (IOException e) { ConsoleLogger.logException("Exception while handling JAR path '" + jarPath + "'", e); } diff --git a/src/main/java/fr/xephi/authme/message/updater/MessageMigraterPropertyReader.java b/src/main/java/fr/xephi/authme/message/updater/MessageMigraterPropertyReader.java index 400b25ff..1174ba04 100644 --- a/src/main/java/fr/xephi/authme/message/updater/MessageMigraterPropertyReader.java +++ b/src/main/java/fr/xephi/authme/message/updater/MessageMigraterPropertyReader.java @@ -7,31 +7,44 @@ import org.yaml.snakeyaml.Yaml; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import java.util.Objects; /** - * Duplication of ConfigMe's {@link ch.jalu.configme.resource.YamlFileReader} with a character encoding - * fix in {@link #reload}. + * Implementation of {@link PropertyReader} which can read a file or a stream with + * a specified charset. */ public class MessageMigraterPropertyReader implements PropertyReader { - private final File file; + public static final Charset CHARSET = StandardCharsets.UTF_8; + private Map root; /** See same field in {@link ch.jalu.configme.resource.YamlFileReader} for details. */ private boolean hasObjectAsRoot = false; - /** - * Constructor. - * - * @param file the file to load - */ - public MessageMigraterPropertyReader(File file) { - this.file = file; - reload(); + private MessageMigraterPropertyReader(Map valuesMap) { + root = valuesMap; + } + + public static MessageMigraterPropertyReader loadFromFile(File file) { + Map valuesMap; + try (InputStream is = new FileInputStream(file)) { + valuesMap = readStreamToMap(is); + } catch (IOException e) { + throw new IllegalStateException("Error while reading file '" + file + "'", e); + } + + return new MessageMigraterPropertyReader(valuesMap); + } + + public static MessageMigraterPropertyReader loadFromStream(InputStream inputStream) { + Map valuesMap = readStreamToMap(inputStream); + return new MessageMigraterPropertyReader(valuesMap); } @Override @@ -110,15 +123,17 @@ public class MessageMigraterPropertyReader implements PropertyReader { @Override public void reload() { - try (FileInputStream fis = new FileInputStream(file); - InputStreamReader isr = new InputStreamReader(fis, StandardCharsets.UTF_8)) { + throw new UnsupportedOperationException("Reload not supported by this implementation"); + } + private static Map readStreamToMap(InputStream inputStream) { + try (InputStreamReader isr = new InputStreamReader(inputStream, CHARSET)) { Object obj = new Yaml().load(isr); - root = obj == null ? new HashMap<>() : (Map) obj; + return obj == null ? new HashMap<>() : (Map) obj; } catch (IOException e) { - throw new ConfigMeException("Could not read file '" + file + "'", e); + throw new ConfigMeException("Could not read stream", e); } catch (ClassCastException e) { - throw new ConfigMeException("Top-level is not a map in '" + file + "'", e); + throw new ConfigMeException("Top-level is not a map", e); } } diff --git a/src/main/java/fr/xephi/authme/message/updater/MessageUpdater.java b/src/main/java/fr/xephi/authme/message/updater/MessageUpdater.java index b4634a71..9e06baff 100644 --- a/src/main/java/fr/xephi/authme/message/updater/MessageUpdater.java +++ b/src/main/java/fr/xephi/authme/message/updater/MessageUpdater.java @@ -1,7 +1,6 @@ package fr.xephi.authme.message.updater; import ch.jalu.configme.SettingsManager; -import ch.jalu.configme.beanmapper.leafproperties.LeafPropertiesGenerator; import ch.jalu.configme.configurationdata.ConfigurationData; import ch.jalu.configme.configurationdata.PropertyListBuilder; import ch.jalu.configme.properties.Property; @@ -10,8 +9,6 @@ import ch.jalu.configme.resource.YamlFileResource; import com.google.common.collect.ImmutableMap; import fr.xephi.authme.ConsoleLogger; import fr.xephi.authme.message.MessageKey; -import org.yaml.snakeyaml.DumperOptions; -import org.yaml.snakeyaml.Yaml; import java.io.File; import java.util.Arrays; @@ -134,29 +131,4 @@ public class MessageUpdater { return new ConfigurationData(builder.create(), comments); } - /** - * Extension of {@link YamlFileResource} to fine-tune the export style. - */ - public static final class MigraterYamlFileResource extends YamlFileResource { - - private Yaml singleQuoteYaml; - - public MigraterYamlFileResource(File file) { - super(file, new MessageMigraterPropertyReader(file), new LeafPropertiesGenerator()); - } - - @Override - protected Yaml getSingleQuoteYaml() { - if (singleQuoteYaml == null) { - DumperOptions options = new DumperOptions(); - options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); - options.setAllowUnicode(true); - options.setDefaultScalarStyle(DumperOptions.ScalarStyle.SINGLE_QUOTED); - // Overridden setting: don't split lines - options.setSplitLines(false); - singleQuoteYaml = new Yaml(options); - } - return singleQuoteYaml; - } - } } diff --git a/src/main/java/fr/xephi/authme/message/updater/MigraterYamlFileResource.java b/src/main/java/fr/xephi/authme/message/updater/MigraterYamlFileResource.java new file mode 100644 index 00000000..69b580ae --- /dev/null +++ b/src/main/java/fr/xephi/authme/message/updater/MigraterYamlFileResource.java @@ -0,0 +1,103 @@ +package fr.xephi.authme.message.updater; + +import ch.jalu.configme.beanmapper.leafproperties.LeafPropertiesGenerator; +import ch.jalu.configme.configurationdata.ConfigurationData; +import ch.jalu.configme.exception.ConfigMeException; +import ch.jalu.configme.properties.Property; +import ch.jalu.configme.resource.PropertyPathTraverser; +import ch.jalu.configme.resource.YamlFileResource; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.List; + +import static fr.xephi.authme.message.updater.MessageMigraterPropertyReader.CHARSET; + +/** + * Extension of {@link YamlFileResource} to fine-tune the export style + * and to be able to specify the character encoding. + */ +public class MigraterYamlFileResource extends YamlFileResource { + + private static final String INDENTATION = " "; + + private final File file; + private Yaml singleQuoteYaml; + + public MigraterYamlFileResource(File file) { + super(file, MessageMigraterPropertyReader.loadFromFile(file), new LeafPropertiesGenerator()); + this.file = file; + } + + @Override + protected Yaml getSingleQuoteYaml() { + if (singleQuoteYaml == null) { + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + options.setAllowUnicode(true); + options.setDefaultScalarStyle(DumperOptions.ScalarStyle.SINGLE_QUOTED); + // Overridden setting: don't split lines + options.setSplitLines(false); + singleQuoteYaml = new Yaml(options); + } + return singleQuoteYaml; + } + + @Override + public void exportProperties(ConfigurationData configurationData) { + try (FileOutputStream fos = new FileOutputStream(file); + OutputStreamWriter writer = new OutputStreamWriter(fos, CHARSET)) { + PropertyPathTraverser pathTraverser = new PropertyPathTraverser(configurationData); + for (Property property : convertPropertiesToExportableTypes(configurationData.getProperties())) { + + List pathElements = pathTraverser.getPathElements(property); + for (PropertyPathTraverser.PathElement pathElement : pathElements) { + writeComments(writer, pathElement.indentationLevel, pathElement.comments); + writer.append("\n") + .append(indent(pathElement.indentationLevel)) + .append(pathElement.name) + .append(":"); + } + + writer.append(" ") + .append(toYaml(property, pathElements.get(pathElements.size() - 1).indentationLevel)); + } + writer.flush(); + writer.close(); + } catch (IOException e) { + throw new ConfigMeException("Could not save config to '" + file.getPath() + "'", e); + } finally { + singleQuoteYaml = null; + } + } + + private void writeComments(Writer writer, int indentation, String[] comments) throws IOException { + if (comments.length == 0) { + return; + } + String commentStart = "\n" + indent(indentation) + "# "; + for (String comment : comments) { + writer.append(commentStart).append(comment); + } + } + + private String toYaml(Property property, int indent) { + Object value = property.getValue(this); + String representation = transformValue(property, value); + String[] lines = representation.split("\\n"); + return String.join("\n" + indent(indent), lines); + } + + private static String indent(int level) { + String result = ""; + for (int i = 0; i < level; i++) { + result += INDENTATION; + } + return result; + } +} diff --git a/src/test/java/fr/xephi/authme/ClassesConsistencyTest.java b/src/test/java/fr/xephi/authme/ClassesConsistencyTest.java index db74ce6f..267f7d7c 100644 --- a/src/test/java/fr/xephi/authme/ClassesConsistencyTest.java +++ b/src/test/java/fr/xephi/authme/ClassesConsistencyTest.java @@ -18,6 +18,7 @@ import org.junit.Test; import java.io.File; import java.lang.reflect.Field; import java.lang.reflect.Modifier; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -49,6 +50,7 @@ public class ClassesConsistencyTest { private static final Set> IMMUTABLE_TYPES = ImmutableSet.of( /* JDK */ int.class, long.class, float.class, String.class, File.class, Enum.class, collectionsUnmodifiableList(), + Charset.class, /* AuthMe */ Property.class, RegistrationMethod.class, /* Guava */ diff --git a/src/test/java/fr/xephi/authme/message/updater/MigraterYamlFileResourceTest.java b/src/test/java/fr/xephi/authme/message/updater/MigraterYamlFileResourceTest.java new file mode 100644 index 00000000..f6e6dcd2 --- /dev/null +++ b/src/test/java/fr/xephi/authme/message/updater/MigraterYamlFileResourceTest.java @@ -0,0 +1,70 @@ +package fr.xephi.authme.message.updater; + +import ch.jalu.configme.configurationdata.ConfigurationData; +import ch.jalu.configme.properties.Property; +import ch.jalu.configme.properties.StringProperty; +import com.google.common.io.Files; +import fr.xephi.authme.TestHelper; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertThat; + +/** + * Test for {@link MigraterYamlFileResource}. + */ +public class MigraterYamlFileResourceTest { + + private static final String CHINESE_MESSAGES_FILE = TestHelper.PROJECT_ROOT + "message/chinese_texts.yml"; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void shouldReadChineseFile() { + // given + File file = TestHelper.getJarFile(CHINESE_MESSAGES_FILE); + + // when + MigraterYamlFileResource resource = new MigraterYamlFileResource(file); + + // then + assertThat(resource.getString("first"), equalTo("错误的密码")); + assertThat(resource.getString("second"), equalTo("为了验证您的身份，您需要将一个电子邮件地址与您的帐户绑定！")); + assertThat(resource.getString("third"), equalTo("您已经可以在当前会话中执行任何敏感命令！")); + } + + @Test + public void shouldWriteWithCorrectCharset() throws IOException { + // given + File file = temporaryFolder.newFile(); + Files.copy(TestHelper.getJarFile(CHINESE_MESSAGES_FILE), file); + MigraterYamlFileResource resource = new MigraterYamlFileResource(file); + String newMessage = "您当前并没有任何邮箱与该账号绑定"; + resource.setValue("third", newMessage); + + // when + resource.exportProperties(buildConfigurationData()); + + // then + resource = new MigraterYamlFileResource(file); + assertThat(resource.getString("first"), equalTo("错误的密码")); + assertThat(resource.getString("second"), equalTo("为了验证您的身份，您需要将一个电子邮件地址与您的帐户绑定！")); + assertThat(resource.getString("third"), equalTo(newMessage)); + } + + private static ConfigurationData buildConfigurationData() { + List> properties = Arrays.asList( + new StringProperty("first", "first"), + new StringProperty("second", "second"), + new StringProperty("third", "third")); + return new ConfigurationData(properties); + } +} diff --git a/src/test/java/tools/messages/MessagesFileWriter.java b/src/test/java/tools/messages/MessagesFileWriter.java index 2af69e2c..534be000 100644 --- a/src/test/java/tools/messages/MessagesFileWriter.java +++ b/src/test/java/tools/messages/MessagesFileWriter.java @@ -6,7 +6,7 @@ import ch.jalu.configme.properties.Property; import ch.jalu.configme.resource.PropertyResource; import ch.jalu.configme.resource.YamlFileResource; import fr.xephi.authme.message.updater.MessageUpdater; -import fr.xephi.authme.message.updater.MessageUpdater.MigraterYamlFileResource; +import fr.xephi.authme.message.updater.MigraterYamlFileResource; import org.bukkit.configuration.file.FileConfiguration; import tools.utils.FileIoUtils; diff --git a/src/test/resources/fr/xephi/authme/message/chinese_texts.yml b/src/test/resources/fr/xephi/authme/message/chinese_texts.yml new file mode 100644 index 00000000..5cb7228b --- /dev/null +++ b/src/test/resources/fr/xephi/authme/message/chinese_texts.yml @@ -0,0 +1,3 @@ +first: '错误的密码' +second: '为了验证您的身份，您需要将一个电子邮件地址与您的帐户绑定！' +third: '您已经可以在当前会话中执行任何敏感命令！'