Migrate to GEO IP 2 (Related #1471) (#1529)

* Migrate to GEO IP 2, because support will drop in April * Change all links of maxmind to https * Update maxmind database dependency and add javatar to extract the database from the tar archive (now only a small difference in jar file size -> ~80KB smaller) * Verify downloaded archive using MD5 (There are no other checksums available) * Migrate to Java NIO instead of old java file I/O (Feedback?) * Internal Optional usage for nullable values (Feedback?) Minor: * Schedule a Bukkit async task instead of creating a thread manually * Validate ip input string * Extract validation into single method * Close all resources safely using try-resources * More https links * Add documentation * Set the same last modification as in the tar archive * Fix tests * Comment how the legacy API responded to unknown entries * Document missing function param * Document our maxmind dependency modifications * Include time unit into constant * More logging for downloading the database * Add missing return if the database cannot be found * Delete temporarily file after working with it
2018-03-17 03:00:24 +01:00
parent 7ff5801cfe
commit f33446ee25
6 changed files with 262 additions and 107 deletions
@@ -2,7 +2,9 @@ package fr.xephi.authme;

 import ch.jalu.injector.Injector;
 import ch.jalu.injector.InjectorBuilder;
+
 import com.google.common.annotations.VisibleForTesting;
+
 import fr.xephi.authme.api.NewAPI;
 import fr.xephi.authme.command.CommandHandler;
 import fr.xephi.authme.datasource.DataSource;
@@ -33,6 +35,9 @@ import fr.xephi.authme.settings.properties.SecuritySettings;
 import fr.xephi.authme.task.CleanupTask;
 import fr.xephi.authme.task.purge.PurgeService;
 import fr.xephi.authme.util.ExceptionUtils;
+
+import java.io.File;
+
 import org.apache.commons.lang.SystemUtils;
 import org.bukkit.Server;
 import org.bukkit.command.Command;
@@ -43,8 +48,6 @@ import org.bukkit.plugin.java.JavaPlugin;
 import org.bukkit.plugin.java.JavaPluginLoader;
 import org.bukkit.scheduler.BukkitScheduler;

-import java.io.File;
-
 import static fr.xephi.authme.service.BukkitService.TICKS_PER_MINUTE;
 import static fr.xephi.authme.util.Utils.isClassLoaded;

@@ -1,46 +1,81 @@
 package fr.xephi.authme.service;

 import com.google.common.annotations.VisibleForTesting;
-import com.maxmind.geoip.LookupService;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+import com.google.common.io.Resources;
+import com.ice.tar.TarEntry;
+import com.ice.tar.TarInputStream;
+import com.maxmind.db.GeoIp2Provider;
+import com.maxmind.db.Reader;
+import com.maxmind.db.Reader.FileMode;
+import com.maxmind.db.cache.CHMCache;
+import com.maxmind.db.model.Country;
+import com.maxmind.db.model.CountryResponse;
+
 import fr.xephi.authme.ConsoleLogger;
 import fr.xephi.authme.initialization.DataFolder;
 import fr.xephi.authme.util.FileUtils;
 import fr.xephi.authme.util.InternetProtocolUtils;

-import javax.inject.Inject;
+import java.io.BufferedInputStream;
 import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.OutputStream;
+import java.net.InetAddress;
 import java.net.URL;
-import java.net.URLConnection;
-import java.util.concurrent.TimeUnit;
+import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.FileTime;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.zip.GZIPInputStream;

-import static com.maxmind.geoip.LookupService.GEOIP_MEMORY_CACHE;
+import javax.inject.Inject;

 public class GeoIpService {
-    private static final String LICENSE =
-        "[LICENSE] This product uses data from the GeoLite API created by MaxMind, available at http://www.maxmind.com";
-    private static final String GEOIP_URL =
-        "http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz";
-    private LookupService lookupService;
-    private Thread downloadTask;

-    private final File dataFile;
+    private static final String LICENSE =
+            "[LICENSE] This product includes GeoLite2 data created by MaxMind, available at https://www.maxmind.com";
+
+    private static final String DATABASE_NAME = "GeoLite2-Country";
+    private static final String DATABASE_EXT = ".mmdb";
+    private static final String DATABASE_FILE = DATABASE_NAME + DATABASE_EXT;
+
+    private static final String ARCHIVE_FILE = DATABASE_NAME + ".tar.gz";
+
+    private static final String ARCHIVE_URL = "https://geolite.maxmind.com/download/geoip/database/" + ARCHIVE_FILE;
+    private static final String CHECKSUM_URL = ARCHIVE_URL + ".md5";
+
+    private static final int UPDATE_INTERVAL_DAYS = 30;
+
+    private final Path dataFile;
+    private final BukkitService bukkitService;
+
+    private GeoIp2Provider databaseReader;
+    private volatile boolean downloading;

    @Inject
-    GeoIpService(@DataFolder File dataFolder) {
-        this.dataFile = new File(dataFolder, "GeoIP.dat");
+    GeoIpService(@DataFolder File dataFolder, BukkitService bukkitService) {
+        this.bukkitService = bukkitService;
+        this.dataFile = dataFolder.toPath().resolve(DATABASE_FILE);
+
        // Fires download of recent data or the initialization of the look up service
        isDataAvailable();
    }

    @VisibleForTesting
-    GeoIpService(@DataFolder File dataFolder, LookupService lookupService) {
-        this.dataFile = dataFolder;
-        this.lookupService = lookupService;
+    GeoIpService(@DataFolder File dataFolder, BukkitService bukkitService, GeoIp2Provider reader) {
+        this.bukkitService = bukkitService;
+        this.dataFile = dataFolder.toPath().resolve(DATABASE_FILE);
+
+        this.databaseReader = reader;
    }

    /**
@@ -49,94 +84,186 @@ public class GeoIpService {
     * @return True if the data is available, false otherwise.
     */
    private synchronized boolean isDataAvailable() {
-        if (downloadTask != null && downloadTask.isAlive()) {
+        if (downloading) {
+            // we are currently downloading the database
            return false;
        }
-        if (lookupService != null) {
+
+        if (databaseReader != null) {
+            // everything is initialized
            return true;
        }

-        if (dataFile.exists()) {
-            boolean dataIsOld = (System.currentTimeMillis() - dataFile.lastModified()) > TimeUnit.DAYS.toMillis(30);
-            if (!dataIsOld) {
-                try {
-                    lookupService = new LookupService(dataFile, GEOIP_MEMORY_CACHE);
+        if (Files.exists(dataFile)) {
+            try {
+                FileTime lastModifiedTime = Files.getLastModifiedTime(dataFile);
+                if (Duration.between(lastModifiedTime.toInstant(), Instant.now()).toDays() <= UPDATE_INTERVAL_DAYS) {
+                    databaseReader = new Reader(dataFile.toFile(), FileMode.MEMORY, new CHMCache());
                    ConsoleLogger.info(LICENSE);
+
+                    // don't fire the update task - we are up to date
                    return true;
-                } catch (IOException e) {
-                    ConsoleLogger.logException("Failed to load GeoLiteAPI database", e);
-                    return false;
+                } else {
+                    ConsoleLogger.debug("GEO Ip database is older than " + UPDATE_INTERVAL_DAYS + " Days");
                }
-            } else {
-                FileUtils.delete(dataFile);
+            } catch (IOException ioEx) {
+                ConsoleLogger.logException("Failed to load GeoLiteAPI database", ioEx);
+                return false;
            }
        }
-        // Ok, let's try to download the data file!
-        downloadTask = createDownloadTask();
-        downloadTask.start();
+
+        // File is outdated or doesn't exist - let's try to download the data file!
+        startDownloadTask();
        return false;
    }

    /**
     * Create a thread which will attempt to download new data from the GeoLite website.
-     *
-     * @return the generated download thread
     */
-    private Thread createDownloadTask() {
-        return new Thread(new Runnable() {
-            @Override
-            public void run() {
-                try {
-                    URL downloadUrl = new URL(GEOIP_URL);
-                    URLConnection conn = downloadUrl.openConnection();
-                    conn.setConnectTimeout(10000);
-                    conn.connect();
-                    InputStream input = conn.getInputStream();
-                    if (conn.getURL().toString().endsWith(".gz")) {
-                        input = new GZIPInputStream(input);
-                    }
-                    OutputStream output = new FileOutputStream(dataFile);
-                    byte[] buffer = new byte[2048];
-                    int length = input.read(buffer);
-                    while (length >= 0) {
-                        output.write(buffer, 0, length);
-                        length = input.read(buffer);
-                    }
-                    output.close();
-                    input.close();
-                } catch (IOException e) {
-                    ConsoleLogger.logException("Could not download GeoLiteAPI database", e);
+    private void startDownloadTask() {
+        downloading = true;
+
+        // use bukkit's cached threads
+        bukkitService.runTaskAsynchronously(() -> {
+            ConsoleLogger.info("Downloading GEO IP database, because the old database is outdated or doesn't exist");
+
+            Path tempFile = null;
+            try {
+                // download database to temporarily location
+                tempFile = Files.createTempFile(ARCHIVE_FILE, null);
+                try (OutputStream out = Files.newOutputStream(tempFile)) {
+                    Resources.copy(new URL(ARCHIVE_URL), out);
+                }
+
+                // MD5 checksum verification
+                String targetChecksum = Resources.toString(new URL(CHECKSUM_URL), StandardCharsets.UTF_8);
+                if (!verifyChecksum(Hashing.md5(), tempFile, targetChecksum)) {
+                    return;
+                }
+
+                // tar extract database and copy to target destination
+                if (!extractDatabase(tempFile, dataFile)) {
+                    ConsoleLogger.warning("Cannot find database inside downloaded GEO IP file at " + tempFile);
+                    return;
+                }
+
+                ConsoleLogger.info("Successfully downloaded new GEO IP database to " + dataFile);
+
+                //only set this value to false on success otherwise errors could lead to endless download triggers
+                downloading = false;
+            } catch (IOException ioEx) {
+                ConsoleLogger.logException("Could not download GeoLiteAPI database", ioEx);
+            } finally {
+                // clean up
+                if (tempFile != null) {
+                    FileUtils.delete(tempFile.toFile());
                }
            }
        });
    }

+    /**
+     * Verify if the expected checksum is equal to the checksum of the given file.
+     *
+     * @param function the checksum function like MD5, SHA256 used to generate the checksum from the file
+     * @param file the file we want to calculate the checksum from
+     * @param expectedChecksum the expected checksum
+     * @return true if equal, false otherwise
+     * @throws IOException on I/O error reading the file
+     */
+    private boolean verifyChecksum(HashFunction function, Path file, String expectedChecksum) throws IOException {
+        HashCode actualHash = function.hashBytes(Files.readAllBytes(file));
+        HashCode expectedHash = HashCode.fromString(expectedChecksum);
+        if (Objects.equals(actualHash, expectedHash)) {
+            return true;
+        }
+
+        ConsoleLogger.warning("GEO IP checksum verification failed");
+        ConsoleLogger.warning("Expected: " + expectedHash + " Actual: " + actualHash);
+        return false;
+    }
+
+    /**
+     * Extract the database from the tar archive. Existing outputFile will be replaced if it already exists.
+     *
+     * @param tarInputFile gzipped tar input file where the database is
+     * @param outputFile destination file for the database
+     * @return true if the database was found, false otherwise
+     * @throws IOException on I/O error reading the tar archive or writing the output
+     */
+    private boolean extractDatabase(Path tarInputFile, Path outputFile) throws IOException {
+        // .gz -> gzipped file
+        try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(tarInputFile));
+             TarInputStream tarIn = new TarInputStream(new GZIPInputStream(in))) {
+            TarEntry entry;
+            while ((entry = tarIn.getNextEntry()) != null) {
+                if (!entry.isDirectory()) {
+                    // filename including folders (absolute path inside the archive)
+                    String filename = entry.getName();
+                    if (filename.endsWith(DATABASE_EXT)) {
+                        // found the database file
+                        Files.copy(tarIn, outputFile, StandardCopyOption.REPLACE_EXISTING);
+
+                        // update the last modification date to be same as in the archive
+                        Files.setLastModifiedTime(outputFile, FileTime.from(entry.getModTime().toInstant()));
+                        return true;
+                    }
+                }
+            }
+        }
+
+        return false;
+    }
+
    /**
     * Get the country code of the given IP address.
     *
     * @param ip textual IP address to lookup.
-     *
     * @return two-character ISO 3166-1 alpha code for the country.
     */
    public String getCountryCode(String ip) {
-        if (!InternetProtocolUtils.isLocalAddress(ip) && isDataAvailable()) {
-            return lookupService.getCountry(ip).getCode();
-        }
-        return "--";
+        return getCountry(ip).map(Country::getIsoCode).orElse("--");
    }

    /**
     * Get the country name of the given IP address.
     *
     * @param ip textual IP address to lookup.
-     *
     * @return The name of the country.
     */
    public String getCountryName(String ip) {
-        if (!InternetProtocolUtils.isLocalAddress(ip) && isDataAvailable()) {
-            return lookupService.getCountry(ip).getName();
-        }
-        return "N/A";
+        return getCountry(ip).map(Country::getName).orElse("N/A");
    }

+    /**
+     * Get the country of the given IP address
+     *
+     * @param ip textual IP address to lookup
+     * @return the wrapped Country model or {@link Optional#empty()} if
+     * <ul>
+     *     <li>Database reader isn't initialized</li>
+     *     <li>MaxMind has no record about this IP address</li>
+     *     <li>IP address is local</li>
+     *     <li>Textual representation is not a valid IP address</li>
+     * </ul>
+     */
+    private Optional<Country> getCountry(String ip) {
+        if (ip == null || ip.isEmpty() || InternetProtocolUtils.isLocalAddress(ip) || !isDataAvailable()) {
+            return Optional.empty();
+        }
+
+        try {
+            InetAddress address = InetAddress.getByName(ip);
+
+            //Reader.getCountry() can be null for unknown addresses
+            return Optional.ofNullable(databaseReader.getCountry(address)).map(CountryResponse::getCountry);
+        } catch (UnknownHostException e) {
+            // Ignore invalid ip addresses
+            // Legacy GEO IP Database returned a unknown country object with Country-Code: '--' and Country-Name: 'N/A'
+        } catch (IOException ioEx) {
+            ConsoleLogger.logException("Cannot lookup country for " + ip + " at GEO IP database", ioEx);
+        }
+
+        return Optional.empty();
+    }
 }
@@ -22,7 +22,7 @@ public final class ProtectionSettings implements SettingsHolder {

    @Comment({
        "Countries allowed to join the server and register. For country codes, see",
-        "http://dev.maxmind.com/geoip/legacy/codes/iso3166/",
+        "https://dev.maxmind.com/geoip/legacy/codes/iso3166/",
        "PLEASE USE QUOTES!"})
    public static final Property<List<String>> COUNTRIES_WHITELIST =
        newListProperty("Protection.countries", "US", "GB");