Added a way to update and setup languages

Signed-off-by: Louis Vallat <louis@louis-vallat.xyz>

MoviesQuoteBot.sql

@@ -1,9 +1,9 @@
-DROP TABLE IF EXISTS application, film, language, subtitle_line, subtitle;
+DROP TABLE IF EXISTS properties, films, languages, subtitle_lines, subtitles;
 
 /**
   Store some information on the application.
  */
-CREATE TABLE IF NOT EXISTS application
+CREATE TABLE IF NOT EXISTS properties
 (
     id        int GENERATED ALWAYS AS IDENTITY,
     app_key   text NOT NULL,
@@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS application
 /**
   Film
  */
-CREATE TABLE IF NOT EXISTS film
+CREATE TABLE IF NOT EXISTS films
 (
     id           int GENERATED ALWAYS AS IDENTITY,
     imdb_id      varchar(10) NOT NULL,
@@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS film
 /**
   Available languages
  */
-CREATE TABLE IF NOT EXISTS language
+CREATE TABLE IF NOT EXISTS languages
 (
     id       int GENERATED ALWAYS AS IDENTITY,
     alpha3_b char(3) NOT NULL,
@@ -36,13 +36,14 @@ CREATE TABLE IF NOT EXISTS language
     alpha2   char(2),
     english  text    NOT NULL,
     french   text    NOT NULL,
-    PRIMARY KEY (id)
+    PRIMARY KEY (id),
+    UNIQUE (alpha3_b)
 );
 
 /**
   Subtitles
  */
-CREATE TABLE IF NOT EXISTS subtitle
+CREATE TABLE IF NOT EXISTS subtitles
 (
     id            int GENERATED ALWAYS AS IDENTITY,
     film_id       int         NOT NULL,
@@ -52,15 +53,15 @@ CREATE TABLE IF NOT EXISTS subtitle
     UNIQUE (film_id, language_id),
     PRIMARY KEY (id),
     FOREIGN KEY (film_id)
-        REFERENCES film (id),
+        REFERENCES films (id),
     FOREIGN KEY (language_id)
-        REFERENCES language (id)
+        REFERENCES languages (id)
 );
 
 /**
   Subtitle lines
  */
-CREATE TABLE IF NOT EXISTS subtitle_line
+CREATE TABLE IF NOT EXISTS subtitle_lines
 (
     id          int GENERATED ALWAYS AS IDENTITY,
     subtitle_id int  NOT NULL,
@@ -68,5 +69,5 @@ CREATE TABLE IF NOT EXISTS subtitle_line
     time_code   text NOT NULL,
     PRIMARY KEY (id),
     FOREIGN KEY (subtitle_id)
-        REFERENCES subtitle (id)
+        REFERENCES subtitles (id)
 );

build.gradle

@@ -34,6 +34,7 @@ dependencies {
     implementation 'com.discord4j:discord4j-core:3.1.1'
     implementation 'com.github.wtekiela:opensub4j:0.3.0'
     implementation 'org.apache.commons:commons-lang3:3.11'
+    implementation 'org.apache.commons:commons-csv:1.8'
     implementation 'ch.qos.logback:logback-classic:1.2.3'
     implementation 'org.postgresql:postgresql:42.2.18.jre7'
 }

src/main/java/xyz/vallat/louis/MovieQuoteBot.java

@@ -43,6 +43,7 @@ public class MovieQuoteBot {
 
     public static void main(String[] args) {
         DBManager.testConnection();
+        DBManager.initDatabase();
         // TODO: FIX CRASH ON LOGIN IF OS IS IN MAINTENANCE OR BROKEN
         OpenSubtitles.login(
                 System.getenv(OS_USERNAME_ENVIRONMENT),

src/main/java/xyz/vallat/louis/database/DBManager.java

@@ -4,11 +4,16 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import xyz.vallat.louis.MovieQuoteBot;
 
+import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.sql.Statement;
 
+import static xyz.vallat.louis.database.LanguageManager.importLanguageIfNeeded;
+import static xyz.vallat.louis.database.LanguageManager.initializeLanguages;
+
 public final class DBManager {
 
     public static final String DB_USERNAME_ENVIRONMENT = "DB_USERNAME";
@@ -17,7 +22,6 @@ public final class DBManager {
     public static final String DB_HOST_ENVIRONMENT = "DB_HOST";
     public static final String DB_NAME_ENVIRONMENT = "DB_NAME";
     private static final Logger logger = LoggerFactory.getLogger(MovieQuoteBot.class.getCanonicalName());
-    private static final String DRIVER_CLASS = "org.postgresql.Driver";
     private static final String CONNECTION_PREFIX = "postgresql";
     private static final String USERNAME = System.getenv(DB_USERNAME_ENVIRONMENT);
     private static final String PASSWORD = System.getenv(DB_PASSWORD_ENVIRONMENT);
@@ -32,10 +36,9 @@ public final class DBManager {
     // TODO: EXIT CODES AS ENUM
     private static Connection getConnection() {
         try {
-            Class.forName(DRIVER_CLASS);
             return DriverManager.getConnection("jdbc:" + CONNECTION_PREFIX + "://" + HOST + ":" + PORT + "/"
                     + DATABASE, USERNAME, PASSWORD);
-        } catch (ClassNotFoundException | SQLException e) {
+        } catch (SQLException e) {
             logger.error("Could not connect to database. Reason: {}.", e.getMessage());
             System.exit(4);
         }
@@ -52,21 +55,24 @@ public final class DBManager {
         logger.debug("Initializing database if not done yet.");
         Connection connection = getConnection();
         try {
-            initializeApplication(connection);
+            initializeProperties(connection);
+            initializeLanguages(connection);
             initializeFilm(connection);
             initializeSubtitle(connection);
             initializeSubtitleLine(connection);
-            initializeLanguages(connection);
+            importLanguageIfNeeded(connection);
         } catch (SQLException e) {
             logger.error("An error happened while initializing the database. Reason: {}", e.getMessage());
             System.exit(5);
+        } catch (NoSuchAlgorithmException | IOException e) {
+            e.printStackTrace();
         }
     }
 
-    private static void initializeApplication(Connection connection) throws SQLException {
+    private static void initializeProperties(Connection connection) throws SQLException {
-        logger.debug("Creating application table.");
+        logger.debug("Creating properties table.");
         try (Statement stmt = connection.createStatement()) {
-            String query = "CREATE TABLE IF NOT EXISTS application\n" +
+            String query = "CREATE TABLE IF NOT EXISTS properties\n" +
                     "(\n" +
                     "    id        int GENERATED ALWAYS AS IDENTITY,\n" +
                     "    app_key   text NOT NULL,\n" +
@@ -81,7 +87,7 @@ public final class DBManager {
     private static void initializeFilm(Connection connection) throws SQLException {
         logger.debug("Creating film table.");
         try (Statement stmt = connection.createStatement()) {
-            String query = "CREATE TABLE IF NOT EXISTS film\n" +
+            String query = "CREATE TABLE IF NOT EXISTS films\n" +
                     "(\n" +
                     "    id           int GENERATED ALWAYS AS IDENTITY,\n" +
                     "    imdb_id      varchar(10) NOT NULL,\n" +
@@ -97,7 +103,7 @@ public final class DBManager {
     private static void initializeSubtitle(Connection connection) throws SQLException {
         logger.debug("Creating subtitle table.");
         try (Statement stmt = connection.createStatement()) {
-            String query = "CREATE TABLE IF NOT EXISTS subtitle\n" +
+            String query = "CREATE TABLE IF NOT EXISTS subtitles\n" +
                     "(\n" +
                     "    id            int GENERATED ALWAYS AS IDENTITY,\n" +
                     "    film_id       int         NOT NULL,\n" +
@@ -107,9 +113,9 @@ public final class DBManager {
                     "    UNIQUE (film_id, language_id),\n" +
                     "    PRIMARY KEY (id),\n" +
                     "    FOREIGN KEY (film_id)\n" +
-                    "        REFERENCES Film (id),\n" +
+                    "        REFERENCES films (id),\n" +
                     "    FOREIGN KEY (language_id)\n" +
-                    "        REFERENCES language (id)\n" +
+                    "        REFERENCES languages (id)\n" +
                     ");";
             stmt.executeUpdate(query);
         }
@@ -118,7 +124,7 @@ public final class DBManager {
     private static void initializeSubtitleLine(Connection connection) throws SQLException {
         logger.debug("Creating subtitle_line table.");
         try (Statement stmt = connection.createStatement()) {
-            String query = "CREATE TABLE IF NOT EXISTS subtitle_line\n" +
+            String query = "CREATE TABLE IF NOT EXISTS subtitle_lines\n" +
                     "(\n" +
                     "    id          int GENERATED ALWAYS AS IDENTITY,\n" +
                     "    subtitle_id int  NOT NULL,\n" +
@@ -126,24 +132,7 @@ public final class DBManager {
                     "    time_code   text NOT NULL,\n" +
                     "    PRIMARY KEY (id),\n" +
                     "    FOREIGN KEY (subtitle_id)\n" +
-                    "        REFERENCES subtitle (id)\n" +
+                    "        REFERENCES subtitles (id)\n" +
-                    ");";
-            stmt.executeUpdate(query);
-        }
-    }
-
-    private static void initializeLanguages(Connection connection) throws SQLException {
-        logger.debug("Creating languages table.");
-        try (Statement stmt = connection.createStatement()) {
-            String query = "CREATE TABLE IF NOT EXISTS language\n" +
-                    "(\n" +
-                    "    id       int GENERATED ALWAYS AS IDENTITY,\n" +
-                    "    alpha3_b char(3) NOT NULL,\n" +
-                    "    alpha3_t char(3),\n" +
-                    "    alpha2   char(2),\n" +
-                    "    english  text    NOT NULL,\n" +
-                    "    french   text    NOT NULL,\n" +
-                    "    PRIMARY KEY (id)\n" +
                     ");";
             stmt.executeUpdate(query);
         }

src/main/java/xyz/vallat/louis/database/LanguageManager.java

@@ -0,0 +1,176 @@
+package xyz.vallat.louis.database;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+public final class LanguageManager {
+
+    private static final Logger logger = LoggerFactory.getLogger(LanguageManager.class.getCanonicalName());
+    private static final String HASH_KEY = "languages_hash";
+    private static final String LANGUAGES_FILE_NAME = "language-codes-full_csv.csv";
+
+    private LanguageManager() {
+    }
+
+    public static void importLanguageIfNeeded(Connection connection) throws SQLException, IOException, NoSuchAlgorithmException {
+        logger.debug("Checking if we need to import languages again.");
+        String storedHash = getStoredHash(connection);
+        logger.debug("Stored hash was '{}'.", storedHash);
+        String actualHash = getActualHash();
+        logger.debug("Actual hash is '{}'.", actualHash);
+        logger.info("Importing new language file.");
+        if (!storedHash.equals(actualHash)) importLanguageFile(connection);
+        logger.debug("Saving new hash in database.");
+        saveNewHash(connection, actualHash);
+    }
+
+    private static void saveNewHash(Connection connection, String actualHash) throws SQLException {
+        String query;
+        if (doesPropertyExist(connection, HASH_KEY)) query = "UPDATE properties SET app_value = ? WHERE app_key = ?;";
+        else query = "INSERT INTO properties(app_value, app_key) VALUES (?, ?)";
+        try (PreparedStatement stmt = connection.prepareStatement(query)) {
+            stmt.setString(1, actualHash);
+            stmt.setString(2, HASH_KEY);
+            stmt.execute();
+        }
+    }
+
+    private static boolean doesPropertyExist(Connection connection, String key) throws SQLException {
+        String query = "SELECT id FROM properties WHERE app_key=?;";
+        try (PreparedStatement stmt = connection.prepareStatement(query)){
+            stmt.setString(1, key);
+            stmt.execute();
+            return stmt.getResultSet().next();
+        }
+    }
+
+    private static void importLanguageFile(Connection connection) throws IOException, SQLException {
+        logger.debug("Reading and parsing CSV file.");
+        connection.setAutoCommit(false);
+        InputStream csvFile = LanguageManager.class.getClassLoader().getResourceAsStream(LANGUAGES_FILE_NAME);
+        assert csvFile != null;
+        CSVParser csvParser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(new InputStreamReader(csvFile));
+        for (CSVRecord record : csvParser)
+            importSanitizedLangInDatabase(connection,
+                    record.get("alpha3-b"),
+                    record.get("alpha3-t"),
+                    record.get("alpha2"),
+                    record.get("English"),
+                    record.get("French")
+            );
+        try {
+            logger.debug("Committing changes.");
+            connection.commit();
+        } catch (SQLException e) {
+            logger.debug("An error occurred while committing the transaction. Rolling back. " +
+                    "Reason: {}", e.getMessage());
+            connection.rollback();
+        } finally {
+            connection.setAutoCommit(true);
+        }
+    }
+
+    private static void importSanitizedLangInDatabase(Connection connection, String alpha3b, String alpha3t,
+                                                      String alpha2b, String english, String french) throws SQLException {
+        if (alpha3b.length() > 3) {
+            for (String newAlpha3b : alpha3b.split(String.valueOf(alpha3b.charAt(3))))
+                importSanitizedLangInDatabase(connection, newAlpha3b, alpha3t, alpha2b, english, french);
+        } else if (alpha3t.length() > 3) {
+            for (String newAlpha3t : alpha3t.split(String.valueOf(alpha3t.charAt(3))))
+                importSanitizedLangInDatabase(connection, alpha3t, newAlpha3t, alpha2b, english, french);
+        } else if (alpha2b.length() > 2) {
+            for (String newAlpha2b : alpha2b.split(String.valueOf(alpha2b.charAt(2))))
+                importSanitizedLangInDatabase(connection, alpha3b, alpha3t, newAlpha2b, english, french);
+        } else if (!isInDatabase(connection, alpha3b)) {
+            insertLang(connection, alpha3b, alpha3t, alpha2b, english, french);
+        }
+    }
+
+    private static boolean isInDatabase(Connection connection, String alpha3b) throws SQLException {
+        String query = "SELECT id FROM languages WHERE alpha3_b = ?;";
+        try (PreparedStatement stmt = connection.prepareStatement(query)){
+            stmt.setString(1, alpha3b);
+            stmt.execute();
+            return stmt.getResultSet().next();
+        }
+    }
+
+    private static void insertLang(Connection connection, String alpha3b, String alpha3t,
+                                   String alpha2b, String english, String french) throws SQLException {
+        String insert = "INSERT INTO languages(alpha3_b, alpha3_t, alpha2, english, french) VALUES(?, ?, ?, ?, ?);";
+        try (PreparedStatement stmt = connection.prepareStatement(insert)) {
+            stmt.setString(1, alpha3b);
+            stmt.setString(2, alpha3t);
+            stmt.setString(3, alpha2b);
+            stmt.setString(4, english);
+            stmt.setString(5, french);
+            stmt.executeUpdate();
+        }
+    }
+
+    private static String getStoredHash(Connection connection) throws SQLException {
+        logger.debug("Getting current hash");
+        String query = "SELECT app_value FROM properties WHERE app_key=?;";
+        try (PreparedStatement stmt = connection.prepareStatement(query)) {
+            stmt.setString(1, HASH_KEY);
+            stmt.execute();
+            if (stmt.getResultSet().next()) return stmt.getResultSet().getString(1);
+            else return "";
+        }
+    }
+
+    private static String getActualHash() throws NoSuchAlgorithmException, IOException {
+        logger.debug("Computing the actual language source file's hash.");
+        return getFileChecksum(MessageDigest.getInstance("MD5"));
+    }
+
+    private static String getFileChecksum(MessageDigest digest) throws IOException {
+        try (InputStream is = LanguageManager.class.getClassLoader().getResourceAsStream(LANGUAGES_FILE_NAME)) {
+            assert is != null;
+            byte[] byteArray = new byte[1024];
+            int bytesCount;
+
+            while ((bytesCount = is.read(byteArray)) != -1) {
+                digest.update(byteArray, 0, bytesCount);
+            }
+
+            byte[] bytes = digest.digest();
+            StringBuilder sb = new StringBuilder();
+            for (byte aByte : bytes)
+                sb.append(Integer.toString((aByte & 0xff) + 0x100, 16).substring(1));
+
+            return sb.toString();
+        }
+    }
+
+    public static void initializeLanguages(Connection connection) throws SQLException {
+        logger.debug("Creating language table.");
+        try (Statement stmt = connection.createStatement()) {
+            String query = "CREATE TABLE IF NOT EXISTS languages\n" +
+                    "(\n" +
+                    "    id       int GENERATED ALWAYS AS IDENTITY,\n" +
+                    "    alpha3_b char(3) NOT NULL,\n" +
+                    "    alpha3_t char(3),\n" +
+                    "    alpha2   char(2),\n" +
+                    "    english  text    NOT NULL,\n" +
+                    "    french   text    NOT NULL,\n" +
+                    "    PRIMARY KEY (id),\n" +
+                    "    UNIQUE (alpha3_b)\n" +
+                    ");";
+            stmt.executeUpdate(query);
+        }
+    }
+}

src/main/resources/logback.xml

@@ -35,7 +35,7 @@
 
     <!-- Here you can set the base logger level. If DEBUG is too chatty for you, you can use INFO -->
     <!-- Possible options are: ALL, TRACE, DEBUG, INFO, WARN, ERROR, OFF -->
-    <root level="INFO">
+    <root level="DEBUG">
         <appender-ref ref="CONSOLE"/>
         <appender-ref ref="FILE"/>
     </root>