diff --git a/MoviesQuoteBot.sql b/MoviesQuoteBot.sql index 1768a13..ba8e27a 100644 --- a/MoviesQuoteBot.sql +++ b/MoviesQuoteBot.sql @@ -1,9 +1,9 @@ -DROP TABLE IF EXISTS application, film, language, subtitle_line, subtitle; +DROP TABLE IF EXISTS properties, films, languages, subtitle_lines, subtitles; /** Store some information on the application. */ -CREATE TABLE IF NOT EXISTS application +CREATE TABLE IF NOT EXISTS properties ( id int GENERATED ALWAYS AS IDENTITY, app_key text NOT NULL, @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS application /** Film */ -CREATE TABLE IF NOT EXISTS film +CREATE TABLE IF NOT EXISTS films ( id int GENERATED ALWAYS AS IDENTITY, imdb_id varchar(10) NOT NULL, @@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS film /** Available languages */ -CREATE TABLE IF NOT EXISTS language +CREATE TABLE IF NOT EXISTS languages ( id int GENERATED ALWAYS AS IDENTITY, alpha3_b char(3) NOT NULL, @@ -36,13 +36,14 @@ CREATE TABLE IF NOT EXISTS language alpha2 char(2), english text NOT NULL, french text NOT NULL, - PRIMARY KEY (id) + PRIMARY KEY (id), + UNIQUE (alpha3_b) ); /** Subtitles */ -CREATE TABLE IF NOT EXISTS subtitle +CREATE TABLE IF NOT EXISTS subtitles ( id int GENERATED ALWAYS AS IDENTITY, film_id int NOT NULL, @@ -52,15 +53,15 @@ CREATE TABLE IF NOT EXISTS subtitle UNIQUE (film_id, language_id), PRIMARY KEY (id), FOREIGN KEY (film_id) - REFERENCES film (id), + REFERENCES films (id), FOREIGN KEY (language_id) - REFERENCES language (id) + REFERENCES languages (id) ); /** Subtitle lines */ -CREATE TABLE IF NOT EXISTS subtitle_line +CREATE TABLE IF NOT EXISTS subtitle_lines ( id int GENERATED ALWAYS AS IDENTITY, subtitle_id int NOT NULL, @@ -68,5 +69,5 @@ CREATE TABLE IF NOT EXISTS subtitle_line time_code text NOT NULL, PRIMARY KEY (id), FOREIGN KEY (subtitle_id) - REFERENCES subtitle (id) + REFERENCES subtitles (id) ); \ No newline at end of file diff --git a/build.gradle b/build.gradle index a1c5235..0b524be 100644 --- a/build.gradle +++ b/build.gradle @@ -34,6 +34,7 @@ dependencies { implementation 'com.discord4j:discord4j-core:3.1.1' implementation 'com.github.wtekiela:opensub4j:0.3.0' implementation 'org.apache.commons:commons-lang3:3.11' + implementation 'org.apache.commons:commons-csv:1.8' implementation 'ch.qos.logback:logback-classic:1.2.3' implementation 'org.postgresql:postgresql:42.2.18.jre7' } \ No newline at end of file diff --git a/src/main/java/xyz/vallat/louis/MovieQuoteBot.java b/src/main/java/xyz/vallat/louis/MovieQuoteBot.java index 954d03c..1e6e9aa 100644 --- a/src/main/java/xyz/vallat/louis/MovieQuoteBot.java +++ b/src/main/java/xyz/vallat/louis/MovieQuoteBot.java @@ -43,6 +43,7 @@ public class MovieQuoteBot { public static void main(String[] args) { DBManager.testConnection(); + DBManager.initDatabase(); // TODO: FIX CRASH ON LOGIN IF OS IS IN MAINTENANCE OR BROKEN OpenSubtitles.login( System.getenv(OS_USERNAME_ENVIRONMENT), diff --git a/src/main/java/xyz/vallat/louis/database/DBManager.java b/src/main/java/xyz/vallat/louis/database/DBManager.java index 1f5a81a..d0645ea 100644 --- a/src/main/java/xyz/vallat/louis/database/DBManager.java +++ b/src/main/java/xyz/vallat/louis/database/DBManager.java @@ -4,11 +4,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import xyz.vallat.louis.MovieQuoteBot; +import java.io.IOException; +import java.security.NoSuchAlgorithmException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import static xyz.vallat.louis.database.LanguageManager.importLanguageIfNeeded; +import static xyz.vallat.louis.database.LanguageManager.initializeLanguages; + public final class DBManager { public static final String DB_USERNAME_ENVIRONMENT = "DB_USERNAME"; @@ -17,7 +22,6 @@ public final class DBManager { public static final String DB_HOST_ENVIRONMENT = "DB_HOST"; public static final String DB_NAME_ENVIRONMENT = "DB_NAME"; private static final Logger logger = LoggerFactory.getLogger(MovieQuoteBot.class.getCanonicalName()); - private static final String DRIVER_CLASS = "org.postgresql.Driver"; private static final String CONNECTION_PREFIX = "postgresql"; private static final String USERNAME = System.getenv(DB_USERNAME_ENVIRONMENT); private static final String PASSWORD = System.getenv(DB_PASSWORD_ENVIRONMENT); @@ -32,10 +36,9 @@ public final class DBManager { // TODO: EXIT CODES AS ENUM private static Connection getConnection() { try { - Class.forName(DRIVER_CLASS); return DriverManager.getConnection("jdbc:" + CONNECTION_PREFIX + "://" + HOST + ":" + PORT + "/" + DATABASE, USERNAME, PASSWORD); - } catch (ClassNotFoundException | SQLException e) { + } catch (SQLException e) { logger.error("Could not connect to database. Reason: {}.", e.getMessage()); System.exit(4); } @@ -52,21 +55,24 @@ public final class DBManager { logger.debug("Initializing database if not done yet."); Connection connection = getConnection(); try { - initializeApplication(connection); + initializeProperties(connection); + initializeLanguages(connection); initializeFilm(connection); initializeSubtitle(connection); initializeSubtitleLine(connection); - initializeLanguages(connection); + importLanguageIfNeeded(connection); } catch (SQLException e) { logger.error("An error happened while initializing the database. Reason: {}", e.getMessage()); System.exit(5); + } catch (NoSuchAlgorithmException | IOException e) { + e.printStackTrace(); } } - private static void initializeApplication(Connection connection) throws SQLException { - logger.debug("Creating application table."); + private static void initializeProperties(Connection connection) throws SQLException { + logger.debug("Creating properties table."); try (Statement stmt = connection.createStatement()) { - String query = "CREATE TABLE IF NOT EXISTS application\n" + + String query = "CREATE TABLE IF NOT EXISTS properties\n" + "(\n" + " id int GENERATED ALWAYS AS IDENTITY,\n" + " app_key text NOT NULL,\n" + @@ -81,7 +87,7 @@ public final class DBManager { private static void initializeFilm(Connection connection) throws SQLException { logger.debug("Creating film table."); try (Statement stmt = connection.createStatement()) { - String query = "CREATE TABLE IF NOT EXISTS film\n" + + String query = "CREATE TABLE IF NOT EXISTS films\n" + "(\n" + " id int GENERATED ALWAYS AS IDENTITY,\n" + " imdb_id varchar(10) NOT NULL,\n" + @@ -97,7 +103,7 @@ public final class DBManager { private static void initializeSubtitle(Connection connection) throws SQLException { logger.debug("Creating subtitle table."); try (Statement stmt = connection.createStatement()) { - String query = "CREATE TABLE IF NOT EXISTS subtitle\n" + + String query = "CREATE TABLE IF NOT EXISTS subtitles\n" + "(\n" + " id int GENERATED ALWAYS AS IDENTITY,\n" + " film_id int NOT NULL,\n" + @@ -107,9 +113,9 @@ public final class DBManager { " UNIQUE (film_id, language_id),\n" + " PRIMARY KEY (id),\n" + " FOREIGN KEY (film_id)\n" + - " REFERENCES Film (id),\n" + + " REFERENCES films (id),\n" + " FOREIGN KEY (language_id)\n" + - " REFERENCES language (id)\n" + + " REFERENCES languages (id)\n" + ");"; stmt.executeUpdate(query); } @@ -118,7 +124,7 @@ public final class DBManager { private static void initializeSubtitleLine(Connection connection) throws SQLException { logger.debug("Creating subtitle_line table."); try (Statement stmt = connection.createStatement()) { - String query = "CREATE TABLE IF NOT EXISTS subtitle_line\n" + + String query = "CREATE TABLE IF NOT EXISTS subtitle_lines\n" + "(\n" + " id int GENERATED ALWAYS AS IDENTITY,\n" + " subtitle_id int NOT NULL,\n" + @@ -126,24 +132,7 @@ public final class DBManager { " time_code text NOT NULL,\n" + " PRIMARY KEY (id),\n" + " FOREIGN KEY (subtitle_id)\n" + - " REFERENCES subtitle (id)\n" + - ");"; - stmt.executeUpdate(query); - } - } - - private static void initializeLanguages(Connection connection) throws SQLException { - logger.debug("Creating languages table."); - try (Statement stmt = connection.createStatement()) { - String query = "CREATE TABLE IF NOT EXISTS language\n" + - "(\n" + - " id int GENERATED ALWAYS AS IDENTITY,\n" + - " alpha3_b char(3) NOT NULL,\n" + - " alpha3_t char(3),\n" + - " alpha2 char(2),\n" + - " english text NOT NULL,\n" + - " french text NOT NULL,\n" + - " PRIMARY KEY (id)\n" + + " REFERENCES subtitles (id)\n" + ");"; stmt.executeUpdate(query); } diff --git a/src/main/java/xyz/vallat/louis/database/LanguageManager.java b/src/main/java/xyz/vallat/louis/database/LanguageManager.java new file mode 100644 index 0000000..3847ad0 --- /dev/null +++ b/src/main/java/xyz/vallat/louis/database/LanguageManager.java @@ -0,0 +1,176 @@ +package xyz.vallat.louis.database; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; + +public final class LanguageManager { + + private static final Logger logger = LoggerFactory.getLogger(LanguageManager.class.getCanonicalName()); + private static final String HASH_KEY = "languages_hash"; + private static final String LANGUAGES_FILE_NAME = "language-codes-full_csv.csv"; + + private LanguageManager() { + } + + public static void importLanguageIfNeeded(Connection connection) throws SQLException, IOException, NoSuchAlgorithmException { + logger.debug("Checking if we need to import languages again."); + String storedHash = getStoredHash(connection); + logger.debug("Stored hash was '{}'.", storedHash); + String actualHash = getActualHash(); + logger.debug("Actual hash is '{}'.", actualHash); + logger.info("Importing new language file."); + if (!storedHash.equals(actualHash)) importLanguageFile(connection); + logger.debug("Saving new hash in database."); + saveNewHash(connection, actualHash); + } + + private static void saveNewHash(Connection connection, String actualHash) throws SQLException { + String query; + if (doesPropertyExist(connection, HASH_KEY)) query = "UPDATE properties SET app_value = ? WHERE app_key = ?;"; + else query = "INSERT INTO properties(app_value, app_key) VALUES (?, ?)"; + try (PreparedStatement stmt = connection.prepareStatement(query)) { + stmt.setString(1, actualHash); + stmt.setString(2, HASH_KEY); + stmt.execute(); + } + } + + private static boolean doesPropertyExist(Connection connection, String key) throws SQLException { + String query = "SELECT id FROM properties WHERE app_key=?;"; + try (PreparedStatement stmt = connection.prepareStatement(query)){ + stmt.setString(1, key); + stmt.execute(); + return stmt.getResultSet().next(); + } + } + + private static void importLanguageFile(Connection connection) throws IOException, SQLException { + logger.debug("Reading and parsing CSV file."); + connection.setAutoCommit(false); + InputStream csvFile = LanguageManager.class.getClassLoader().getResourceAsStream(LANGUAGES_FILE_NAME); + assert csvFile != null; + CSVParser csvParser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(new InputStreamReader(csvFile)); + for (CSVRecord record : csvParser) + importSanitizedLangInDatabase(connection, + record.get("alpha3-b"), + record.get("alpha3-t"), + record.get("alpha2"), + record.get("English"), + record.get("French") + ); + try { + logger.debug("Committing changes."); + connection.commit(); + } catch (SQLException e) { + logger.debug("An error occurred while committing the transaction. Rolling back. " + + "Reason: {}", e.getMessage()); + connection.rollback(); + } finally { + connection.setAutoCommit(true); + } + } + + private static void importSanitizedLangInDatabase(Connection connection, String alpha3b, String alpha3t, + String alpha2b, String english, String french) throws SQLException { + if (alpha3b.length() > 3) { + for (String newAlpha3b : alpha3b.split(String.valueOf(alpha3b.charAt(3)))) + importSanitizedLangInDatabase(connection, newAlpha3b, alpha3t, alpha2b, english, french); + } else if (alpha3t.length() > 3) { + for (String newAlpha3t : alpha3t.split(String.valueOf(alpha3t.charAt(3)))) + importSanitizedLangInDatabase(connection, alpha3t, newAlpha3t, alpha2b, english, french); + } else if (alpha2b.length() > 2) { + for (String newAlpha2b : alpha2b.split(String.valueOf(alpha2b.charAt(2)))) + importSanitizedLangInDatabase(connection, alpha3b, alpha3t, newAlpha2b, english, french); + } else if (!isInDatabase(connection, alpha3b)) { + insertLang(connection, alpha3b, alpha3t, alpha2b, english, french); + } + } + + private static boolean isInDatabase(Connection connection, String alpha3b) throws SQLException { + String query = "SELECT id FROM languages WHERE alpha3_b = ?;"; + try (PreparedStatement stmt = connection.prepareStatement(query)){ + stmt.setString(1, alpha3b); + stmt.execute(); + return stmt.getResultSet().next(); + } + } + + private static void insertLang(Connection connection, String alpha3b, String alpha3t, + String alpha2b, String english, String french) throws SQLException { + String insert = "INSERT INTO languages(alpha3_b, alpha3_t, alpha2, english, french) VALUES(?, ?, ?, ?, ?);"; + try (PreparedStatement stmt = connection.prepareStatement(insert)) { + stmt.setString(1, alpha3b); + stmt.setString(2, alpha3t); + stmt.setString(3, alpha2b); + stmt.setString(4, english); + stmt.setString(5, french); + stmt.executeUpdate(); + } + } + + private static String getStoredHash(Connection connection) throws SQLException { + logger.debug("Getting current hash"); + String query = "SELECT app_value FROM properties WHERE app_key=?;"; + try (PreparedStatement stmt = connection.prepareStatement(query)) { + stmt.setString(1, HASH_KEY); + stmt.execute(); + if (stmt.getResultSet().next()) return stmt.getResultSet().getString(1); + else return ""; + } + } + + private static String getActualHash() throws NoSuchAlgorithmException, IOException { + logger.debug("Computing the actual language source file's hash."); + return getFileChecksum(MessageDigest.getInstance("MD5")); + } + + private static String getFileChecksum(MessageDigest digest) throws IOException { + try (InputStream is = LanguageManager.class.getClassLoader().getResourceAsStream(LANGUAGES_FILE_NAME)) { + assert is != null; + byte[] byteArray = new byte[1024]; + int bytesCount; + + while ((bytesCount = is.read(byteArray)) != -1) { + digest.update(byteArray, 0, bytesCount); + } + + byte[] bytes = digest.digest(); + StringBuilder sb = new StringBuilder(); + for (byte aByte : bytes) + sb.append(Integer.toString((aByte & 0xff) + 0x100, 16).substring(1)); + + return sb.toString(); + } + } + + public static void initializeLanguages(Connection connection) throws SQLException { + logger.debug("Creating language table."); + try (Statement stmt = connection.createStatement()) { + String query = "CREATE TABLE IF NOT EXISTS languages\n" + + "(\n" + + " id int GENERATED ALWAYS AS IDENTITY,\n" + + " alpha3_b char(3) NOT NULL,\n" + + " alpha3_t char(3),\n" + + " alpha2 char(2),\n" + + " english text NOT NULL,\n" + + " french text NOT NULL,\n" + + " PRIMARY KEY (id),\n" + + " UNIQUE (alpha3_b)\n" + + ");"; + stmt.executeUpdate(query); + } + } +} diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 5c3704d..2319ec6 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -35,7 +35,7 @@ - +