louis/moviequotebot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added a way to update and setup languages

Browse Source 
Signed-off-by: Louis Vallat <louis@louis-vallat.xyz>
This commit is contained in:
Louis Vallat 2020-10-27 21:38:19 +01:00
parent ee08b6b29c
commit 8f1ae7a5fe
6 changed files with 210 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
MoviesQuoteBot.sql
View File

@ -1,9 +1,9 @@
DROP TABLE IF EXISTS application, film, language, subtitle_line, subtitle;
DROP TABLE IF EXISTS properties, films, languages, subtitle_lines, subtitles;
/**
Store some information on the application.
*/
CREATE TABLE IF NOT EXISTS application
CREATE TABLE IF NOT EXISTS properties
(
id int GENERATED ALWAYS AS IDENTITY,
app_key text NOT NULL,
@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS application
/**
Film
*/
CREATE TABLE IF NOT EXISTS film
CREATE TABLE IF NOT EXISTS films
(
id int GENERATED ALWAYS AS IDENTITY,
imdb_id varchar(10) NOT NULL,
@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS film
/**
Available languages
*/
CREATE TABLE IF NOT EXISTS language
CREATE TABLE IF NOT EXISTS languages
(
id int GENERATED ALWAYS AS IDENTITY,
alpha3_b char(3) NOT NULL,
@ -36,13 +36,14 @@ CREATE TABLE IF NOT EXISTS language
alpha2 char(2),
english text NOT NULL,
french text NOT NULL,
PRIMARY KEY (id)
PRIMARY KEY (id),
UNIQUE (alpha3_b)
);
/**
Subtitles
*/
CREATE TABLE IF NOT EXISTS subtitle
CREATE TABLE IF NOT EXISTS subtitles
(
id int GENERATED ALWAYS AS IDENTITY,
film_id int NOT NULL,
@ -52,15 +53,15 @@ CREATE TABLE IF NOT EXISTS subtitle
UNIQUE (film_id, language_id),
PRIMARY KEY (id),
FOREIGN KEY (film_id)
REFERENCES film (id),
REFERENCES films (id),
FOREIGN KEY (language_id)
REFERENCES language (id)
REFERENCES languages (id)
);
/**
Subtitle lines
*/
CREATE TABLE IF NOT EXISTS subtitle_line
CREATE TABLE IF NOT EXISTS subtitle_lines
(
id int GENERATED ALWAYS AS IDENTITY,
subtitle_id int NOT NULL,
@ -68,5 +69,5 @@ CREATE TABLE IF NOT EXISTS subtitle_line
time_code text NOT NULL,
PRIMARY KEY (id),
FOREIGN KEY (subtitle_id)
REFERENCES subtitle (id)
REFERENCES subtitles (id)
);

1
build.gradle
View File

@ -34,6 +34,7 @@ dependencies {
implementation 'com.discord4j:discord4j-core:3.1.1'
implementation 'com.github.wtekiela:opensub4j:0.3.0'
implementation 'org.apache.commons:commons-lang3:3.11'
implementation 'org.apache.commons:commons-csv:1.8'
implementation 'ch.qos.logback:logback-classic:1.2.3'
implementation 'org.postgresql:postgresql:42.2.18.jre7'
}

1
src/main/java/xyz/vallat/louis/MovieQuoteBot.java
View File

@ -43,6 +43,7 @@ public class MovieQuoteBot {
public static void main(String[] args) {
DBManager.testConnection();
DBManager.initDatabase();
// TODO: FIX CRASH ON LOGIN IF OS IS IN MAINTENANCE OR BROKEN
OpenSubtitles.login(
System.getenv(OS_USERNAME_ENVIRONMENT),

51
src/main/java/xyz/vallat/louis/database/DBManager.java
View File

@ -4,11 +4,16 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import xyz.vallat.louis.MovieQuoteBot;
import java.io.IOException;
import java.security.NoSuchAlgorithmException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import static xyz.vallat.louis.database.LanguageManager.importLanguageIfNeeded;
import static xyz.vallat.louis.database.LanguageManager.initializeLanguages;
public final class DBManager {
public static final String DB_USERNAME_ENVIRONMENT = "DB_USERNAME";
@ -17,7 +22,6 @@ public final class DBManager {
public static final String DB_HOST_ENVIRONMENT = "DB_HOST";
public static final String DB_NAME_ENVIRONMENT = "DB_NAME";
private static final Logger logger = LoggerFactory.getLogger(MovieQuoteBot.class.getCanonicalName());
private static final String DRIVER_CLASS = "org.postgresql.Driver";
private static final String CONNECTION_PREFIX = "postgresql";
private static final String USERNAME = System.getenv(DB_USERNAME_ENVIRONMENT);
private static final String PASSWORD = System.getenv(DB_PASSWORD_ENVIRONMENT);
@ -32,10 +36,9 @@ public final class DBManager {
// TODO: EXIT CODES AS ENUM
private static Connection getConnection() {
try {
Class.forName(DRIVER_CLASS);
return DriverManager.getConnection("jdbc:" + CONNECTION_PREFIX + "://" + HOST + ":" + PORT + "/"
+ DATABASE, USERNAME, PASSWORD);
} catch (ClassNotFoundException | SQLException e) {
} catch (SQLException e) {
logger.error("Could not connect to database. Reason: {}.", e.getMessage());
System.exit(4);
}
@ -52,21 +55,24 @@ public final class DBManager {
logger.debug("Initializing database if not done yet.");
Connection connection = getConnection();
try {
initializeApplication(connection);
initializeProperties(connection);
initializeLanguages(connection);
initializeFilm(connection);
initializeSubtitle(connection);
initializeSubtitleLine(connection);
initializeLanguages(connection);
importLanguageIfNeeded(connection);
} catch (SQLException e) {
logger.error("An error happened while initializing the database. Reason: {}", e.getMessage());
System.exit(5);
} catch (NoSuchAlgorithmException | IOException e) {
e.printStackTrace();
}
}
private static void initializeApplication(Connection connection) throws SQLException {
logger.debug("Creating application table.");
private static void initializeProperties(Connection connection) throws SQLException {
logger.debug("Creating properties table.");
try (Statement stmt = connection.createStatement()) {
String query = "CREATE TABLE IF NOT EXISTS application\n" +
String query = "CREATE TABLE IF NOT EXISTS properties\n" +
"(\n" +
" id int GENERATED ALWAYS AS IDENTITY,\n" +
" app_key text NOT NULL,\n" +
@ -81,7 +87,7 @@ public final class DBManager {
private static void initializeFilm(Connection connection) throws SQLException {
logger.debug("Creating film table.");
try (Statement stmt = connection.createStatement()) {
String query = "CREATE TABLE IF NOT EXISTS film\n" +
String query = "CREATE TABLE IF NOT EXISTS films\n" +
"(\n" +
" id int GENERATED ALWAYS AS IDENTITY,\n" +
" imdb_id varchar(10) NOT NULL,\n" +
@ -97,7 +103,7 @@ public final class DBManager {
private static void initializeSubtitle(Connection connection) throws SQLException {
logger.debug("Creating subtitle table.");
try (Statement stmt = connection.createStatement()) {
String query = "CREATE TABLE IF NOT EXISTS subtitle\n" +
String query = "CREATE TABLE IF NOT EXISTS subtitles\n" +
"(\n" +
" id int GENERATED ALWAYS AS IDENTITY,\n" +
" film_id int NOT NULL,\n" +
@ -107,9 +113,9 @@ public final class DBManager {
" UNIQUE (film_id, language_id),\n" +
" PRIMARY KEY (id),\n" +
" FOREIGN KEY (film_id)\n" +
" REFERENCES Film (id),\n" +
" REFERENCES films (id),\n" +
" FOREIGN KEY (language_id)\n" +
" REFERENCES language (id)\n" +
" REFERENCES languages (id)\n" +
");";
stmt.executeUpdate(query);
}
@ -118,7 +124,7 @@ public final class DBManager {
private static void initializeSubtitleLine(Connection connection) throws SQLException {
logger.debug("Creating subtitle_line table.");
try (Statement stmt = connection.createStatement()) {
String query = "CREATE TABLE IF NOT EXISTS subtitle_line\n" +
String query = "CREATE TABLE IF NOT EXISTS subtitle_lines\n" +
"(\n" +
" id int GENERATED ALWAYS AS IDENTITY,\n" +
" subtitle_id int NOT NULL,\n" +
@ -126,24 +132,7 @@ public final class DBManager {
" time_code text NOT NULL,\n" +
" PRIMARY KEY (id),\n" +
" FOREIGN KEY (subtitle_id)\n" +
" REFERENCES subtitle (id)\n" +
");";
stmt.executeUpdate(query);
}
}
private static void initializeLanguages(Connection connection) throws SQLException {
logger.debug("Creating languages table.");
try (Statement stmt = connection.createStatement()) {
String query = "CREATE TABLE IF NOT EXISTS language\n" +
"(\n" +
" id int GENERATED ALWAYS AS IDENTITY,\n" +
" alpha3_b char(3) NOT NULL,\n" +
" alpha3_t char(3),\n" +
" alpha2 char(2),\n" +
" english text NOT NULL,\n" +
" french text NOT NULL,\n" +
" PRIMARY KEY (id)\n" +
" REFERENCES subtitles (id)\n" +
");";
stmt.executeUpdate(query);
}

176
src/main/java/xyz/vallat/louis/database/LanguageManager.java Normal file
View File

@ -0,0 +1,176 @@
package xyz.vallat.louis.database;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
public final class LanguageManager {
private static final Logger logger = LoggerFactory.getLogger(LanguageManager.class.getCanonicalName());
private static final String HASH_KEY = "languages_hash";
private static final String LANGUAGES_FILE_NAME = "language-codes-full_csv.csv";
private LanguageManager() {
}
public static void importLanguageIfNeeded(Connection connection) throws SQLException, IOException, NoSuchAlgorithmException {
logger.debug("Checking if we need to import languages again.");
String storedHash = getStoredHash(connection);
logger.debug("Stored hash was '{}'.", storedHash);
String actualHash = getActualHash();
logger.debug("Actual hash is '{}'.", actualHash);
logger.info("Importing new language file.");
if (!storedHash.equals(actualHash)) importLanguageFile(connection);
logger.debug("Saving new hash in database.");
saveNewHash(connection, actualHash);
}
private static void saveNewHash(Connection connection, String actualHash) throws SQLException {
String query;
if (doesPropertyExist(connection, HASH_KEY)) query = "UPDATE properties SET app_value = ? WHERE app_key = ?;";
else query = "INSERT INTO properties(app_value, app_key) VALUES (?, ?)";
try (PreparedStatement stmt = connection.prepareStatement(query)) {
stmt.setString(1, actualHash);
stmt.setString(2, HASH_KEY);
stmt.execute();
}
}
private static boolean doesPropertyExist(Connection connection, String key) throws SQLException {
String query = "SELECT id FROM properties WHERE app_key=?;";
try (PreparedStatement stmt = connection.prepareStatement(query)){
stmt.setString(1, key);
stmt.execute();
return stmt.getResultSet().next();
}
}
private static void importLanguageFile(Connection connection) throws IOException, SQLException {
logger.debug("Reading and parsing CSV file.");
connection.setAutoCommit(false);
InputStream csvFile = LanguageManager.class.getClassLoader().getResourceAsStream(LANGUAGES_FILE_NAME);
assert csvFile != null;
CSVParser csvParser = CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(new InputStreamReader(csvFile));
for (CSVRecord record : csvParser)
importSanitizedLangInDatabase(connection,
record.get("alpha3-b"),
record.get("alpha3-t"),
record.get("alpha2"),
record.get("English"),
record.get("French")
);
try {
logger.debug("Committing changes.");
connection.commit();
} catch (SQLException e) {
logger.debug("An error occurred while committing the transaction. Rolling back. " +
"Reason: {}", e.getMessage());
connection.rollback();
} finally {
connection.setAutoCommit(true);
}
}
private static void importSanitizedLangInDatabase(Connection connection, String alpha3b, String alpha3t,
String alpha2b, String english, String french) throws SQLException {
if (alpha3b.length() > 3) {
for (String newAlpha3b : alpha3b.split(String.valueOf(alpha3b.charAt(3))))
importSanitizedLangInDatabase(connection, newAlpha3b, alpha3t, alpha2b, english, french);
} else if (alpha3t.length() > 3) {
for (String newAlpha3t : alpha3t.split(String.valueOf(alpha3t.charAt(3))))
importSanitizedLangInDatabase(connection, alpha3t, newAlpha3t, alpha2b, english, french);
} else if (alpha2b.length() > 2) {
for (String newAlpha2b : alpha2b.split(String.valueOf(alpha2b.charAt(2))))
importSanitizedLangInDatabase(connection, alpha3b, alpha3t, newAlpha2b, english, french);
} else if (!isInDatabase(connection, alpha3b)) {
insertLang(connection, alpha3b, alpha3t, alpha2b, english, french);
}
}
private static boolean isInDatabase(Connection connection, String alpha3b) throws SQLException {
String query = "SELECT id FROM languages WHERE alpha3_b = ?;";
try (PreparedStatement stmt = connection.prepareStatement(query)){
stmt.setString(1, alpha3b);
stmt.execute();
return stmt.getResultSet().next();
}
}
private static void insertLang(Connection connection, String alpha3b, String alpha3t,
String alpha2b, String english, String french) throws SQLException {
String insert = "INSERT INTO languages(alpha3_b, alpha3_t, alpha2, english, french) VALUES(?, ?, ?, ?, ?);";
try (PreparedStatement stmt = connection.prepareStatement(insert)) {
stmt.setString(1, alpha3b);
stmt.setString(2, alpha3t);
stmt.setString(3, alpha2b);
stmt.setString(4, english);
stmt.setString(5, french);
stmt.executeUpdate();
}
}
private static String getStoredHash(Connection connection) throws SQLException {
logger.debug("Getting current hash");
String query = "SELECT app_value FROM properties WHERE app_key=?;";
try (PreparedStatement stmt = connection.prepareStatement(query)) {
stmt.setString(1, HASH_KEY);
stmt.execute();
if (stmt.getResultSet().next()) return stmt.getResultSet().getString(1);
else return "";
}
}
private static String getActualHash() throws NoSuchAlgorithmException, IOException {
logger.debug("Computing the actual language source file's hash.");
return getFileChecksum(MessageDigest.getInstance("MD5"));
}
private static String getFileChecksum(MessageDigest digest) throws IOException {
try (InputStream is = LanguageManager.class.getClassLoader().getResourceAsStream(LANGUAGES_FILE_NAME)) {
assert is != null;
byte[] byteArray = new byte[1024];
int bytesCount;
while ((bytesCount = is.read(byteArray)) != -1) {
digest.update(byteArray, 0, bytesCount);
}
byte[] bytes = digest.digest();
StringBuilder sb = new StringBuilder();
for (byte aByte : bytes)
sb.append(Integer.toString((aByte & 0xff) + 0x100, 16).substring(1));
return sb.toString();
}
}
public static void initializeLanguages(Connection connection) throws SQLException {
logger.debug("Creating language table.");
try (Statement stmt = connection.createStatement()) {
String query = "CREATE TABLE IF NOT EXISTS languages\n" +
"(\n" +
" id int GENERATED ALWAYS AS IDENTITY,\n" +
" alpha3_b char(3) NOT NULL,\n" +
" alpha3_t char(3),\n" +
" alpha2 char(2),\n" +
" english text NOT NULL,\n" +
" french text NOT NULL,\n" +
" PRIMARY KEY (id),\n" +
" UNIQUE (alpha3_b)\n" +
");";
stmt.executeUpdate(query);
}
}
}

2
src/main/resources/logback.xml
View File

@ -35,7 +35,7 @@
<!-- Here you can set the base logger level. If DEBUG is too chatty for you, you can use INFO -->
<!-- Possible options are: ALL, TRACE, DEBUG, INFO, WARN, ERROR, OFF -->
<root level="INFO">
<root level="DEBUG">
<appender-ref ref="CONSOLE"/>
<appender-ref ref="FILE"/>
</root>