From 2f5195ad0aa4b34a14d5aef3639765b30b9e69e9 Mon Sep 17 00:00:00 2001 From: Louis Vallat Date: Thu, 29 Oct 2020 11:51:19 +0100 Subject: [PATCH] Downloading subtitles is now kinda working Signed-off-by: Louis Vallat --- MoviesQuoteBot.sql | 2 +- .../xyz/vallat/louis/commands/Command.java | 31 +++++----- .../xyz/vallat/louis/commands/Download.java | 61 ++++++++++++++----- .../vallat/louis/database/FilmManager.java | 2 +- .../louis/database/LanguageManager.java | 17 ++++++ .../louis/database/SubtitleLineManager.java | 40 +++++++++++- .../louis/database/SubtitleManager.java | 25 +++++++- .../xyz/vallat/louis/omdb/OMDBClient.java | 11 ++++ .../vallat/louis/subtitles/OpenSubtitles.java | 39 ++++++++++-- 9 files changed, 187 insertions(+), 41 deletions(-) diff --git a/MoviesQuoteBot.sql b/MoviesQuoteBot.sql index 82df511..27add9d 100644 --- a/MoviesQuoteBot.sql +++ b/MoviesQuoteBot.sql @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS subtitles id int GENERATED ALWAYS AS IDENTITY, film_id int NOT NULL, language_id int NOT NULL, - importer varchar(37), + importer_id varchar(37), importer_guild_id text, imported_date timestamptz NOT NULL DEFAULT now(), UNIQUE (film_id, language_id), diff --git a/src/main/java/xyz/vallat/louis/commands/Command.java b/src/main/java/xyz/vallat/louis/commands/Command.java index 6dbb716..23a8cc6 100644 --- a/src/main/java/xyz/vallat/louis/commands/Command.java +++ b/src/main/java/xyz/vallat/louis/commands/Command.java @@ -25,10 +25,11 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -public abstract class Command { - private static final Logger logger = LoggerFactory.getLogger(ListLangMovie.class.getCanonicalName()); - protected static final ReactionEmoji WAITING = ReactionEmoji.unicode("⌛"); +import static xyz.vallat.louis.subtitles.OpenSubtitles.getSubtitleStream; +public abstract class Command { + protected static final ReactionEmoji WAITING = ReactionEmoji.unicode("⌛"); + private static final Logger logger = LoggerFactory.getLogger(ListLangMovie.class.getCanonicalName()); protected final String name; protected final String description; protected final String usage; @@ -79,6 +80,13 @@ public abstract class Command { .then(); } + protected Mono unknownLanguage(MessageCreateEvent event, String lang) { + return event.getMessage().getChannel() + .flatMap(channel -> + channel.createMessage("Error, this lang isn't recognized: '" + lang + "'.\n Usage: " + getUsage())) + .then(); + } + protected void createEmbedListLang(String arg, EmbedCreateSpec embed, boolean isId) { try { Movie movie = OMDBClient.getMovie(arg, isId); @@ -111,8 +119,9 @@ public abstract class Command { private void handleOk(String arg, EmbedCreateSpec embed, boolean isId, Movie movie, ListResponse subtitles) { embed.setColor(Color.BISMARK); - Stream subtitlesStream = getSubtitleStream(subtitles); - String formattedSubtitles = subtitlesStream.collect(Collectors.joining("\n")); + Stream subtitlesStream = getSubtitleStream(subtitles.getData()) + .map(subtitleInfo -> subtitleInfo.getSubtitleFileId() + " - " + subtitleInfo.getLanguage()); + String formattedSubtitles = subtitlesStream.limit(20).collect(Collectors.joining("\n")); embed.setDescription("You requested a list of the languages available for a movie with the " + (isId ? "IMDB Identifier" : "title") + " '" + arg + "'. "); embed.addField("Movie", movie.toString(), true); @@ -122,18 +131,6 @@ public abstract class Command { formattedSubtitles.isBlank() ? "None." : formattedSubtitles, false); } - private Stream getSubtitleStream(ListResponse subtitles) { - return subtitles.getData().stream() - .sorted(Comparator.comparingInt(SubtitleInfo::getDownloadsNo)) - .filter(distinctByKey(SubtitleInfo::getLanguage)) - .filter(subtitleInfo -> - (subtitleInfo.getEncoding().equalsIgnoreCase("ascii") - || subtitleInfo.getEncoding().equalsIgnoreCase("cp1252"))) - .map(subtitleInfo -> subtitleInfo.getSubtitleFileId() - + " - " + subtitleInfo.getLanguage()) - .limit(20); - } - protected List getArguments(MessageCreateEvent event) { return List.of(event.getMessage().getContent().substring(Math.min(name.length() + 1, name.length())).split(" ")); } diff --git a/src/main/java/xyz/vallat/louis/commands/Download.java b/src/main/java/xyz/vallat/louis/commands/Download.java index 1c76078..083e02c 100644 --- a/src/main/java/xyz/vallat/louis/commands/Download.java +++ b/src/main/java/xyz/vallat/louis/commands/Download.java @@ -3,48 +3,81 @@ package xyz.vallat.louis.commands; import com.github.wtekiela.opensub4j.response.ListResponse; import com.github.wtekiela.opensub4j.response.ResponseStatus; import com.github.wtekiela.opensub4j.response.SubtitleFile; +import com.github.wtekiela.opensub4j.response.SubtitleInfo; import discord4j.core.event.domain.message.MessageCreateEvent; import discord4j.rest.util.Color; -import org.apache.commons.lang3.StringUtils; import org.apache.xmlrpc.XmlRpcException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import reactor.core.publisher.Mono; +import xyz.vallat.louis.database.LanguageManager; import xyz.vallat.louis.database.SubtitleLineManager; +import xyz.vallat.louis.database.SubtitleManager; +import xyz.vallat.louis.omdb.OMDBClient; +import xyz.vallat.louis.omdb.objects.Movie; import xyz.vallat.louis.subtitles.OpenSubtitles; +import xyz.vallat.louis.subtitles.parser.SubtitleBlock; import xyz.vallat.louis.subtitles.parser.SubtitleParser; +import java.io.IOException; import java.time.Instant; import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; public class Download extends Command { private static final Logger logger = LoggerFactory.getLogger(Download.class.getCanonicalName()); public Download(String name) { - super(name, "Download a subtitle.", name + " subtitleId", 1, 1); + super(name, "Download a subtitle.", name + " lang imdb|title value", 3, 3); } @Override public Mono execute(MessageCreateEvent event) { List args = getArguments(event); - if (args.isEmpty()) return notEnoughArguments(event); - else if (!StringUtils.isNumeric(args.get(0))) return badArgument(event, args.get(0)); + if (args.size() < minArgs) return notEnoughArguments(event); + String value = String.join(" ", args.subList(3, args.size())); + if (!args.get(2).equalsIgnoreCase("imdb") && !args.get(2).equalsIgnoreCase("title")) + return badArgument(event, args.get(2)); + int languageId = LanguageManager.getIdLanguage(args.get(1)); + if (languageId < 0) return unknownLanguage(event, args.get(1)); return event.getMessage().getChannel().flatMap(channel -> event.getMessage().addReaction(WAITING).then( channel.createEmbed(embed -> { embed.setTitle("Importation").setColor(Color.RED); try { - ListResponse subs = OpenSubtitles.downloadSubtitle(Integer.parseInt(args.get(0))); - if (subs.getStatus().getCode() == ResponseStatus.OK.getCode()) { - SubtitleLineManager.importSubtitleLines( - new SubtitleParser().parseSRT( - subs.getData().get(0).getContentAsString("cp1252"))); - embed.setColor(Color.MEDIUM_SEA_GREEN).setDescription("Everything went well."); - } else { - logger.error("Could not download subtitle '{}': {}", args.get(0), subs.getStatus()); - embed.setDescription("An error occurred. Please contact my administrator."); + Movie movie = OMDBClient.getMovie(value, args.get(2).equalsIgnoreCase("imdb")); + if (movie == null || movie.getId() <= 0) + embed.setDescription("We couldn't find any movie with these information. Sorry!"); + else if (SubtitleManager.getSubtitlesId(movie, languageId) > 0) + embed.setDescription("This movie already has already this language imported.") + .setColor(Color.ORANGE); + else { + Stream subtitleInfoStream = OpenSubtitles.getSubtitleStreamFromMovie(movie); + Optional subtitleInfo = subtitleInfoStream == null ? Optional.empty() : + subtitleInfoStream.filter(info -> + LanguageManager.getIdLanguage(info.getLanguage()) == languageId) + .findFirst(); + if (subtitleInfo.isEmpty()) + embed.setDescription("Nothing was found with these parameters."); + else { + ListResponse subs = OpenSubtitles.downloadSubtitle( + subtitleInfo.get().getSubtitleFileId()); + if (subs.getStatus().equals(ResponseStatus.OK)) { + List blocks = new SubtitleParser().parseSRT( + subs.getData().get(0).getContentAsString("cp1252")); + SubtitleLineManager.importSubtitleLines(blocks, languageId, movie, + event.getGuildId().isPresent() ? event.getGuildId().get() : null, + event.getMember().isPresent() ? event.getMessage().getId() : null); + embed.setColor(Color.MEDIUM_SEA_GREEN).setDescription("Everything went well. " + + "Congratulations and thank you for your contribution!"); + } else { + logger.error("Could not download subtitle '{}': {}", args.get(0), subs.getStatus()); + embed.setDescription("An error occurred. Please contact my administrator."); + } + } } - } catch (XmlRpcException e) { + } catch (XmlRpcException | IOException e) { logger.error("An error occurred while fetching the data from opensubtitles: {}", e.getMessage()); embed.setDescription("An error occurred, please contact my administrator."); } diff --git a/src/main/java/xyz/vallat/louis/database/FilmManager.java b/src/main/java/xyz/vallat/louis/database/FilmManager.java index b7d5e85..cab24a2 100644 --- a/src/main/java/xyz/vallat/louis/database/FilmManager.java +++ b/src/main/java/xyz/vallat/louis/database/FilmManager.java @@ -67,7 +67,7 @@ public final class FilmManager { Movie movie = null; try (Connection connection = DBManager.getConnection()) { String query = "SELECT id, imdb_id, title, year, film_type, poster_link " + - "FROM films WHERE " + (isId ? "imdb_id" : "title") + " = ? AND film_type = ?;"; + "FROM films WHERE UPPER(" + (isId ? "imdb_id" : "title") + ") = UPPER(?) AND film_type = ?;"; try (PreparedStatement stmt = connection.prepareStatement(query)) { stmt.setString(1, value); stmt.setString(2, "movie"); diff --git a/src/main/java/xyz/vallat/louis/database/LanguageManager.java b/src/main/java/xyz/vallat/louis/database/LanguageManager.java index 14c585d..dcc94d8 100644 --- a/src/main/java/xyz/vallat/louis/database/LanguageManager.java +++ b/src/main/java/xyz/vallat/louis/database/LanguageManager.java @@ -102,6 +102,23 @@ public final class LanguageManager { } } + public static int getIdLanguage(String name) { + name = name.toLowerCase(); + try (Connection connection = DBManager.getConnection()) { + String query = "SELECT id FROM languages WHERE alpha3_b = ? " + + "OR alpha3_t = ? OR alpha2 = ? OR english = ? OR french = ?;"; + try (PreparedStatement stmt = connection.prepareStatement(query)){ + for (int i = 1; i < 6; i++) stmt.setString(i, name); + stmt.execute(); + if (stmt.getResultSet().next()) return stmt.getResultSet().getInt("id"); + } + } catch (SQLException e) { + logger.error("Cannot connect to database: {}", e.getMessage()); + System.exit(ExitCodes.CANNOT_CONNECT_TO_DB.getValue()); + } + return -1; + } + private static void insertLang(Connection connection, String alpha3b, String alpha3t, String alpha2b, String english, String french) throws SQLException { String insert = "INSERT INTO languages(alpha3_b, alpha3_t, alpha2, english, french) VALUES(?, ?, ?, ?, ?);"; diff --git a/src/main/java/xyz/vallat/louis/database/SubtitleLineManager.java b/src/main/java/xyz/vallat/louis/database/SubtitleLineManager.java index 8eb5e75..4c0936c 100644 --- a/src/main/java/xyz/vallat/louis/database/SubtitleLineManager.java +++ b/src/main/java/xyz/vallat/louis/database/SubtitleLineManager.java @@ -1,8 +1,10 @@ package xyz.vallat.louis.database; +import discord4j.common.util.Snowflake; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import xyz.vallat.louis.codes.ExitCodes; +import xyz.vallat.louis.omdb.objects.Movie; import xyz.vallat.louis.subtitles.parser.SubtitleBlock; import java.sql.Connection; @@ -71,8 +73,44 @@ public final class SubtitleLineManager { } } - public static void importSubtitleLines(List subtitleBlocks) { + public static void importSubtitleLines(List subtitleBlocks, int language, Movie movie, + Snowflake importer, Snowflake importer_guild) { + try (Connection connection = DBManager.getConnection()) { + connection.setAutoCommit(false); + String querySubtitles = "INSERT INTO subtitles(film_id, language_id, importer_id, importer_guild_id) " + + "VALUES(?, ?, ?, ?);"; + try (PreparedStatement stmt = connection.prepareStatement(querySubtitles, PreparedStatement.RETURN_GENERATED_KEYS)) { + stmt.setInt(1, movie.getId()); + stmt.setInt(2, language); + stmt.setString(3, importer == null ? null : importer.asString()); + stmt.setString(4, importer_guild == null ? null : importer_guild.asString()); + stmt.executeUpdate(); + stmt.getGeneratedKeys().next(); + int insertedSub = stmt.getGeneratedKeys().getInt(1); + logger.debug("Inserted subtitle with id '{}'.", insertedSub); + String querySubtitleLines = "INSERT INTO subtitle_lines(subtitle_id, dialog_line, time_code) " + + "VALUES(?, ?, ?);"; + for (SubtitleBlock block : subtitleBlocks) { + try (PreparedStatement stmtLines = connection.prepareStatement(querySubtitleLines, PreparedStatement.RETURN_GENERATED_KEYS)) { + stmtLines.setInt(1, insertedSub); + stmtLines.setString(2, block.getDialogue()); + stmtLines.setString(3, block.getTimecode()); + stmtLines.executeUpdate(); + stmtLines.getGeneratedKeys().next(); + logger.debug("Inserted subtitle line with id '{}'.", stmtLines.getGeneratedKeys().getInt(1)); + } + } + } + try { + connection.commit(); + } catch (SQLException e) { + connection.rollback(); + } + } catch (SQLException e) { + logger.error("Cannot connect to database right now: {}", e.getMessage()); + System.exit(ExitCodes.CANNOT_CONNECT_TO_DB.getValue()); + } } } diff --git a/src/main/java/xyz/vallat/louis/database/SubtitleManager.java b/src/main/java/xyz/vallat/louis/database/SubtitleManager.java index 7050817..df49205 100644 --- a/src/main/java/xyz/vallat/louis/database/SubtitleManager.java +++ b/src/main/java/xyz/vallat/louis/database/SubtitleManager.java @@ -3,8 +3,8 @@ package xyz.vallat.louis.database; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import xyz.vallat.louis.codes.ExitCodes; +import xyz.vallat.louis.omdb.objects.Movie; -import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -51,6 +51,27 @@ public class SubtitleManager { return -1; } + public static int getSubtitlesId(Movie movie, int languageId) { + logger.debug("Getting the id of subtitles from movie '{}' and language id '{}'.", movie.getImdbID(), languageId); + try (Connection connection = DBManager.getConnection()) { + String query = "SELECT subtitles.id FROM subtitles INNER JOIN films ON films.id = subtitles.film_id " + + "WHERE films.imdb_id = ? AND language_id = ?;"; + try (PreparedStatement stmt = connection.prepareStatement(query)) { + stmt.setString(1, movie.getImdbID()); + stmt.setInt(2, languageId); + stmt.execute(); + if (stmt.getResultSet().next()) { + logger.debug("Found subtitle with id '{}'.", stmt.getResultSet().getInt("id")); + return stmt.getResultSet().getInt("id"); + } + } + } catch (SQLException e) { + logger.error("Cannot connect to database right now: {}", e.getMessage()); + System.exit(ExitCodes.CANNOT_CONNECT_TO_DB.getValue()); + } + return -1; + } + static void initializeSubtitle(Connection connection) throws SQLException { logger.debug("Creating subtitle table."); try (Statement stmt = connection.createStatement()) { @@ -60,7 +81,7 @@ public class SubtitleManager { id int GENERATED ALWAYS AS IDENTITY, film_id int NOT NULL, language_id int NOT NULL, - importer varchar(37), + importer_id varchar(37), importer_guild_id text, imported_date timestamptz NOT NULL DEFAULT now(), UNIQUE (film_id, language_id), diff --git a/src/main/java/xyz/vallat/louis/omdb/OMDBClient.java b/src/main/java/xyz/vallat/louis/omdb/OMDBClient.java index edb49a7..7a6c149 100644 --- a/src/main/java/xyz/vallat/louis/omdb/OMDBClient.java +++ b/src/main/java/xyz/vallat/louis/omdb/OMDBClient.java @@ -1,14 +1,21 @@ package xyz.vallat.louis.omdb; +import com.github.wtekiela.opensub4j.response.ListResponse; +import com.github.wtekiela.opensub4j.response.ResponseStatus; +import com.github.wtekiela.opensub4j.response.SubtitleFile; +import com.github.wtekiela.opensub4j.response.SubtitleInfo; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import org.apache.xmlrpc.XmlRpcException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import xyz.vallat.louis.database.FilmManager; import xyz.vallat.louis.database.exceptions.ImportationException; import xyz.vallat.louis.env.EnvironmentVariables; import xyz.vallat.louis.omdb.objects.Movie; +import xyz.vallat.louis.subtitles.OpenSubtitles; +import xyz.vallat.louis.subtitles.parser.SubtitleBlock; import java.io.IOException; import java.net.URI; @@ -18,7 +25,11 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.stream.Stream; + +import static xyz.vallat.louis.commands.Command.distinctByKey; public final class OMDBClient { diff --git a/src/main/java/xyz/vallat/louis/subtitles/OpenSubtitles.java b/src/main/java/xyz/vallat/louis/subtitles/OpenSubtitles.java index 3f075b1..a5c9c17 100644 --- a/src/main/java/xyz/vallat/louis/subtitles/OpenSubtitles.java +++ b/src/main/java/xyz/vallat/louis/subtitles/OpenSubtitles.java @@ -7,9 +7,17 @@ import org.apache.xmlrpc.XmlRpcException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import xyz.vallat.louis.env.EnvironmentVariables; +import xyz.vallat.louis.omdb.OMDBClient; +import xyz.vallat.louis.omdb.objects.Movie; +import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Stream; + +import static xyz.vallat.louis.commands.Command.distinctByKey; public final class OpenSubtitles { @@ -46,11 +54,6 @@ public final class OpenSubtitles { } } - // TODO: NUKE THAT FUNCTION AND REPLACE IT WITH OMDB IMPLEMENTATION - public static ListResponse searchImdb(String name) throws XmlRpcException { - return client.searchMoviesOnImdb(name); - } - public static ListResponse searchSubtitles(String lang, String imdbId) throws XmlRpcException { return client.searchSubtitles(lang, imdbId); } @@ -62,4 +65,30 @@ public final class OpenSubtitles { public static boolean isLoggedIn() { return client != null && client.isLoggedIn(); } + + + public static Stream getSubtitleStream(List subtitles) { + return subtitles.stream() + .sorted(Comparator.comparingInt(SubtitleInfo::getDownloadsNo)) + .filter(distinctByKey(SubtitleInfo::getLanguage)) + .filter(subtitleInfo -> + (subtitleInfo.getEncoding().equalsIgnoreCase("ascii") + || subtitleInfo.getEncoding().equalsIgnoreCase("cp1252"))); + } + + public static Stream getSubtitleStreamFromMovie(Movie movie) { + Stream subtitles = null; + try { + if (movie != null) { + ListResponse subtitleInfoList = + OpenSubtitles.searchSubtitles("", String.valueOf(movie.getNumericImdbId())); + if (subtitleInfoList.getStatus().equals(ResponseStatus.OK)) + subtitles = getSubtitleStream(subtitleInfoList.getData()); + } + } catch (XmlRpcException e) { + logger.error("An error occurred: {}", e.getMessage()); + logger.warn("Please fix this issue as soon as possible, or else people won't be able to get subtitles."); + } + return subtitles; + } }