louis/moviequotebot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Downloading subtitles is now kinda working

Browse Source 
Signed-off-by: Louis Vallat <louis@louis-vallat.xyz>
This commit is contained in:
Louis Vallat 2020-10-29 11:51:19 +01:00
parent d645ccb711
commit 2f5195ad0a
9 changed files with 187 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
MoviesQuoteBot.sql
View File

@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS subtitles
id int GENERATED ALWAYS AS IDENTITY,
film_id int NOT NULL,
language_id int NOT NULL,
importer varchar(37),
importer_id varchar(37),
importer_guild_id text,
imported_date timestamptz NOT NULL DEFAULT now(),
UNIQUE (film_id, language_id),

31
src/main/java/xyz/vallat/louis/commands/Command.java
View File

@ -25,10 +25,11 @@ import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public abstract class Command {
private static final Logger logger = LoggerFactory.getLogger(ListLangMovie.class.getCanonicalName());
protected static final ReactionEmoji WAITING = ReactionEmoji.unicode("⌛");
import static xyz.vallat.louis.subtitles.OpenSubtitles.getSubtitleStream;
public abstract class Command {
protected static final ReactionEmoji WAITING = ReactionEmoji.unicode("⌛");
private static final Logger logger = LoggerFactory.getLogger(ListLangMovie.class.getCanonicalName());
protected final String name;
protected final String description;
protected final String usage;
@ -79,6 +80,13 @@ public abstract class Command {
.then();
}
protected Mono<Void> unknownLanguage(MessageCreateEvent event, String lang) {
return event.getMessage().getChannel()
.flatMap(channel ->
channel.createMessage("Error, this lang isn't recognized: '" + lang + "'.\n Usage: " + getUsage()))
.then();
}
protected void createEmbedListLang(String arg, EmbedCreateSpec embed, boolean isId) {
try {
Movie movie = OMDBClient.getMovie(arg, isId);
@ -111,8 +119,9 @@ public abstract class Command {
private void handleOk(String arg, EmbedCreateSpec embed, boolean isId, Movie movie, ListResponse<SubtitleInfo> subtitles) {
embed.setColor(Color.BISMARK);
Stream<String> subtitlesStream = getSubtitleStream(subtitles);
String formattedSubtitles = subtitlesStream.collect(Collectors.joining("\n"));
Stream<String> subtitlesStream = getSubtitleStream(subtitles.getData())
.map(subtitleInfo -> subtitleInfo.getSubtitleFileId() + " - " + subtitleInfo.getLanguage());
String formattedSubtitles = subtitlesStream.limit(20).collect(Collectors.joining("\n"));
embed.setDescription("You requested a list of the languages available for a movie with the " +
(isId ? "IMDB Identifier" : "title") + " '" + arg + "'. ");
embed.addField("Movie", movie.toString(), true);
@ -122,18 +131,6 @@ public abstract class Command {
formattedSubtitles.isBlank() ? "None." : formattedSubtitles, false);
}
private Stream<String> getSubtitleStream(ListResponse<SubtitleInfo> subtitles) {
return subtitles.getData().stream()
.sorted(Comparator.comparingInt(SubtitleInfo::getDownloadsNo))
.filter(distinctByKey(SubtitleInfo::getLanguage))
.filter(subtitleInfo ->
(subtitleInfo.getEncoding().equalsIgnoreCase("ascii")
|| subtitleInfo.getEncoding().equalsIgnoreCase("cp1252")))
.map(subtitleInfo -> subtitleInfo.getSubtitleFileId()
+ " - " + subtitleInfo.getLanguage())
.limit(20);
}
protected List<String> getArguments(MessageCreateEvent event) {
return List.of(event.getMessage().getContent().substring(Math.min(name.length() + 1, name.length())).split(" "));
}

55
src/main/java/xyz/vallat/louis/commands/Download.java
View File

@ -3,48 +3,81 @@ package xyz.vallat.louis.commands;
import com.github.wtekiela.opensub4j.response.ListResponse;
import com.github.wtekiela.opensub4j.response.ResponseStatus;
import com.github.wtekiela.opensub4j.response.SubtitleFile;
import com.github.wtekiela.opensub4j.response.SubtitleInfo;
import discord4j.core.event.domain.message.MessageCreateEvent;
import discord4j.rest.util.Color;
import org.apache.commons.lang3.StringUtils;
import org.apache.xmlrpc.XmlRpcException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import reactor.core.publisher.Mono;
import xyz.vallat.louis.database.LanguageManager;
import xyz.vallat.louis.database.SubtitleLineManager;
import xyz.vallat.louis.database.SubtitleManager;
import xyz.vallat.louis.omdb.OMDBClient;
import xyz.vallat.louis.omdb.objects.Movie;
import xyz.vallat.louis.subtitles.OpenSubtitles;
import xyz.vallat.louis.subtitles.parser.SubtitleBlock;
import xyz.vallat.louis.subtitles.parser.SubtitleParser;
import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Optional;
import java.util.stream.Stream;
public class Download extends Command {
private static final Logger logger = LoggerFactory.getLogger(Download.class.getCanonicalName());
public Download(String name) {
super(name, "Download a subtitle.", name + " subtitleId", 1, 1);
super(name, "Download a subtitle.", name + " lang imdb|title value", 3, 3);
}
@Override
public Mono<Void> execute(MessageCreateEvent event) {
List<String> args = getArguments(event);
if (args.isEmpty()) return notEnoughArguments(event);
else if (!StringUtils.isNumeric(args.get(0))) return badArgument(event, args.get(0));
if (args.size() < minArgs) return notEnoughArguments(event);
String value = String.join(" ", args.subList(3, args.size()));
if (!args.get(2).equalsIgnoreCase("imdb") && !args.get(2).equalsIgnoreCase("title"))
return badArgument(event, args.get(2));
int languageId = LanguageManager.getIdLanguage(args.get(1));
if (languageId < 0) return unknownLanguage(event, args.get(1));
return event.getMessage().getChannel().flatMap(channel -> event.getMessage().addReaction(WAITING).then(
channel.createEmbed(embed -> {
embed.setTitle("Importation").setColor(Color.RED);
try {
ListResponse<SubtitleFile> subs = OpenSubtitles.downloadSubtitle(Integer.parseInt(args.get(0)));
if (subs.getStatus().getCode() == ResponseStatus.OK.getCode()) {
SubtitleLineManager.importSubtitleLines(
new SubtitleParser().parseSRT(
subs.getData().get(0).getContentAsString("cp1252")));
embed.setColor(Color.MEDIUM_SEA_GREEN).setDescription("Everything went well.");
Movie movie = OMDBClient.getMovie(value, args.get(2).equalsIgnoreCase("imdb"));
if (movie == null || movie.getId() <= 0)
embed.setDescription("We couldn't find any movie with these information. Sorry!");
else if (SubtitleManager.getSubtitlesId(movie, languageId) > 0)
embed.setDescription("This movie already has already this language imported.")
.setColor(Color.ORANGE);
else {
Stream<SubtitleInfo> subtitleInfoStream = OpenSubtitles.getSubtitleStreamFromMovie(movie);
Optional<SubtitleInfo> subtitleInfo = subtitleInfoStream == null ? Optional.empty() :
subtitleInfoStream.filter(info ->
LanguageManager.getIdLanguage(info.getLanguage()) == languageId)
.findFirst();
if (subtitleInfo.isEmpty())
embed.setDescription("Nothing was found with these parameters.");
else {
ListResponse<SubtitleFile> subs = OpenSubtitles.downloadSubtitle(
subtitleInfo.get().getSubtitleFileId());
if (subs.getStatus().equals(ResponseStatus.OK)) {
List<SubtitleBlock> blocks = new SubtitleParser().parseSRT(
subs.getData().get(0).getContentAsString("cp1252"));
SubtitleLineManager.importSubtitleLines(blocks, languageId, movie,
event.getGuildId().isPresent() ? event.getGuildId().get() : null,
event.getMember().isPresent() ? event.getMessage().getId() : null);
embed.setColor(Color.MEDIUM_SEA_GREEN).setDescription("Everything went well. " +
"Congratulations and thank you for your contribution!");
} else {
logger.error("Could not download subtitle '{}': {}", args.get(0), subs.getStatus());
embed.setDescription("An error occurred. Please contact my administrator.");
}
} catch (XmlRpcException e) {
}
}
} catch (XmlRpcException | IOException e) {
logger.error("An error occurred while fetching the data from opensubtitles: {}", e.getMessage());
embed.setDescription("An error occurred, please contact my administrator.");
}

2
src/main/java/xyz/vallat/louis/database/FilmManager.java
View File

@ -67,7 +67,7 @@ public final class FilmManager {
Movie movie = null;
try (Connection connection = DBManager.getConnection()) {
String query = "SELECT id, imdb_id, title, year, film_type, poster_link " +
"FROM films WHERE " + (isId ? "imdb_id" : "title") + " = ? AND film_type = ?;";
"FROM films WHERE UPPER(" + (isId ? "imdb_id" : "title") + ") = UPPER(?) AND film_type = ?;";
try (PreparedStatement stmt = connection.prepareStatement(query)) {
stmt.setString(1, value);
stmt.setString(2, "movie");

17
src/main/java/xyz/vallat/louis/database/LanguageManager.java
View File

@ -102,6 +102,23 @@ public final class LanguageManager {
}
}
public static int getIdLanguage(String name) {
name = name.toLowerCase();
try (Connection connection = DBManager.getConnection()) {
String query = "SELECT id FROM languages WHERE alpha3_b = ? " +
"OR alpha3_t = ? OR alpha2 = ? OR english = ? OR french = ?;";
try (PreparedStatement stmt = connection.prepareStatement(query)){
for (int i = 1; i < 6; i++) stmt.setString(i, name);
stmt.execute();
if (stmt.getResultSet().next()) return stmt.getResultSet().getInt("id");
}
} catch (SQLException e) {
logger.error("Cannot connect to database: {}", e.getMessage());
System.exit(ExitCodes.CANNOT_CONNECT_TO_DB.getValue());
}
return -1;
}
private static void insertLang(Connection connection, String alpha3b, String alpha3t,
String alpha2b, String english, String french) throws SQLException {
String insert = "INSERT INTO languages(alpha3_b, alpha3_t, alpha2, english, french) VALUES(?, ?, ?, ?, ?);";

40
src/main/java/xyz/vallat/louis/database/SubtitleLineManager.java
View File

@ -1,8 +1,10 @@
package xyz.vallat.louis.database;
import discord4j.common.util.Snowflake;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import xyz.vallat.louis.codes.ExitCodes;
import xyz.vallat.louis.omdb.objects.Movie;
import xyz.vallat.louis.subtitles.parser.SubtitleBlock;
import java.sql.Connection;
@ -71,8 +73,44 @@ public final class SubtitleLineManager {
}
}
public static void importSubtitleLines(List<SubtitleBlock> subtitleBlocks) {
public static void importSubtitleLines(List<SubtitleBlock> subtitleBlocks, int language, Movie movie,
Snowflake importer, Snowflake importer_guild) {
try (Connection connection = DBManager.getConnection()) {
connection.setAutoCommit(false);
String querySubtitles = "INSERT INTO subtitles(film_id, language_id, importer_id, importer_guild_id) " +
"VALUES(?, ?, ?, ?);";
try (PreparedStatement stmt = connection.prepareStatement(querySubtitles, PreparedStatement.RETURN_GENERATED_KEYS)) {
stmt.setInt(1, movie.getId());
stmt.setInt(2, language);
stmt.setString(3, importer == null ? null : importer.asString());
stmt.setString(4, importer_guild == null ? null : importer_guild.asString());
stmt.executeUpdate();
stmt.getGeneratedKeys().next();
int insertedSub = stmt.getGeneratedKeys().getInt(1);
logger.debug("Inserted subtitle with id '{}'.", insertedSub);
String querySubtitleLines = "INSERT INTO subtitle_lines(subtitle_id, dialog_line, time_code) " +
"VALUES(?, ?, ?);";
for (SubtitleBlock block : subtitleBlocks) {
try (PreparedStatement stmtLines = connection.prepareStatement(querySubtitleLines, PreparedStatement.RETURN_GENERATED_KEYS)) {
stmtLines.setInt(1, insertedSub);
stmtLines.setString(2, block.getDialogue());
stmtLines.setString(3, block.getTimecode());
stmtLines.executeUpdate();
stmtLines.getGeneratedKeys().next();
logger.debug("Inserted subtitle line with id '{}'.", stmtLines.getGeneratedKeys().getInt(1));
}
}
}
try {
connection.commit();
} catch (SQLException e) {
connection.rollback();
}
} catch (SQLException e) {
logger.error("Cannot connect to database right now: {}", e.getMessage());
System.exit(ExitCodes.CANNOT_CONNECT_TO_DB.getValue());
}
}
}

25
src/main/java/xyz/vallat/louis/database/SubtitleManager.java
View File

@ -3,8 +3,8 @@ package xyz.vallat.louis.database;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import xyz.vallat.louis.codes.ExitCodes;
import xyz.vallat.louis.omdb.objects.Movie;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
@ -51,6 +51,27 @@ public class SubtitleManager {
return -1;
}
public static int getSubtitlesId(Movie movie, int languageId) {
logger.debug("Getting the id of subtitles from movie '{}' and language id '{}'.", movie.getImdbID(), languageId);
try (Connection connection = DBManager.getConnection()) {
String query = "SELECT subtitles.id FROM subtitles INNER JOIN films ON films.id = subtitles.film_id " +
"WHERE films.imdb_id = ? AND language_id = ?;";
try (PreparedStatement stmt = connection.prepareStatement(query)) {
stmt.setString(1, movie.getImdbID());
stmt.setInt(2, languageId);
stmt.execute();
if (stmt.getResultSet().next()) {
logger.debug("Found subtitle with id '{}'.", stmt.getResultSet().getInt("id"));
return stmt.getResultSet().getInt("id");
}
}
} catch (SQLException e) {
logger.error("Cannot connect to database right now: {}", e.getMessage());
System.exit(ExitCodes.CANNOT_CONNECT_TO_DB.getValue());
}
return -1;
}
static void initializeSubtitle(Connection connection) throws SQLException {
logger.debug("Creating subtitle table.");
try (Statement stmt = connection.createStatement()) {
@ -60,7 +81,7 @@ public class SubtitleManager {
id int GENERATED ALWAYS AS IDENTITY,
film_id int NOT NULL,
language_id int NOT NULL,
importer varchar(37),
importer_id varchar(37),
importer_guild_id text,
imported_date timestamptz NOT NULL DEFAULT now(),
UNIQUE (film_id, language_id),

11
src/main/java/xyz/vallat/louis/omdb/OMDBClient.java
View File

@ -1,14 +1,21 @@
package xyz.vallat.louis.omdb;
import com.github.wtekiela.opensub4j.response.ListResponse;
import com.github.wtekiela.opensub4j.response.ResponseStatus;
import com.github.wtekiela.opensub4j.response.SubtitleFile;
import com.github.wtekiela.opensub4j.response.SubtitleInfo;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.xmlrpc.XmlRpcException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import xyz.vallat.louis.database.FilmManager;
import xyz.vallat.louis.database.exceptions.ImportationException;
import xyz.vallat.louis.env.EnvironmentVariables;
import xyz.vallat.louis.omdb.objects.Movie;
import xyz.vallat.louis.subtitles.OpenSubtitles;
import xyz.vallat.louis.subtitles.parser.SubtitleBlock;
import java.io.IOException;
import java.net.URI;
@ -18,7 +25,11 @@ import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Stream;
import static xyz.vallat.louis.commands.Command.distinctByKey;
public final class OMDBClient {

39
src/main/java/xyz/vallat/louis/subtitles/OpenSubtitles.java
View File

@ -7,9 +7,17 @@ import org.apache.xmlrpc.XmlRpcException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import xyz.vallat.louis.env.EnvironmentVariables;
import xyz.vallat.louis.omdb.OMDBClient;
import xyz.vallat.louis.omdb.objects.Movie;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Stream;
import static xyz.vallat.louis.commands.Command.distinctByKey;
public final class OpenSubtitles {
@ -46,11 +54,6 @@ public final class OpenSubtitles {
}
}
// TODO: NUKE THAT FUNCTION AND REPLACE IT WITH OMDB IMPLEMENTATION
public static ListResponse<MovieInfo> searchImdb(String name) throws XmlRpcException {
return client.searchMoviesOnImdb(name);
}
public static ListResponse<SubtitleInfo> searchSubtitles(String lang, String imdbId) throws XmlRpcException {
return client.searchSubtitles(lang, imdbId);
}
@ -62,4 +65,30 @@ public final class OpenSubtitles {
public static boolean isLoggedIn() {
return client != null && client.isLoggedIn();
}
public static Stream<SubtitleInfo> getSubtitleStream(List<SubtitleInfo> subtitles) {
return subtitles.stream()
.sorted(Comparator.comparingInt(SubtitleInfo::getDownloadsNo))
.filter(distinctByKey(SubtitleInfo::getLanguage))
.filter(subtitleInfo ->
(subtitleInfo.getEncoding().equalsIgnoreCase("ascii")
|| subtitleInfo.getEncoding().equalsIgnoreCase("cp1252")));
}
public static Stream<SubtitleInfo> getSubtitleStreamFromMovie(Movie movie) {
Stream<SubtitleInfo> subtitles = null;
try {
if (movie != null) {
ListResponse<SubtitleInfo> subtitleInfoList =
OpenSubtitles.searchSubtitles("", String.valueOf(movie.getNumericImdbId()));
if (subtitleInfoList.getStatus().equals(ResponseStatus.OK))
subtitles = getSubtitleStream(subtitleInfoList.getData());
}
} catch (XmlRpcException e) {
logger.error("An error occurred: {}", e.getMessage());
logger.warn("Please fix this issue as soon as possible, or else people won't be able to get subtitles.");
}
return subtitles;
}
}