louis/reddit-reposter-bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

huge refactor

Browse Source 
hypervisor now don't rely on collections but only on the SQLite database for saving what post it has computed. It saves each post after each post, and marks it as shared before sharing it so if something bad happens, then it won't crash on it again. RAM usage has been improved too. After 8hrs running, it used more than 1GB. Now it barely uses more than 50MB.
Added a try catch for null pointer exception so it won't crash if a part of the JSON is missing. It'll just continue and don't compute this reddit post.
This commit is contained in:
Louis Vallat 2019-04-27 00:37:32 +02:00
parent a2c7c4d8da
commit b3af1f7982
2 changed files with 114 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
twitter_techsupportgore_bot/src/TwitterTechSupportGoreBot/Hypervisor.java
View File

@ -61,16 +61,6 @@ public class Hypervisor {
*/ */
private final String tableName; private final String tableName;
/**
* All the reddit posts parsed.
*/
private final HashMap<String, RedditPost> redditPosts;
/**
* All the already shared posts, so there are no boule shares.
*/
private final HashMap<String, RedditPost> alreadySharedPosts;
/** /**
* Delay between two scans, in seconds. * Delay between two scans, in seconds.
*/ */
@ -120,13 +110,11 @@ public class Hypervisor {
this.delay = Integer.valueOf(reader.getProperties("delay")); this.delay = Integer.valueOf(reader.getProperties("delay"));
this.sqliteDatabase = reader.getProperties("sqlite_db_name"); this.sqliteDatabase = reader.getProperties("sqlite_db_name");
this.socialMedias = new ArrayList<>(); this.socialMedias = new ArrayList<>();
this.redditPosts = new HashMap<>();
this.workingDirectory = reader.getProperties("working_directory"); this.workingDirectory = reader.getProperties("working_directory");
setupTheBotDirectory(); setupTheBotDirectory();
this.connexion = DriverManager.getConnection("jdbc:sqlite:" this.connexion = DriverManager.getConnection("jdbc:sqlite:"
+ this.workingDirectory + File.separator + this.sqliteDatabase); + this.workingDirectory + File.separator + this.sqliteDatabase);
this.myRedditExtractor = new RedditExtractor(subreddit); this.myRedditExtractor = new RedditExtractor(subreddit);
this.alreadySharedPosts = new HashMap<>();
if ("Y".equals(reader.getProperties("clear_database"))) { if ("Y".equals(reader.getProperties("clear_database"))) {
clearDatabase(); clearDatabase();
} }
@ -172,7 +160,6 @@ public class Hypervisor {
for (RedditPost post : myRedditExtractor.getRedditPosts()) { for (RedditPost post : myRedditExtractor.getRedditPosts()) {
computeRedditPost(post); computeRedditPost(post);
} }
save();
System.out.println( System.out.println(
"[*] Hypervisor is waiting for " + this.delay + " seconds."); "[*] Hypervisor is waiting for " + this.delay + " seconds.");
Thread.sleep(this.delay * 1000); Thread.sleep(this.delay * 1000);
@ -186,69 +173,13 @@ public class Hypervisor {
* @throws SQLException * @throws SQLException
*/ */
private void load() throws ClassNotFoundException, SQLException { private void load() throws ClassNotFoundException, SQLException {
System.out.println("[*] Loading the Reddit posts from database.");
createTable(); createTable();
PreparedStatement recherche = this.connexion.prepareStatement( PreparedStatement recherche = this.connexion.prepareStatement(
"SELECT * FROM " + this.tableName + ";"); "SELECT COUNT(id) AS cpt FROM " + this.tableName + ";");
try (ResultSet res = recherche.executeQuery()) { try (ResultSet res = recherche.executeQuery()) {
while (res.next()) { res.next();
RedditPost post System.out.println("[*] " + res.getInt("cpt") + " posts in database.");
= "image".equals(res.getString("postType"))
? new RedditPostImage(res.getString("postId"),
res.getString("title"),
res.getBoolean("quarantine"),
res.getDouble("score"),
res.getString("postHint"),
res.getBoolean("crosspostable"),
res.getBoolean("over18"),
res.getString("author"),
res.getString("permalink"),
res.getBoolean("spoiler"),
res.getString("url"))
: "link".equals(res.getString("postType"))
? new RedditPostLink(res.getString("postId"),
res.getString("title"),
res.getBoolean("quarantine"),
res.getDouble("score"),
res.getString("postHint"),
res.getBoolean("crosspostable"),
res.getBoolean("over18"),
res.getString("author"),
res.getString("permalink"),
res.getBoolean("spoiler"),
res.getString("url"))
: "video".equals(res.getString("postType"))
? new RedditPostVideo(res.getString("postId"),
res.getString("title"),
res.getBoolean("quarantine"),
res.getDouble("score"),
res.getString("postHint"),
res.getBoolean("crosspostable"),
res.getBoolean("over18"),
res.getString("author"),
res.getString("permalink"),
res.getBoolean("spoiler"),
res.getString("url"))
: new RedditPostText(res.getString("postId"),
res.getString("title"),
res.getBoolean("quarantine"),
res.getDouble("score"),
res.getString("postHint"),
res.getBoolean("crosspostable"),
res.getBoolean("over18"),
res.getString("author"),
res.getString("permalink"),
res.getBoolean("spoiler"),
res.getString("url"));
this.redditPosts.put(post.getPostId(), post);
if (res.getBoolean("shared")) {
this.alreadySharedPosts.put(post.getPostId(), post);
}
}
} }
System.out.println("[*] Loading done. "
+ this.redditPosts.size() + " posts loaded. "
+ this.alreadySharedPosts.size() + " posts already shared.");
} }
/** /**
@ -268,16 +199,17 @@ public class Hypervisor {
} }
/** /**
* Save all the reddit posts to the database. * Add a given reddit post to the database.
*
* @param current a given reddit post to add to the database
* *
* @throws ClassNotFoundException * @throws ClassNotFoundException
* @throws SQLException * @throws SQLException
*/ */
private void save() throws ClassNotFoundException, SQLException { private void addRedditPostToDatabase(RedditPost current)
throws ClassNotFoundException, SQLException {
createTable(); createTable();
for (String postId : this.redditPosts.keySet()) { if (!isInDatabase(current.getPostId())) {
RedditPost current = this.redditPosts.get(postId);
if (!isInDatabase(postId)) {
PreparedStatement ajout = this.connexion.prepareStatement( PreparedStatement ajout = this.connexion.prepareStatement(
"INSERT INTO " + this.tableName "INSERT INTO " + this.tableName
+ "(" + "("
@ -314,10 +246,9 @@ public class Hypervisor {
ajout.setString(10, current.getPermalink()); ajout.setString(10, current.getPermalink());
ajout.setBoolean(11, current.isSpoiler()); ajout.setBoolean(11, current.isSpoiler());
ajout.setString(12, current.getUrl()); ajout.setString(12, current.getUrl());
ajout.setBoolean(13, this.alreadySharedPosts ajout.setBoolean(13, false);
.containsKey(current.getPostId()));
ajout.execute(); ajout.execute();
}
} }
} }
@ -340,6 +271,28 @@ public class Hypervisor {
} }
} }
/**
* Check if a post is marked as shared in database.
*
* @param postId the post id
* @return if the post is shared in the database
*
* @throws SQLException
* @throws ClassNotFoundException
*/
private boolean isShared(String postId)
throws SQLException, ClassNotFoundException {
PreparedStatement recherche = this.connexion.prepareStatement(
"SELECT shared FROM " + this.tableName + " "
+ "WHERE postId = '" + postId + "'");
try (ResultSet resultats = recherche.executeQuery()) {
if (resultats.next()) {
return resultats.getBoolean("shared");
}
return false;
}
}
/** /**
* Set a post as shared onsocial networks. * Set a post as shared onsocial networks.
* *
@ -348,21 +301,17 @@ public class Hypervisor {
* @throws SQLException * @throws SQLException
* @throws ClassNotFoundException * @throws ClassNotFoundException
*/ */
private void setPostAsShared(String postId) private void setPostAsShared(RedditPost r)
throws SQLException, ClassNotFoundException { throws SQLException, ClassNotFoundException {
System.out.println("[+] Post \"" System.out.println("[+] Post \""
+ this.redditPosts.get(postId).getTitle() + r.getTitle()
+ "\" has been shared successfully."); + "\" is about to be shared.");
this.alreadySharedPosts.put(postId, this.redditPosts.get(postId)); PreparedStatement update = this.connexion.prepareStatement(
if (isInDatabase(postId)) { "UPDATE " + this.tableName + " "
PreparedStatement update = this.connexion.prepareStatement( + "SET shared=? "
"UPDATE " + this.tableName + " " + "WHERE postId = '" + r.getPostId() + "';");
+ "SET shared=? " update.setBoolean(1, true);
+ "WHERE postId = '" + postId + "';" update.execute();
);
update.setBoolean(1, true);
update.execute();
}
} }
/** /**
@ -404,13 +353,13 @@ public class Hypervisor {
*/ */
private void computeRedditPost(RedditPost r) private void computeRedditPost(RedditPost r)
throws SQLException, ClassNotFoundException { throws SQLException, ClassNotFoundException {
if (!this.alreadySharedPosts.containsKey(r.getPostId()) if (!isShared(r.getPostId()) && !r.isQuarantine()) {
&& !r.isQuarantine()) {
System.out.println( System.out.println(
"[*] Computing the post \"" + r.getTitle() + "\""); "[*] Computing the post \"" + r.getTitle() + "\"");
this.redditPosts.put(r.getPostId(), r); addRedditPostToDatabase(r);
if (r.hasMediaUrl()) { if (r.hasMediaUrl()) {
String fileName = saveImage(r.getUrl()); String fileName = saveImage(r.getUrl());
setPostAsShared(r);
socialMedias.forEach((s) -> { socialMedias.forEach((s) -> {
long postRef = s.postImage( long postRef = s.postImage(
formatPost(r.getTitle()), fileName); formatPost(r.getTitle()), fileName);
@ -422,8 +371,10 @@ public class Hypervisor {
} }
}); });
deleteFile(fileName); deleteFile(fileName);
System.out.println("[+] Post \""
+ r.getTitle()
+ "\" has been shared successfully.");
} }
setPostAsShared(r.getPostId());
} }
} }

124
twitter_techsupportgore_bot/src/TwitterTechSupportGoreBot/redditHandler/RedditExtractor.java
View File

@ -145,67 +145,73 @@ public final class RedditExtractor {
JsonArray children = new JsonParser().parse(data.get("children") JsonArray children = new JsonParser().parse(data.get("children")
.toString()).getAsJsonArray(); .toString()).getAsJsonArray();
for (int i = 0; i < children.size(); i++) { for (int i = 0; i < children.size(); i++) {
JsonObject child = new JsonParser().parse(children.get(i) try {
.toString()).getAsJsonObject(); JsonObject child = new JsonParser().parse(children.get(i)
JsonObject childData = new JsonParser().parse(child.get("data") .toString()).getAsJsonObject();
.toString()).getAsJsonObject(); JsonObject childData = new JsonParser().parse(child.get("data")
if (childData.get("id").toString() != null .toString()).getAsJsonObject();
&& !childData.get("quarantine").getAsBoolean() if (childData.get("id").toString() != null
&& childData.get("url").getAsString() != null) { && !childData.get("quarantine").getAsBoolean()
String id = childData.get("id").toString(); && childData.get("url").getAsString() != null) {
String title = childData.get("title") != null String id = childData.get("id").toString();
? childData.get("title").toString() String title = childData.get("title") != null
: this.sub.getName(); ? childData.get("title").toString()
title = title.replace("\"", "").replace("\\", "\""); : this.sub.getName();
String author = childData.get("author") != null title = title.replace("\"", "").replace("\\", "\"");
? childData.get("author").toString() : "anonymous"; String author = childData.get("author") != null
author = author.replace("\"", ""); ? childData.get("author").toString() : "anonymous";
boolean quarantine = childData.get("quarantine") author = author.replace("\"", "");
.getAsBoolean(); boolean quarantine = childData.get("quarantine")
double score = childData.get("score").getAsDouble(); .getAsBoolean();
String postHint = childData.get("post_hint").getAsString(); double score = childData.get("score").getAsDouble();
boolean crosspostable = !childData.get("is_crosspostable") String postHint = childData.get("post_hint").getAsString();
.getAsBoolean(); boolean crosspostable = !childData.get("is_crosspostable")
boolean over18 = childData.get("over_18").getAsBoolean(); .getAsBoolean();
String url; boolean over18 = childData.get("over_18").getAsBoolean();
try { String url;
JsonObject preview = new JsonParser().parse(childData try {
.get("preview").toString()).getAsJsonObject(); JsonObject preview = new JsonParser().parse(childData
JsonArray previewImages = new JsonParser().parse(preview .get("preview").toString()).getAsJsonObject();
.get("images").toString()).getAsJsonArray(); JsonArray previewImages = new JsonParser().parse(preview
JsonObject source = new JsonParser().parse(previewImages .get("images").toString()).getAsJsonArray();
.get(0).toString()).getAsJsonObject(); JsonObject source = new JsonParser().parse(previewImages
JsonObject urlSrc = new JsonParser().parse(source .get(0).toString()).getAsJsonObject();
.get("source").toString()).getAsJsonObject(); JsonObject urlSrc = new JsonParser().parse(source
url = urlSrc.get("url").toString().replace("amp;", "") .get("source").toString()).getAsJsonObject();
.replace("\"", ""); url = urlSrc.get("url").toString().replace("amp;", "")
} catch (NullPointerException n) { .replace("\"", "");
url = childData.get("url").getAsString(); } catch (NullPointerException n) {
} url = childData.get("url").getAsString();
String permalink = childData.get("permalink").getAsString(); }
boolean spoiler = childData.get("spoiler").getAsBoolean(); String permalink = childData.get("permalink").getAsString();
boolean spoiler = childData.get("spoiler").getAsBoolean();
if (postHint.contains("video")) { if (postHint.contains("video")) {
set.add(new RedditPostVideo( set.add(new RedditPostVideo(
id, title, quarantine, score, postHint, id, title, quarantine, score, postHint,
crosspostable, over18, author, crosspostable, over18, author,
permalink, spoiler, url)); permalink, spoiler, url));
} else if ("link".equals(postHint)) { } else if ("link".equals(postHint)) {
set.add(new RedditPostLink( set.add(new RedditPostLink(
id, title, quarantine, score, postHint, id, title, quarantine, score, postHint,
crosspostable, over18, author, crosspostable, over18, author,
permalink, spoiler, url)); permalink, spoiler, url));
} else if ("text".equals(postHint)) { } else if ("text".equals(postHint)) {
set.add(new RedditPostText( set.add(new RedditPostText(
id, title, quarantine, score, postHint, id, title, quarantine, score, postHint,
crosspostable, over18, author, crosspostable, over18, author,
permalink, spoiler, url)); permalink, spoiler, url));
} else if ("image".equals(postHint)) { } else if ("image".equals(postHint)) {
set.add(new RedditPostImage( set.add(new RedditPostImage(
id, title, quarantine, score, postHint, id, title, quarantine, score, postHint,
crosspostable, over18, author, crosspostable, over18, author,
permalink, spoiler, url)); permalink, spoiler, url));
}
} }
} catch (NullPointerException e) {
System.out.println(
"[*] There were a problem while parsing. "
+ "Continuing");
} }
} }
return set; return set;