From b3af1f7982243577e5f0cd93bb1a6b183a3b8725 Mon Sep 17 00:00:00 2001 From: Louis Vallat Date: Sat, 27 Apr 2019 00:37:32 +0200 Subject: [PATCH] huge refactor hypervisor now don't rely on collections but only on the SQLite database for saving what post it has computed. It saves each post after each post, and marks it as shared before sharing it so if something bad happens, then it won't crash on it again. RAM usage has been improved too. After 8hrs running, it used more than 1GB. Now it barely uses more than 50MB. Added a try catch for null pointer exception so it won't crash if a part of the JSON is missing. It'll just continue and don't compute this reddit post. --- .../TwitterTechSupportGoreBot/Hypervisor.java | 147 ++++++------------ .../redditHandler/RedditExtractor.java | 124 ++++++++------- 2 files changed, 114 insertions(+), 157 deletions(-) diff --git a/twitter_techsupportgore_bot/src/TwitterTechSupportGoreBot/Hypervisor.java b/twitter_techsupportgore_bot/src/TwitterTechSupportGoreBot/Hypervisor.java index 2dbb14d..7ea0808 100644 --- a/twitter_techsupportgore_bot/src/TwitterTechSupportGoreBot/Hypervisor.java +++ b/twitter_techsupportgore_bot/src/TwitterTechSupportGoreBot/Hypervisor.java @@ -60,17 +60,7 @@ public class Hypervisor { * SQLITE table name. */ private final String tableName; - - /** - * All the reddit posts parsed. - */ - private final HashMap redditPosts; - - /** - * All the already shared posts, so there are no boule shares. - */ - private final HashMap alreadySharedPosts; - + /** * Delay between two scans, in seconds. */ @@ -120,13 +110,11 @@ public class Hypervisor { this.delay = Integer.valueOf(reader.getProperties("delay")); this.sqliteDatabase = reader.getProperties("sqlite_db_name"); this.socialMedias = new ArrayList<>(); - this.redditPosts = new HashMap<>(); this.workingDirectory = reader.getProperties("working_directory"); setupTheBotDirectory(); this.connexion = DriverManager.getConnection("jdbc:sqlite:" + this.workingDirectory + File.separator + this.sqliteDatabase); this.myRedditExtractor = new RedditExtractor(subreddit); - this.alreadySharedPosts = new HashMap<>(); if ("Y".equals(reader.getProperties("clear_database"))) { clearDatabase(); } @@ -172,7 +160,6 @@ public class Hypervisor { for (RedditPost post : myRedditExtractor.getRedditPosts()) { computeRedditPost(post); } - save(); System.out.println( "[*] Hypervisor is waiting for " + this.delay + " seconds."); Thread.sleep(this.delay * 1000); @@ -186,69 +173,13 @@ public class Hypervisor { * @throws SQLException */ private void load() throws ClassNotFoundException, SQLException { - System.out.println("[*] Loading the Reddit posts from database."); createTable(); PreparedStatement recherche = this.connexion.prepareStatement( - "SELECT * FROM " + this.tableName + ";"); + "SELECT COUNT(id) AS cpt FROM " + this.tableName + ";"); try (ResultSet res = recherche.executeQuery()) { - while (res.next()) { - RedditPost post - = "image".equals(res.getString("postType")) - ? new RedditPostImage(res.getString("postId"), - res.getString("title"), - res.getBoolean("quarantine"), - res.getDouble("score"), - res.getString("postHint"), - res.getBoolean("crosspostable"), - res.getBoolean("over18"), - res.getString("author"), - res.getString("permalink"), - res.getBoolean("spoiler"), - res.getString("url")) - : "link".equals(res.getString("postType")) - ? new RedditPostLink(res.getString("postId"), - res.getString("title"), - res.getBoolean("quarantine"), - res.getDouble("score"), - res.getString("postHint"), - res.getBoolean("crosspostable"), - res.getBoolean("over18"), - res.getString("author"), - res.getString("permalink"), - res.getBoolean("spoiler"), - res.getString("url")) - : "video".equals(res.getString("postType")) - ? new RedditPostVideo(res.getString("postId"), - res.getString("title"), - res.getBoolean("quarantine"), - res.getDouble("score"), - res.getString("postHint"), - res.getBoolean("crosspostable"), - res.getBoolean("over18"), - res.getString("author"), - res.getString("permalink"), - res.getBoolean("spoiler"), - res.getString("url")) - : new RedditPostText(res.getString("postId"), - res.getString("title"), - res.getBoolean("quarantine"), - res.getDouble("score"), - res.getString("postHint"), - res.getBoolean("crosspostable"), - res.getBoolean("over18"), - res.getString("author"), - res.getString("permalink"), - res.getBoolean("spoiler"), - res.getString("url")); - this.redditPosts.put(post.getPostId(), post); - if (res.getBoolean("shared")) { - this.alreadySharedPosts.put(post.getPostId(), post); - } - } + res.next(); + System.out.println("[*] " + res.getInt("cpt") + " posts in database."); } - System.out.println("[*] Loading done. " - + this.redditPosts.size() + " posts loaded. " - + this.alreadySharedPosts.size() + " posts already shared."); } /** @@ -268,16 +199,17 @@ public class Hypervisor { } /** - * Save all the reddit posts to the database. + * Add a given reddit post to the database. + * + * @param current a given reddit post to add to the database * * @throws ClassNotFoundException * @throws SQLException */ - private void save() throws ClassNotFoundException, SQLException { + private void addRedditPostToDatabase(RedditPost current) + throws ClassNotFoundException, SQLException { createTable(); - for (String postId : this.redditPosts.keySet()) { - RedditPost current = this.redditPosts.get(postId); - if (!isInDatabase(postId)) { + if (!isInDatabase(current.getPostId())) { PreparedStatement ajout = this.connexion.prepareStatement( "INSERT INTO " + this.tableName + "(" @@ -314,10 +246,9 @@ public class Hypervisor { ajout.setString(10, current.getPermalink()); ajout.setBoolean(11, current.isSpoiler()); ajout.setString(12, current.getUrl()); - ajout.setBoolean(13, this.alreadySharedPosts - .containsKey(current.getPostId())); + ajout.setBoolean(13, false); ajout.execute(); - } + } } @@ -340,6 +271,28 @@ public class Hypervisor { } } + /** + * Check if a post is marked as shared in database. + * + * @param postId the post id + * @return if the post is shared in the database + * + * @throws SQLException + * @throws ClassNotFoundException + */ + private boolean isShared(String postId) + throws SQLException, ClassNotFoundException { + PreparedStatement recherche = this.connexion.prepareStatement( + "SELECT shared FROM " + this.tableName + " " + + "WHERE postId = '" + postId + "'"); + try (ResultSet resultats = recherche.executeQuery()) { + if (resultats.next()) { + return resultats.getBoolean("shared"); + } + return false; + } + } + /** * Set a post as shared onsocial networks. * @@ -348,21 +301,17 @@ public class Hypervisor { * @throws SQLException * @throws ClassNotFoundException */ - private void setPostAsShared(String postId) + private void setPostAsShared(RedditPost r) throws SQLException, ClassNotFoundException { System.out.println("[+] Post \"" - + this.redditPosts.get(postId).getTitle() - + "\" has been shared successfully."); - this.alreadySharedPosts.put(postId, this.redditPosts.get(postId)); - if (isInDatabase(postId)) { - PreparedStatement update = this.connexion.prepareStatement( - "UPDATE " + this.tableName + " " - + "SET shared=? " - + "WHERE postId = '" + postId + "';" - ); - update.setBoolean(1, true); - update.execute(); - } + + r.getTitle() + + "\" is about to be shared."); + PreparedStatement update = this.connexion.prepareStatement( + "UPDATE " + this.tableName + " " + + "SET shared=? " + + "WHERE postId = '" + r.getPostId() + "';"); + update.setBoolean(1, true); + update.execute(); } /** @@ -404,13 +353,13 @@ public class Hypervisor { */ private void computeRedditPost(RedditPost r) throws SQLException, ClassNotFoundException { - if (!this.alreadySharedPosts.containsKey(r.getPostId()) - && !r.isQuarantine()) { + if (!isShared(r.getPostId()) && !r.isQuarantine()) { System.out.println( "[*] Computing the post \"" + r.getTitle() + "\""); - this.redditPosts.put(r.getPostId(), r); + addRedditPostToDatabase(r); if (r.hasMediaUrl()) { String fileName = saveImage(r.getUrl()); + setPostAsShared(r); socialMedias.forEach((s) -> { long postRef = s.postImage( formatPost(r.getTitle()), fileName); @@ -422,8 +371,10 @@ public class Hypervisor { } }); deleteFile(fileName); + System.out.println("[+] Post \"" + + r.getTitle() + + "\" has been shared successfully."); } - setPostAsShared(r.getPostId()); } } diff --git a/twitter_techsupportgore_bot/src/TwitterTechSupportGoreBot/redditHandler/RedditExtractor.java b/twitter_techsupportgore_bot/src/TwitterTechSupportGoreBot/redditHandler/RedditExtractor.java index dfc97d8..48d1b2e 100644 --- a/twitter_techsupportgore_bot/src/TwitterTechSupportGoreBot/redditHandler/RedditExtractor.java +++ b/twitter_techsupportgore_bot/src/TwitterTechSupportGoreBot/redditHandler/RedditExtractor.java @@ -145,67 +145,73 @@ public final class RedditExtractor { JsonArray children = new JsonParser().parse(data.get("children") .toString()).getAsJsonArray(); for (int i = 0; i < children.size(); i++) { - JsonObject child = new JsonParser().parse(children.get(i) - .toString()).getAsJsonObject(); - JsonObject childData = new JsonParser().parse(child.get("data") - .toString()).getAsJsonObject(); - if (childData.get("id").toString() != null - && !childData.get("quarantine").getAsBoolean() - && childData.get("url").getAsString() != null) { - String id = childData.get("id").toString(); - String title = childData.get("title") != null - ? childData.get("title").toString() - : this.sub.getName(); - title = title.replace("\"", "").replace("\\", "\""); - String author = childData.get("author") != null - ? childData.get("author").toString() : "anonymous"; - author = author.replace("\"", ""); - boolean quarantine = childData.get("quarantine") - .getAsBoolean(); - double score = childData.get("score").getAsDouble(); - String postHint = childData.get("post_hint").getAsString(); - boolean crosspostable = !childData.get("is_crosspostable") - .getAsBoolean(); - boolean over18 = childData.get("over_18").getAsBoolean(); - String url; - try { - JsonObject preview = new JsonParser().parse(childData - .get("preview").toString()).getAsJsonObject(); - JsonArray previewImages = new JsonParser().parse(preview - .get("images").toString()).getAsJsonArray(); - JsonObject source = new JsonParser().parse(previewImages - .get(0).toString()).getAsJsonObject(); - JsonObject urlSrc = new JsonParser().parse(source - .get("source").toString()).getAsJsonObject(); - url = urlSrc.get("url").toString().replace("amp;", "") - .replace("\"", ""); - } catch (NullPointerException n) { - url = childData.get("url").getAsString(); - } - String permalink = childData.get("permalink").getAsString(); - boolean spoiler = childData.get("spoiler").getAsBoolean(); + try { + JsonObject child = new JsonParser().parse(children.get(i) + .toString()).getAsJsonObject(); + JsonObject childData = new JsonParser().parse(child.get("data") + .toString()).getAsJsonObject(); + if (childData.get("id").toString() != null + && !childData.get("quarantine").getAsBoolean() + && childData.get("url").getAsString() != null) { + String id = childData.get("id").toString(); + String title = childData.get("title") != null + ? childData.get("title").toString() + : this.sub.getName(); + title = title.replace("\"", "").replace("\\", "\""); + String author = childData.get("author") != null + ? childData.get("author").toString() : "anonymous"; + author = author.replace("\"", ""); + boolean quarantine = childData.get("quarantine") + .getAsBoolean(); + double score = childData.get("score").getAsDouble(); + String postHint = childData.get("post_hint").getAsString(); + boolean crosspostable = !childData.get("is_crosspostable") + .getAsBoolean(); + boolean over18 = childData.get("over_18").getAsBoolean(); + String url; + try { + JsonObject preview = new JsonParser().parse(childData + .get("preview").toString()).getAsJsonObject(); + JsonArray previewImages = new JsonParser().parse(preview + .get("images").toString()).getAsJsonArray(); + JsonObject source = new JsonParser().parse(previewImages + .get(0).toString()).getAsJsonObject(); + JsonObject urlSrc = new JsonParser().parse(source + .get("source").toString()).getAsJsonObject(); + url = urlSrc.get("url").toString().replace("amp;", "") + .replace("\"", ""); + } catch (NullPointerException n) { + url = childData.get("url").getAsString(); + } + String permalink = childData.get("permalink").getAsString(); + boolean spoiler = childData.get("spoiler").getAsBoolean(); - if (postHint.contains("video")) { - set.add(new RedditPostVideo( - id, title, quarantine, score, postHint, - crosspostable, over18, author, - permalink, spoiler, url)); - } else if ("link".equals(postHint)) { - set.add(new RedditPostLink( - id, title, quarantine, score, postHint, - crosspostable, over18, author, - permalink, spoiler, url)); - } else if ("text".equals(postHint)) { - set.add(new RedditPostText( - id, title, quarantine, score, postHint, - crosspostable, over18, author, - permalink, spoiler, url)); - } else if ("image".equals(postHint)) { - set.add(new RedditPostImage( - id, title, quarantine, score, postHint, - crosspostable, over18, author, - permalink, spoiler, url)); + if (postHint.contains("video")) { + set.add(new RedditPostVideo( + id, title, quarantine, score, postHint, + crosspostable, over18, author, + permalink, spoiler, url)); + } else if ("link".equals(postHint)) { + set.add(new RedditPostLink( + id, title, quarantine, score, postHint, + crosspostable, over18, author, + permalink, spoiler, url)); + } else if ("text".equals(postHint)) { + set.add(new RedditPostText( + id, title, quarantine, score, postHint, + crosspostable, over18, author, + permalink, spoiler, url)); + } else if ("image".equals(postHint)) { + set.add(new RedditPostImage( + id, title, quarantine, score, postHint, + crosspostable, over18, author, + permalink, spoiler, url)); + } } + } catch (NullPointerException e) { + System.out.println( + "[*] There were a problem while parsing. " + + "Continuing"); } } return set;