added an 'undetermined' category of reddit posts as some subs don't have post_hint value, used to determine if a post is an image or a video for example. this fixes a but where all of these posts that don't have a post_hint value (quite a lot) would simply be ignored by the system

This commit is contained in:
Louis Vallat 2020-08-26 23:24:00 +02:00
parent 12b2a4f723
commit ec67b3d43a
9 changed files with 110 additions and 130 deletions

View File

@ -16,25 +16,18 @@
*/ */
package xyz.vallat.louis; package xyz.vallat.louis;
import xyz.vallat.louis.socialMediaHandler.SocialMediaPoster;
import xyz.vallat.louis.redditHandler.RedditExtractor;
import xyz.vallat.louis.exceptions.NoSuchFile; import xyz.vallat.louis.exceptions.NoSuchFile;
import xyz.vallat.louis.exceptions.NoSuchOrder; import xyz.vallat.louis.exceptions.NoSuchOrder;
import xyz.vallat.louis.exceptions.NotSufficientRights;
import xyz.vallat.louis.exceptions.NoSuchProperty; import xyz.vallat.louis.exceptions.NoSuchProperty;
import xyz.vallat.louis.exceptions.NotSufficientRights;
import xyz.vallat.louis.redditHandler.RedditExtractor;
import xyz.vallat.louis.redditHandler.RedditPost; import xyz.vallat.louis.redditHandler.RedditPost;
import java.io.File; import xyz.vallat.louis.socialMediaHandler.SocialMediaPoster;
import java.io.FileOutputStream;
import java.io.IOException; import java.io.*;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL; import java.net.URL;
import java.sql.*;
import java.util.ArrayList; import java.util.ArrayList;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
/** /**
* Singleton Hypervisor. This is the object that does everything. So we want it * Singleton Hypervisor. This is the object that does everything. So we want it
@ -111,8 +104,8 @@ public class Hypervisor {
this.workingDirectory = reader.getProperties("working_directory"); this.workingDirectory = reader.getProperties("working_directory");
setupTheBotDirectory(); setupTheBotDirectory();
this.connexion = DriverManager.getConnection("jdbc:sqlite:" this.connexion = DriverManager.getConnection("jdbc:sqlite:"
+ this.workingDirectory + File.separator + this.workingDirectory + File.separator
+ this.sqliteDatabase); + this.sqliteDatabase);
this.myRedditExtractor = new RedditExtractor(subreddit); this.myRedditExtractor = new RedditExtractor(subreddit);
if ("Y".equals(reader.getProperties("clear_database"))) { if ("Y".equals(reader.getProperties("clear_database"))) {
clearDatabase(); clearDatabase();
@ -127,7 +120,6 @@ public class Hypervisor {
* Get the Singleton Hypervisor instance. * Get the Singleton Hypervisor instance.
* *
* @return the instance. * @return the instance.
*
* @throws ClassNotFoundException * @throws ClassNotFoundException
* @throws NotSufficientRights * @throws NotSufficientRights
* @throws SQLException * @throws SQLException
@ -156,7 +148,7 @@ public class Hypervisor {
throws ClassNotFoundException, SQLException, throws ClassNotFoundException, SQLException,
InterruptedException { InterruptedException {
System.out.println("[+] Hypervisor is now running."); System.out.println("[+] Hypervisor is now running.");
for (;;) { for (; ; ) {
this.connexion = DriverManager.getConnection("jdbc:sqlite:" this.connexion = DriverManager.getConnection("jdbc:sqlite:"
+ this.workingDirectory + File.separator + this.workingDirectory + File.separator
+ this.sqliteDatabase); + this.sqliteDatabase);
@ -166,7 +158,7 @@ public class Hypervisor {
this.connexion.close(); this.connexion.close();
System.out.println( System.out.println(
"[*] Hypervisor is waiting for " "[*] Hypervisor is waiting for "
+ this.delay + " seconds."); + this.delay + " seconds.");
Thread.sleep(this.delay * 1000); Thread.sleep(this.delay * 1000);
} }
} }
@ -207,7 +199,6 @@ public class Hypervisor {
* Add a given reddit post to the database. * Add a given reddit post to the database.
* *
* @param current a given reddit post to add to the database * @param current a given reddit post to add to the database
*
* @throws ClassNotFoundException * @throws ClassNotFoundException
* @throws SQLException * @throws SQLException
*/ */
@ -217,29 +208,30 @@ public class Hypervisor {
if (!isInDatabase(current.getPostId())) { if (!isInDatabase(current.getPostId())) {
PreparedStatement ajout = this.connexion.prepareStatement( PreparedStatement ajout = this.connexion.prepareStatement(
"INSERT INTO " + this.tableName "INSERT INTO " + this.tableName
+ "(" + "("
+ "postType, " + "postType, "
+ "postId, " + "postId, "
+ "title, " + "title, "
+ "quarantine, " + "quarantine, "
+ "score, " + "score, "
+ "postHint, " + "postHint, "
+ "crosspostable, " + "crosspostable, "
+ "over18, " + "over18, "
+ "author, " + "author, "
+ "permalink, " + "permalink, "
+ "spoiler, " + "spoiler, "
+ "url, " + "url, "
+ "shared" + "shared"
+ ") " + ") "
+ "VALUES " + "VALUES "
+ "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);" + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"
); );
ajout.setString(1, ajout.setString(1,
current.isImage() ? "image" current.isImage() ? "image"
: current.isLink() ? "link" : current.isLink() ? "link"
: current.isText() ? "text" : current.isText() ? "text"
: "video"); : current.isVideo() ? "video"
: null);
ajout.setString(2, current.getPostId()); ajout.setString(2, current.getPostId());
ajout.setString(3, current.getTitle()); ajout.setString(3, current.getTitle());
ajout.setBoolean(4, current.isQuarantine()); ajout.setBoolean(4, current.isQuarantine());
@ -262,7 +254,6 @@ public class Hypervisor {
* *
* @param postId the post id * @param postId the post id
* @return if the post is in the database * @return if the post is in the database
*
* @throws SQLException * @throws SQLException
* @throws ClassNotFoundException * @throws ClassNotFoundException
*/ */
@ -270,7 +261,7 @@ public class Hypervisor {
throws SQLException, ClassNotFoundException { throws SQLException, ClassNotFoundException {
PreparedStatement recherche = this.connexion.prepareStatement( PreparedStatement recherche = this.connexion.prepareStatement(
"SELECT * FROM " + this.tableName + " " "SELECT * FROM " + this.tableName + " "
+ "WHERE postId = '" + postId + "'"); + "WHERE postId = '" + postId + "'");
try (ResultSet resultats = recherche.executeQuery()) { try (ResultSet resultats = recherche.executeQuery()) {
return resultats.next(); return resultats.next();
} }
@ -309,7 +300,6 @@ public class Hypervisor {
* Compute a given reddit post. * Compute a given reddit post.
* *
* @param r the reddit post to compute. * @param r the reddit post to compute.
*
* @throws ClassNotFoundException * @throws ClassNotFoundException
* @throws SQLException * @throws SQLException
*/ */
@ -426,7 +416,7 @@ public class Hypervisor {
if (!f.canRead() || !f.canWrite()) { if (!f.canRead() || !f.canWrite()) {
throw new NotSufficientRights( throw new NotSufficientRights(
"[!] This program does not have the sufficient " "[!] This program does not have the sufficient "
+ "rights on the folder \"" + workingDirectory + "\"."); + "rights on the folder \"" + workingDirectory + "\".");
} }
} }

View File

@ -62,7 +62,7 @@ public class RedditReposterBot {
System.out.println("[*] App version " + VERSION); System.out.println("[*] App version " + VERSION);
Hypervisor master = Hypervisor.getSingleton(); Hypervisor master = Hypervisor.getSingleton();
master.addSocialMedia(new TwitterBot()); //master.addSocialMedia(new TwitterBot());
master.run(); master.run();
} }
} }

View File

@ -28,6 +28,7 @@ import java.io.InputStreamReader;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.Arrays;
import java.util.HashSet; import java.util.HashSet;
/** /**
@ -46,7 +47,6 @@ public final class RedditExtractor {
* Main Constructor. * Main Constructor.
* *
* @param subreddit Subreddit name. Just after /r/ * @param subreddit Subreddit name. Just after /r/
*
* @throws IOException * @throws IOException
* @throws NoSuchProperty * @throws NoSuchProperty
* @throws NoSuchFile * @throws NoSuchFile
@ -74,8 +74,8 @@ public final class RedditExtractor {
System.out.println("[*] Checking if subreddit /r/" + subredditName System.out.println("[*] Checking if subreddit /r/" + subredditName
+ " exists."); + " exists.");
return ((JsonParser.parseString(getJsonFromURL("" return ((JsonParser.parseString(getJsonFromURL(""
+ "https://www.reddit.com/api/search_reddit_names.json" + "https://www.reddit.com/api/search_reddit_names.json"
+ "?query=" + subredditName + "&exact=true")).getAsJsonObject() + "?query=" + subredditName + "&exact=true")).getAsJsonObject()
.get("names").getAsJsonArray() .get("names").getAsJsonArray()
.size()) >= 1); .size()) >= 1);
} }
@ -162,14 +162,15 @@ public final class RedditExtractor {
.replace("&", "&"); .replace("&", "&");
String author = childData.get("author") != null String author = childData.get("author") != null
? childData.get("author").toString() ? childData.get("author").toString()
.replace("&lt;", "<") .replace("&lt;", "<")
.replace("&gt;", ">") .replace("&gt;", ">")
.replace("&amp;", "&") : "anonymous"; .replace("&amp;", "&") : "anonymous";
author = author.replace("\"", ""); author = author.replace("\"", "");
boolean quarantine = childData.get("quarantine") boolean quarantine = childData.get("quarantine")
.getAsBoolean(); .getAsBoolean();
double score = childData.get("score").getAsDouble(); double score = childData.get("score").getAsDouble();
String postHint = childData.get("post_hint").getAsString(); JsonElement postHintElement = childData.get("post_hint");
String postHint = postHintElement == null ? "" : postHintElement.getAsString();
boolean crosspostable = !childData.get("is_crosspostable") boolean crosspostable = !childData.get("is_crosspostable")
.getAsBoolean(); .getAsBoolean();
boolean over18 = childData.get("over_18").getAsBoolean(); boolean over18 = childData.get("over_18").getAsBoolean();
@ -190,7 +191,6 @@ public final class RedditExtractor {
} }
String permalink = childData.get("permalink").getAsString(); String permalink = childData.get("permalink").getAsString();
boolean spoiler = childData.get("spoiler").getAsBoolean(); boolean spoiler = childData.get("spoiler").getAsBoolean();
if (postHint.contains("video")) { if (postHint.contains("video")) {
set.add(new RedditPostVideo( set.add(new RedditPostVideo(
id, title, quarantine, score, postHint, id, title, quarantine, score, postHint,
@ -211,12 +211,16 @@ public final class RedditExtractor {
id, title, quarantine, score, postHint, id, title, quarantine, score, postHint,
crosspostable, over18, author, crosspostable, over18, author,
permalink, spoiler, url)); permalink, spoiler, url));
} else {
set.add(new RedditPostUndetermined(id, title, quarantine, score, postHint,
crosspostable, over18, author,
permalink, spoiler, url));
} }
} }
} catch (NullPointerException e) { } catch (NullPointerException e) {
System.out.println( System.out.println(
"[*] There were a problem while parsing. " "[*] There were a problem while parsing. "
+ "Continuing"); + "Continuing");
} }
} }
return set; return set;

View File

@ -81,21 +81,21 @@ public abstract class RedditPost {
/** /**
* Main constructor for a Reddit post. * Main constructor for a Reddit post.
* *
* @param id post's id * @param id post's id
* @param title post's title * @param title post's title
* @param quarantine is this post in quarantine? * @param quarantine is this post in quarantine?
* @param score post's score * @param score post's score
* @param postHint post's hint * @param postHint post's hint
* @param crosspostable is post crosspostable? * @param crosspostable is post crosspostable?
* @param over18 is post NSFW? * @param over18 is post NSFW?
* @param author post's author * @param author post's author
* @param permalink post's permalink * @param permalink post's permalink
* @param spoiler is this post a spoiler? * @param spoiler is this post a spoiler?
* @param url post's media url * @param url post's media url
*/ */
public RedditPost(String id, String title, boolean quarantine, double score, public RedditPost(String id, String title, boolean quarantine, double score,
String postHint, boolean crosspostable, boolean over18, String postHint, boolean crosspostable, boolean over18,
String author, String permalink, boolean spoiler, String url) { String author, String permalink, boolean spoiler, String url) {
this.postId = id; this.postId = id;
this.title = title; this.title = title;
this.quarantine = quarantine; this.quarantine = quarantine;
@ -240,26 +240,40 @@ public abstract class RedditPost {
* *
* @return if the post is an image. * @return if the post is an image.
*/ */
public abstract boolean isImage(); public boolean isImage() {
return false;
}
;
/** /**
* Is this post a text? * Is this post a text?
* *
* @return if the post is a text. * @return if the post is a text.
*/ */
public abstract boolean isText(); public boolean isText() {
return false;
}
;
/** /**
* Is this post a video? * Is this post a video?
* *
* @return if the post is a video. * @return if the post is a video.
*/ */
public abstract boolean isVideo(); public boolean isVideo() {
return false;
}
/** /**
* Is this post a link? * Is this post a link?
* *
* @return if the post is a link. * @return if the post is a link.
*/ */
public abstract boolean isLink(); public boolean isLink() {
return false;
}
;
} }

View File

@ -35,19 +35,4 @@ public class RedditPostImage extends RedditPost {
public boolean isImage() { public boolean isImage() {
return true; return true;
} }
@Override
public boolean isText() {
return false;
}
@Override
public boolean isVideo() {
return false;
}
@Override
public boolean isLink() {
return false;
}
} }

View File

@ -31,21 +31,6 @@ public class RedditPostLink extends RedditPost {
crosspostable, over18, author, permalink, spoiler, url); crosspostable, over18, author, permalink, spoiler, url);
} }
@Override
public boolean isImage() {
return false;
}
@Override
public boolean isText() {
return false;
}
@Override
public boolean isVideo() {
return false;
}
@Override @Override
public boolean isLink() { public boolean isLink() {
return true; return true;

View File

@ -31,23 +31,8 @@ public class RedditPostText extends RedditPost {
crosspostable, over18, author, permalink, spoiler, url); crosspostable, over18, author, permalink, spoiler, url);
} }
@Override
public boolean isImage() {
return false;
}
@Override @Override
public boolean isText() { public boolean isText() {
return true; return true;
} }
@Override
public boolean isVideo() {
return false;
}
@Override
public boolean isLink() {
return false;
}
} }

View File

@ -0,0 +1,33 @@
/*
* Copyright (C) 2019 louis
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package xyz.vallat.louis.redditHandler;
/**
* Reddit image post object representation.
*
* @author louis
*/
public class RedditPostUndetermined extends RedditPost {
public RedditPostUndetermined(String id, String title, boolean quarantine,
double score, String postHint, boolean crosspostable,
boolean over18, String author, String permalink,
boolean spoiler, String url) {
super(id, title, quarantine, score, postHint,
crosspostable, over18, author, permalink, spoiler, url);
}
}

View File

@ -31,24 +31,8 @@ public class RedditPostVideo extends RedditPost {
crosspostable, over18, author, permalink, spoiler, url); crosspostable, over18, author, permalink, spoiler, url);
} }
@Override
public boolean isImage() {
return false;
}
@Override
public boolean isText() {
return false;
}
@Override @Override
public boolean isVideo() { public boolean isVideo() {
return true; return true;
} }
@Override
public boolean isLink() {
return false;
}
} }