diff --git a/04-Rename-Tags.py b/04-Rename-Tags.py index 2316ca5..a0f90a0 100755 --- a/04-Rename-Tags.py +++ b/04-Rename-Tags.py @@ -22,7 +22,6 @@ tw_tags = list(csv.DictReader(csvfile)) tag_headers = tags.tag_export_map total = len(tw_tags) - for cur, row in enumerate(tw_tags): tags.update_tag_row(row) print_progress(cur, total, "tags") diff --git a/README.md b/README.md index 92eebe3..fd7f879 100755 --- a/README.md +++ b/README.md @@ -255,6 +255,17 @@ On it we can either type (and press enter): Given a comma-separated list of story ids specified in the `story_ids_to_remove` parameter, deletes the corresponding rows from the stories table in the final output database. +## Extract notes, summaries, and commercial links from chapters + +Extracts notes, summaries, content warnings, and commercial links from story files + +Due to the nature of fandom and archives, most are going to be different and have unique keywords. If your archive has additional keywords, you can enter them into a plaintext file for each category and add them to the .yml configuration file (see example.yml) + +This is mostly for non-eFiction archives where the author's notes and content warnings are present in the main bodies of work instead of separate fields. If you use this with the 'remove_option' in the properties file populated, the script will create a backup of all works. If you hit an error or KeyboardInterrupt, the script prints out the index of the chapters list you were at. + +note that this does NOT load chapters into the stories table + + ## Parameters diff --git a/example.yml b/example.yml index 25e0e67..a0e218b 100755 --- a/example.yml +++ b/example.yml @@ -28,3 +28,21 @@ bookmark_ids_to_remove: /path/to/ids to remove.txt # Chapter file location chapters_path: /path/to/chapter/files/stories chapters_file_extensions: txt, html + + + +# options for story cleanup +# leave blank to ignore story files; any value here will remove flagged text from the original +remove_option: +chapters_backup_path: /path/to/backup/chapter/story/output/files +output_csv: /path/to/output/Desired_CSV_name.csv +# Comma-separated list of types to scan for +scan_types: [notes, warnings, summary, commercial, pairing, rating, wordcount] +additional_keywords: + - notes: path/to/additional_notes_keywords.txt + - warnings: path/to/additional_warnings_keywords.txt + - summary: path/to/additional_summary_keywords.txt + - commercial: + - pairing: + - rating: + - wordcount: \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 476d90d..d81daee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ pyyaml==6.0.1 colorlog==6.7.0 prompt_toolkit==3.0.39 ruff==0.1.5 +colorama==0.4.6 \ No newline at end of file diff --git a/shared_python/Args.py b/shared_python/Args.py index 2fa2b8f..94e6599 100755 --- a/shared_python/Args.py +++ b/shared_python/Args.py @@ -263,3 +263,35 @@ def args_for_08(self): ) self._print_args(self.args) return self.args + + def args_for_sn_extraction(self): + if self.args.chapters_path is None: + self.args.chapters_path = input( + "Location of the text files containing the stories:" + ) + if self.args.remove_option is not None: + if self.args.chapters_backup_path is None: + self.args.chapters_backup_path = input( + "Desired location (full path) of the original text backup" + ) + if os.path.exists(self.args.chapters_backup_path): + os.rmdir(self.args.chapters_backup_path) + + if self.args.output_csv is None: + self.args.output_csv = input("Output path for the extracted CSV") + if self.args.scan_types is None: + self.args.scan_types = [ + "notes", + "warnings", + "summary", + "commercial", + "pairing", + "rating", + "wordcount", + ] + + if self.args.additional_keywords is not None: + self.args.additional_keywords = args.additional_keywords + + self._print_args(self.args) + return self.args diff --git a/shared_python/Common.py b/shared_python/Common.py index 1214162..c8b2509 100755 --- a/shared_python/Common.py +++ b/shared_python/Common.py @@ -1,6 +1,7 @@ # -- coding: utf-8 -- -import sys from importlib import reload +import os +import sys reload(sys) # sys.setdefaultencoding('utf8') #setdefaultencoding is disabled in Python 3. UTF-8 is also default coding. @@ -13,3 +14,19 @@ def print_progress(cur, total, prog_type="stories"): sys.stdout.write("\r{0}/{1} {2}".format(cur, total, prog_type)) sys.stdout.flush() return cur + + +def recursive_story_listdir(main_path): + storyfiles = [ + x.name for x in os.scandir(main_path) if "html" in x.name or "txt" in x.name + ] + subdirs = [x.name for x in os.scandir(main_path) if x.is_dir()] + for dir in subdirs: + stories = [ + x + for x in os.listdir(os.path.join(main_path, dir)) + if "html" in x or "txt" in x + ] + for s in stories: + storyfiles.append(os.path.join(dir, s)) + return storyfiles diff --git a/story_cleanup/keywords.py b/story_cleanup/keywords.py new file mode 100644 index 0000000..b2202bf --- /dev/null +++ b/story_cleanup/keywords.py @@ -0,0 +1,73 @@ +class Keywords: + def __init__(self, additional_keywords={}): + self.notes = ["note", "a/n", "disclaimer", "tumblr", "author"] + self.warnings = [ + "warning", + "cw", + "content warning", + "trigger warning", + "tw", + ] + self.summary = ["summary", "summaries", "prompt", "synopsis"] + self.commercial = [ + "patreon", + "ko-fi", + "kofundme", + ] + self.rating = ["rating", "rated"] + self.wordcount = [ + "words", + "words", + "wordcount", + "word count", + ] + self.pairings = ["pairing", "relationship"] + + self.additional_keywords = additional_keywords + + def _open_file_and_read(self, keywordfile): + with open(keywordfile, "r") as f: + words = f.readlines() + return words + + def append_additional_keywords(self): + for t in self.additional_keywords.keys(): + words = self._open_file_and_read(self.additional_keywords[t]) + match t: + case "notes": + self.notes.append(words) + case "warnings": + self.warnings.append(words) + case "summary": + self.summary.append(words) + case "rating": + self.ratings.append(words) + case "wordcount": + self.wordcount.append(words) + case "pairings": + self.pairings.append(words) + + def detect(self, type, text): + match type: + case "notes": + keywords = self.notes + case "warnings": + keywords = self.warnings + case "summary": + keywords = self.summary + case "commercial": + keywords = self.commercial + case "rating": + keywords = self.ratings + case "wordcount": + keywords = self.wordcount + case "pairings": + keywords = self.pairings + case _: + raise NameError( + "unsupported type: must be 'notes', 'warnings', 'summary', 'rating', 'pairings', 'wordcount', or 'commercial'; check config file" + ) + for k in keywords: + if k in text.lower(): + return True + return False diff --git a/story_cleanup/messages.py b/story_cleanup/messages.py new file mode 100644 index 0000000..6335957 --- /dev/null +++ b/story_cleanup/messages.py @@ -0,0 +1,48 @@ +from colorama import Fore, Style + + +class Messaging: + """ + class that handles interactivity, message colors, etc + """ + + def __init__(self, text): + self.text = text + self.summary_color = Fore.BLUE + self.notes_color = Fore.MAGENTA + self.warning_color = Fore.LIGHTBLUE_EX + self.commercial_color = Fore.RED + self.additional_lines_color = Fore.LIGHTBLACK_EX + self.general = Fore.GREEN + self.reset = Style.RESET_ALL + + def check_init_response(self, type): + match type: + case "summary": + print(self.summary_color + f"{type}:\n" + self.reset + self.text) + case "notes" | "rating" | "wordcount" | "pairing" | "warnings": + print(self.notes_color + f"{type}:\n" + self.reset + self.text) + case "commercial": + print(self.commercial_color + f"{type}:\n" + self.reset + self.text) + r = input( + self.general + + "Enter 'n' if this is not correct, otherwise press any key to continue: \n" + + self.reset + ) + if r.lower() == "n": + return False + return True + + def check_additional_lines(self, preview): + print(f"{self.text}\n") + print(self.additional_lines_color + preview + self.reset) + r = input( + ( + self.general + + "Enter 'n' if you would like to add the next line to the string identified and removed, otherwise press any key to continue: \n" + + self.reset + ) + ) + if r == "n": + return True + return False diff --git a/story_cleanup/outputs.py b/story_cleanup/outputs.py new file mode 100644 index 0000000..42e75d8 --- /dev/null +++ b/story_cleanup/outputs.py @@ -0,0 +1,35 @@ +import csv +import os + + +class OutputCSV: + def __init__(self, filename: str, columns: list): + self.filename = filename + self.columns = columns + ["story_identifier"] + + def init_csv(self): + with open(self.filename, "w", encoding="utf_8_sig", newline="") as fp: + csvf = csv.DictWriter(fp, fieldnames=self.columns) + csvf.writeheader() + + def write_data(self, data: dict): + with open(self.filename, "a", encoding="utf_8_sig", newline="") as f: + csvf = csv.DictWriter(f, fieldnames=self.columns) + csvf.writerow(data) + + +class OutputStoryFiles: + def __init__(self, output_path): + self.output_path = output_path + + def strip_and_rewrite(self, text, options: dict): + text = "".join(text) + for o in options.keys(): + hit_lines = options[o].split("\n") + for h in hit_lines: + text = text.replace(h, "") + return text + + def out_to_file(self, filename, strippedtext): + with open(os.path.join(self.output_path, filename), "w") as f: + f.write(strippedtext) diff --git a/story_cleanup/parsing.py b/story_cleanup/parsing.py new file mode 100644 index 0000000..7663b11 --- /dev/null +++ b/story_cleanup/parsing.py @@ -0,0 +1,41 @@ +from .keywords import Keywords +from .messages import Messaging + + +class Parser: + def __init__(self, keywords, textlines: list, options: list): + self.textlines = textlines + self.max = len(self.textlines) + self.options_dict = {} + for opt in options: + self.options_dict.update({opt: ""}) + self.end_message = "***no more lines to preview, press enter to continue***" + self.keywords = keywords + + def detection_loop(self, type: str, i: int): + m = Messaging(self.textlines[i]) + if m.check_init_response(type): + local_count = i + 1 + txt = self.textlines[i] + try: + a = m.check_additional_lines(self.textlines[local_count]) + except IndexError: + input(self.end_message) + return txt + while a and local_count < self.max: + preview = self.textlines[local_count + 1] + txt = txt + self.textlines[local_count] + a = m.check_additional_lines(preview) + local_count += 1 + return txt + return "" + + def parse_lines(self): + # this is per story file + for i in range(0, int(self.max)): + for o in self.options_dict.keys(): + if self.kwords.detect(o, self.textlines[i]): + self.options_dict[o] = ( + self.options_dict[o] + "\n" + self.detection_loop(o, i) + ) + return self.options_dict diff --git a/xx-Extract-Notes-Summaries-Other.py b/xx-Extract-Notes-Summaries-Other.py new file mode 100644 index 0000000..36131d5 --- /dev/null +++ b/xx-Extract-Notes-Summaries-Other.py @@ -0,0 +1,63 @@ +import os +import shutil + +from shared_python.Args import Args +from shared_python.Common import recursive_story_listdir + +from story_cleanup.outputs import OutputCSV, OutputStoryFiles +from story_cleanup.parsing import Parser + +""" +For archives where authors' notes and summaries are in the main body of text, this script scans for and removes them +note that this does /not/ load fields into the database tables, only extracts them to a CSV (and deletes if necessary) +""" + +if __name__ == "__main__": + args_obj = Args() + args = args_obj.args_for_sn_extraction() + stories = recursive_story_listdir(args.chapters_path) + log = args_obj.logger_with_filename() + + options = args.scan_types + to_remove = args.remove_option + additional_keywords = args.additional_keywords + + csv = OutputCSV(args.output_csv, options) + j = int( + input("the index you left off at, if you are starting from scratch enter 0\n") + ) + + if to_remove is not None and j == 0: + shutil.copytree(args.chapters_path, args.chapters_backup_path) + rewrite = OutputStoryFiles(args.chapters_path) + + if additional_keywords: + keywords = Keywords(additional_keywords) + else: + keywords = Keywords() + + try: + # TODO: function that queries db for chapter and filenames + if j == 0: + csv.init_csv() + for s in stories[j:]: + log.info(f"story: {s}") + fname = os.path.join(args.chapters_path, s) + with open(fname, "r") as f: + text = f.readlines() + parser = Parser(keywords, text, options) + hits = parser.parse_lines() + + if to_remove is not None: + new_text = rewrite.strip_and_rewrite(text, hits) + rewrite.out_to_file(s, new_text) + + hits.update({"story_identifier": s}) + csv.write_data(hits) + j += 1 + + except KeyboardInterrupt: + print(f"progress at restart: {j}") + except Exception as e: + print(f"there's been an error: {e}\n") + print(f"progress at restart: {j}")