From 6a89dfe70410b3104c8a0adfb902f61b077e713e Mon Sep 17 00:00:00 2001
From: gingermae <gingermae@deborah.deborah>
Date: Sat, 24 Oct 2026 12:34:46 -0400
Subject: [PATCH 1/5] third time's the charm

---
 shared_python/Parsers.py       | 26 +++++++++++++
 shared_python/Requests.py      | 40 +++++++++++++++++++
 xx-Test-Story-Content-Links.py | 70 ++++++++++++++++++++++++++++++++++
 3 files changed, 136 insertions(+)
 create mode 100644 shared_python/Parsers.py
 create mode 100644 shared_python/Requests.py
 create mode 100644 xx-Test-Story-Content-Links.py

diff --git a/shared_python/Parsers.py b/shared_python/Parsers.py
new file mode 100644
index 0000000..cdd4305
--- /dev/null
+++ b/shared_python/Parsers.py
@@ -0,0 +1,26 @@
+from bs4 import BeautifulSoup as bs
+
+
+class LinkParser:
+    def __init__(self, filetext):
+        self.filetext = filetext
+
+    def _is_relative(self, link):
+        if link.startswith(".") or link.startswith("/"):
+            return True
+        elif not link.startswith("http"):
+            return True
+        else:
+            return False
+
+    def extract_links(self):
+        soup = bs(self.filetext, "html.parser")
+        links = [
+            l.get("href")
+            for l in soup.find_all("a", href=True)
+            if not self._is_relative(l.get("href"))
+        ]
+        for i in soup.find_all("img"):
+            if not self._is_relative(i.get("src")):
+                links.append(i.get("src"))
+        return links
\ No newline at end of file
diff --git a/shared_python/Requests.py b/shared_python/Requests.py
new file mode 100644
index 0000000..5a7a531
--- /dev/null
+++ b/shared_python/Requests.py
@@ -0,0 +1,40 @@
+import requests
+
+
+class RequestHandler:
+    def __init__(self):
+        self.headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:92.0) Gecko/20100101 Firefox/92.0"
+        }
+        self.redirect_codes = [301, 302, 307, 308]
+
+    def _send_head(self, url):
+        try:
+            r = requests.head(url=url, headers=self.headers, timeout=10)
+        except requests.exceptions.Timeout:
+            r = "request timeout"
+        except Exception as e:
+            r = str(e)
+        return r
+
+    def status_code_handler(self, url):
+        return_dict = {
+            "status_code": "",
+            "redirect_location": "",
+            "redirect_response": "",
+        }
+        status = self._send_head(url)
+        if isinstance(status, str):
+            return_dict.update({"status_code": status})
+        else:
+            if status.status_code in self.redirect_codes:
+                return_dict.update({"redirect_location": status.headers["location"]})
+                redirect_status = self._send_head(status.headers["location"])
+                if isinstance(redirect_status, str):
+                    return_dict.update({"redirect_response": redirect_status})
+                else:
+                    return_dict.update(
+                        {"redirect_response": redirect_status.status_code}
+                    )
+            return_dict.update({"status_code": status.status_code})
+        return return_dict
diff --git a/xx-Test-Story-Content-Links.py b/xx-Test-Story-Content-Links.py
new file mode 100644
index 0000000..71e3689
--- /dev/null
+++ b/xx-Test-Story-Content-Links.py
@@ -0,0 +1,70 @@
+import csv
+import os
+
+from shared_python.Args import Args
+from shared_python.Common import print_progress, recursive_story_listdir
+from shared_python.Logging import logger
+from shared_python.Parsers import LinkParser
+from shared_python.Requests import RequestHandler
+
+
+args_obj = Args()
+args = args_obj.args_for_link_check()
+log = args_obj.logger_with_filename()
+handler = RequestHandler()
+
+story_files_folder = args.chapters_path
+output_folder = args.output_folder
+file_encoding = input(
+    "Enter encoding of the story files (leave blank for utf-8):\n"
+).lower()
+if file_encoding == "":
+    file_encoding = "utf-8"
+
+stories = recursive_story_listdir(story_files_folder)
+log.info(f"Preparing to parse through {len(stories)} stories")
+
+with open(f"{output_folder}/story_file_links.csv", "a") as f:
+    writer = csv.DictWriter(
+        f,
+        fieldnames=[
+            "story_id",
+            "link",
+            "response",
+            "redirect_location",
+            "redirect_response",
+        ],
+    )
+    cur = 0
+    for s in stories:
+        try:
+            with open(
+                f"{story_files_folder}/{s}", mode="r", encoding=file_encoding
+            ) as f:
+                story_text = f.read()
+            parser = LinkParser(story_text)
+            links = parser.extract_links()
+            for link in links:
+                response_meta = handler.status_code_handler(link)
+                writer.writerow(
+                    {
+                        "story_id": s.split(".")[0],
+                        "link": link,
+                        "response": response_meta["status_code"],
+                        "redirect_location": response_meta["redirect_location"],
+                        "redirect_response": response_meta["redirect_response"],
+                    }
+                )
+        except UnicodeDecodeError:
+            log.info(f"\nencoding error with story file {s}")
+            writer.writerow(
+                {
+                    "story_id": s.split(".")[0],
+                    "link": link,
+                    "response": "read error",
+                    "redirect_location": "",
+                    "redirect_response": "",
+                }
+            )
+        cur += 1
+        print_progress(cur, len(stories), "stories parsed")

From d6ab24cc53ebf670e985d72bc0133b426b4109c1 Mon Sep 17 00:00:00 2001
From: gingermae <gingermae@deborah.deborah>
Date: Sat, 24 Oct 2026 12:35:27 -0400
Subject: [PATCH 2/5] updated README

---
 README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.md b/README.md
index 92eebe3..04379ac 100644
--- a/README.md
+++ b/README.md
@@ -255,6 +255,12 @@ On it we can either type (and press enter):
 Given a comma-separated list of story ids specified in the `story_ids_to_remove` parameter, deletes the corresponding
 rows from the stories table in the final output database.
 
+### Test links in story files
+Note: at the very least, requires links in the story files to be formatted as html, as it uses BeautifulSoup and html parsing to detect links
+
+given chapters filepath and an output file path (which you can specify using the .yml file), writes a csv with each non-relative link, the returned status or error, and the status codes of any automatic redirections
+
+the script prompts the user for the chapter file encoding, so make sure you have run the mojibake tool beforehand!
 
 ## Parameters
 

From 5da2ae1b917ba81b98d7cce2d6982c6368a68b48 Mon Sep 17 00:00:00 2001
From: gingermae <gingermae@deborah.deborah>
Date: Sat, 24 Oct 2026 12:36:04 -0400
Subject: [PATCH 3/5] updated Args

---
 shared_python/Args.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/shared_python/Args.py b/shared_python/Args.py
index c041e71..78cceca 100644
--- a/shared_python/Args.py
+++ b/shared_python/Args.py
@@ -264,3 +264,20 @@ def args_for_08(self):
             )
             self._print_args(self.args)
         return self.args
+        
+    def args_for_link_check(self):
+        if self.args.chapters_path is None:
+            self.args.chapters_path = input(
+                "Location of the text files containing the stories:"
+            )
+        if not os.path.exists(self.args.output_folder):
+            os.makedirs(self.args.output_folder)
+        while self.args.output_folder is None or not os.path.isdir(
+            self.args.output_folder
+        ):
+            self.args.output_folder = input("Path for output files: ")
+            if not os.path.exists(self.args.output_folder):
+                os.makedirs(self.args.output_folder)
+        self._print_args(self.args)
+        return self.args
+

From 97cfb4a32c75fcea2aef2c22527e785acc53b8d8 Mon Sep 17 00:00:00 2001
From: gingermae <gingermae@deborah.deborah>
Date: Sat, 24 Oct 2026 12:36:31 -0400
Subject: [PATCH 4/5] updated Common

---
 shared_python/Common.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/shared_python/Common.py b/shared_python/Common.py
index 728b2e2..a9a6f81 100644
--- a/shared_python/Common.py
+++ b/shared_python/Common.py
@@ -12,3 +12,18 @@ def print_progress(cur, total, prog_type="stories"):
     sys.stdout.write(f"\r{cur}/{total} {prog_type}")
     sys.stdout.flush()
     return cur
+
+def recursive_story_listdir(main_path):
+    storyfiles = [
+        x.name for x in os.scandir(main_path) if "html" in x.name or "txt" in x.name
+    ]
+    subdirs = [x.name for x in os.scandir(main_path) if x.is_dir()]
+    for dir in subdirs:
+        stories = [
+            x
+            for x in os.listdir(os.path.join(main_path, dir))
+            if "html" in x or "txt" in x
+        ]
+        for s in stories:
+            storyfiles.append(os.path.join(dir, s))
+    return storyfiles
\ No newline at end of file

From 129bb0daae97ac1e1afd15c33cc6fe993bd4f8b5 Mon Sep 17 00:00:00 2001
From: gingermae <gingermae@deborah.deborah>
Date: Sat, 24 Oct 2026 12:36:45 -0400
Subject: [PATCH 5/5] ruff formatting

---
 shared_python/Args.py    | 3 +--
 shared_python/Common.py  | 3 ++-
 shared_python/Parsers.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/shared_python/Args.py b/shared_python/Args.py
index 78cceca..0d07a7e 100644
--- a/shared_python/Args.py
+++ b/shared_python/Args.py
@@ -264,7 +264,7 @@ def args_for_08(self):
             )
             self._print_args(self.args)
         return self.args
-        
+
     def args_for_link_check(self):
         if self.args.chapters_path is None:
             self.args.chapters_path = input(
@@ -280,4 +280,3 @@ def args_for_link_check(self):
                 os.makedirs(self.args.output_folder)
         self._print_args(self.args)
         return self.args
-
diff --git a/shared_python/Common.py b/shared_python/Common.py
index a9a6f81..03a86ba 100644
--- a/shared_python/Common.py
+++ b/shared_python/Common.py
@@ -13,6 +13,7 @@ def print_progress(cur, total, prog_type="stories"):
     sys.stdout.flush()
     return cur
 
+
 def recursive_story_listdir(main_path):
     storyfiles = [
         x.name for x in os.scandir(main_path) if "html" in x.name or "txt" in x.name
@@ -26,4 +27,4 @@ def recursive_story_listdir(main_path):
         ]
         for s in stories:
             storyfiles.append(os.path.join(dir, s))
-    return storyfiles
\ No newline at end of file
+    return storyfiles
diff --git a/shared_python/Parsers.py b/shared_python/Parsers.py
index cdd4305..05befde 100644
--- a/shared_python/Parsers.py
+++ b/shared_python/Parsers.py
@@ -23,4 +23,4 @@ def extract_links(self):
         for i in soup.find_all("img"):
             if not self._is_relative(i.get("src")):
                 links.append(i.get("src"))
-        return links
\ No newline at end of file
+        return links